From bdee8ab3c11b360c3d4f57d52e297acc906f0f36 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 12 May 2023 19:00:26 +0800 Subject: [PATCH 01/42] enhance: save work and sync home/office --- source/libs/planner/src/planOptimizer.c | 6 +- source/libs/scalar/src/filter.c | 156 +++++++++++++++++++++++- 2 files changed, 159 insertions(+), 3 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 4f8b57de5f..a6d4898bc8 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2630,11 +2630,13 @@ static bool tbCntScanOptIsEligibleConds(STbCntScanOptInfo* pInfo, SNode* pCondit } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pConditions)) { - return tbCntScanOptIsEligibleLogicCond(pInfo, (SLogicConditionNode*)pConditions); + return tbCntScanOptIsEligibleLogicCond(pInfo, (SLogicConditionNode*)pConditions) && + LIST_LENGTH(pInfo->pAgg->pGroupKeys) == 0; } if (QUERY_NODE_OPERATOR == nodeType(pConditions)) { - return tbCntScanOptIsEligibleOpCond((SOperatorNode*)pConditions); + return tbCntScanOptIsEligibleOpCond((SOperatorNode*)pConditions) && + LIST_LENGTH(pInfo->pAgg->pGroupKeys) == 0; } return false; diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 02a21b66ed..77534db411 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3942,8 +3942,162 @@ _return: FLT_RET(code); } +// compare ranges, null < min < val < max. null=null, min=min, max=max +typedef enum { + FLT_SCL_DATUM_KIND_NULL, + FLT_SCL_DATUM_KIND_MIN, + FLT_SCL_DATUM_KIND_INT64, + FLT_SCL_DATUM_KIND_UINT64, + FLT_SCL_DATUM_KIND_FLOAT64, + FLT_SCL_DATUM_KIND_VARCHAR, + FLT_SCL_DATUM_KIND_NCHAR, + FLT_SCL_DATUM_KIND_MAX, +} SFltSclDatumKind; + +typedef struct { + SFltSclDatumKind kind; + union { + int64_t val; // may be int64, uint64 and double + struct { + uint32_t nData; + uint8_t *pData; + }; // maybe varchar, nchar + } datum; +} SFltSclDatum; + +typedef struct { + SFltSclDatum val; + bool excl; + bool start; +} SFltSclPoint; + +typedef struct { + SFltSclDatum low; + SFltSclDatum high; + bool lowExcl; + bool highExcl; +} SFltSclRange; + +int32_t fltSclCompareDatum(SFltSclDatum* val1, SFltSclDatum* val2) { +} + +bool fltSclLessPoint(SFltSclPoint* pt1, SFltSclPoint* pt2) { + // first value compare + int32_t cmp = fltSclCompareDatum(&pt1->val, &pt2->val); + if (cmp != 0) { + return cmp < 0 ; + } + + if (pt1->start && pt2->start) { + return !pt1->excl && pt2->excl; + } else if (pt1->start) { + return pt1->excl && !pt2->excl; + } else if (pt2->start) { + return pt1->excl || pt2->excl; + } + return pt1->excl && !pt2->excl; +} + +int32_t fltSclMergeSort(SArray* pts1, SArray* pts2, SArray* result) { + size_t len1 = taosArrayGetSize(pts1); + size_t len2 = taosArrayGetSize(pts2); + size_t i = 0; + size_t j = 0; + while (i < len1 && j < len2) { + SFltSclPoint* pt1 = taosArrayGet(pts1, i); + SFltSclPoint* pt2 = taosArrayGet(pts2, j); + bool less = fltSclLessPoint(pt1, pt2); + if (less) { + taosArrayPush(result, pt1); + ++i; + } else { + taosArrayPush(result, pt2); + ++j; + } + } + if (i < len1) { + for (; i < len1; ++i) { + SFltSclPoint* pt1 = taosArrayGet(pts1, i); + taosArrayPush(result, pt1); + } + } + if (j < len2) { + for (; j < len2; ++j) { + SFltSclPoint *pt2 = taosArrayGet(pts2, j); + taosArrayPush(result, pt2); + } + } + return 0; +} + +// pts1 and pts2 must be ordered and de-duplicated and each range can not be a range of another range +int32_t fltSclMerge(SArray* pts1, SArray* pts2, bool isUnion, SArray* merged) { + size_t len1 = taosArrayGetSize(pts1); + size_t len2 = taosArrayGetSize(pts2); + //first merge sort pts1 and pts2 + SArray* all = taosArrayInit(len1 + len2, sizeof(SFltSclPoint)); + fltSclMergeSort(pts1, pts2, all); + int32_t countRequired = (isUnion) ? 1 : 2; + int32_t count = 0; + for (int32_t i = 0; i < taosArrayGetSize(all); ++i) { + SFltSclPoint* pt = taosArrayGet(pts1, i); + if (pt->start) { + ++count; + if (count == countRequired) { + taosArrayPush(merged, pt); + } + } else { + if (count == countRequired) { + taosArrayPush(merged, pt); + } + --count; + } + } + taosArrayDestroy(all); + return 0; +} + + +int32_t fltSclIntersect(SArray* pts1, SArray* pts2, SArray* merged) { + return fltSclMerge(pts1, pts2, false, merged); +} + +int32_t fltSclUnion(SArray* pts1, SArray* pts2, SArray* merged) { + return fltSclMerge(pts1, pts2, true, merged); +} + + +typedef struct { + SColumnNode* colNode; + SValueNode* valNode; + EOperatorType type; +} SFltSclScalarFunc; + +typedef struct { + SColumnNode* colNode; + +} SFltSclConstColumn; + +typedef struct { + SValueNode* value; +} SFltSclConstant; + +typedef struct { + bool useRange; + SHashObj* colRanges; + +} SFltSclOptCtx; + +static EDealRes fltSclMayBeOptimed(SNode* pNode, void* pCtx) { + SFltSclOptCtx* ctx = (SFltSclOptCtx*)pCtx; + +} + + int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) { - // TODO + SFltSclOptCtx ctx; + nodesWalkExprPostOrder(*pNode, fltSclMayBeOptimed, &ctx); + return TSDB_CODE_SUCCESS; } From 51fa363fd50e148ec730f4b1814e1957caa41a4b Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 15 May 2023 11:34:05 +0800 Subject: [PATCH 02/42] feature: save work --- source/libs/scalar/src/filter.c | 57 +++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 77534db411..8c3e97ccbc 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3957,12 +3957,13 @@ typedef enum { typedef struct { SFltSclDatumKind kind; union { - int64_t val; // may be int64, uint64 and double + int64_t val; // for int64, uint64 and double and bool (1 true, 0 false) struct { - uint32_t nData; + uint32_t nData; // TODO maybe remove this and make pData len prefixed? uint8_t *pData; - }; // maybe varchar, nchar + }; // for varchar, nchar } datum; + SDataType type; // TODO: original data type, may not be used? } SFltSclDatum; typedef struct { @@ -3979,6 +3980,42 @@ typedef struct { } SFltSclRange; int32_t fltSclCompareDatum(SFltSclDatum* val1, SFltSclDatum* val2) { + switch (val1->kind) { + case FLT_SCL_DATUM_KIND_NULL: { + if (val2->kind == FLT_SCL_DATUM_KIND_NULL) return 0 ; else return -1; + break; + } + case FLT_SCL_DATUM_KIND_MIN: { + if (val2->kind == FLT_SCL_DATUM_KIND_NULL) return 1; + else { + if (val2->kind == FLT_SCL_DATUM_KIND_MIN) return 0; else return -1; + } + break; + } + case FLT_SCL_DATUM_KIND_MAX: { + if (val2->kind == FLT_SCL_DATUM_KIND_MAX) return 0; else return 1; + } + case FLT_SCL_DATUM_KIND_UINT64: { + return compareUint64Val(&val1->datum.val, &val1->datum.val); + } + case FLT_SCL_DATUM_KIND_INT64: { + return compareInt64Val(&val1->datum.val, &val2->datum.val); + } + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareFloatDouble(&val1->datum.val, &val2->datum.val); + } + case FLT_SCL_DATUM_KIND_NCHAR: { + return compareLenPrefixedWStr(&val1->datum.pData, &val2->datum.pData); //TODO + } + case FLT_SCL_DATUM_KIND_VARCHAR: { + return compareLenPrefixedStr(&val1->datum.pData, &val2->datum.pData); //TODO + } + default: + fltError("not supported kind. just return 0"); + return 0; + break; + } + return 0; } bool fltSclLessPoint(SFltSclPoint* pt1, SFltSclPoint* pt2) { @@ -4071,26 +4108,32 @@ typedef struct { SColumnNode* colNode; SValueNode* valNode; EOperatorType type; -} SFltSclScalarFunc; +} SFltSclScalarOp; +// simple define it, TODO: implement later typedef struct { SColumnNode* colNode; - } SFltSclConstColumn; +// simple define it, TODO: implement later typedef struct { SValueNode* value; } SFltSclConstant; + +int32_t fltSclBuildRangeFromScalarFunc(SFltSclScalarOp* func, SFltSclPoint* points) { + +} + + typedef struct { bool useRange; SHashObj* colRanges; - } SFltSclOptCtx; static EDealRes fltSclMayBeOptimed(SNode* pNode, void* pCtx) { SFltSclOptCtx* ctx = (SFltSclOptCtx*)pCtx; - + return DEAL_RES_CONTINUE; } From e16f4c7c440ae02905e18b397eb985aa3b91f668 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 17 May 2023 16:57:09 +0800 Subject: [PATCH 03/42] fix: refactor and save work --- source/libs/scalar/src/filter.c | 220 ++++++++++++++++++-------------- 1 file changed, 124 insertions(+), 96 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 8c3e97ccbc..85974bcd3a 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -22,6 +22,7 @@ #include "sclInt.h" #include "tcompare.h" #include "tdatablock.h" +#include "tsimplehash.h" #include "ttime.h" bool filterRangeCompGi(const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { @@ -260,7 +261,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { comparFn = 19; } else if (optr == OP_TYPE_NMATCH) { comparFn = 20; - } else if (optr == OP_TYPE_LIKE) { /* wildcard query using like operator */ + } else if (optr == OP_TYPE_LIKE) { /* wildcard query using like operator */ comparFn = 7; } else if (optr == OP_TYPE_NOT_LIKE) { /* wildcard query using like operator */ comparFn = 26; @@ -1633,7 +1634,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) SDataType *dType = &var->node.resType; qDebug("VAL%d => [type:%d][val:%" PRIx64 "]", i, dType->type, var->datum.i); // TODO } else if (field->data) { - qDebug("VAL%d => [type:NIL][val:NIL]", i); // TODO + qDebug("VAL%d => [type:NIL][val:NIL]", i); // TODO } } @@ -3957,59 +3958,96 @@ typedef enum { typedef struct { SFltSclDatumKind kind; union { - int64_t val; // for int64, uint64 and double and bool (1 true, 0 false) - struct { - uint32_t nData; // TODO maybe remove this and make pData len prefixed? - uint8_t *pData; - }; // for varchar, nchar + int64_t val; // for int64, uint64 and double and bool (1 true, 0 false) + uint8_t *pData; // for varchar, nchar } datum; - SDataType type; // TODO: original data type, may not be used? + SDataType type; // TODO: original data type, may not be used? } SFltSclDatum; typedef struct { SFltSclDatum val; - bool excl; - bool start; + bool excl; + bool start; } SFltSclPoint; typedef struct { SFltSclDatum low; SFltSclDatum high; - bool lowExcl; - bool highExcl; + bool lowExcl; + bool highExcl; } SFltSclRange; -int32_t fltSclCompareDatum(SFltSclDatum* val1, SFltSclDatum* val2) { +int32_t fltSclCompareWithFloat64(SFltSclDatum *val1, SFltSclDatum *val2) { + // val2->kind == int64 switch (val1->kind) { - case FLT_SCL_DATUM_KIND_NULL: { - if (val2->kind == FLT_SCL_DATUM_KIND_NULL) return 0 ; else return -1; - break; + case FLT_SCL_DATUM_KIND_UINT64: + return compareUint64Double(&val1->datum.val, &val2->datum.val); + case FLT_SCL_DATUM_KIND_INT64: + return compareInt64Double(&val1->datum.val, &val2->datum.val); + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareDoubleVal(&val1->datum.val, &val2->datum.val); } - case FLT_SCL_DATUM_KIND_MIN: { - if (val2->kind == FLT_SCL_DATUM_KIND_NULL) return 1; - else { - if (val2->kind == FLT_SCL_DATUM_KIND_MIN) return 0; else return -1; - } - break; + //TODO: varchar, nchar + default: + qError("not support comparsion. %d, %d", val1->kind, val2->kind); + return (val1->kind - val2->kind); + } +} + +int32_t fltSclCompareWithInt64(SFltSclDatum *val1, SFltSclDatum *val2) { + // val2->kind == int64 + switch (val1->kind) { + case FLT_SCL_DATUM_KIND_UINT64: + return compareUint64Int64(&val1->datum.val, &val2->datum.val); + case FLT_SCL_DATUM_KIND_INT64: + return compareInt64Val(&val1->datum.val, &val2->datum.val); + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareDoubleInt64(&val1->datum.val, &val2->datum.val); } - case FLT_SCL_DATUM_KIND_MAX: { - if (val2->kind == FLT_SCL_DATUM_KIND_MAX) return 0; else return 1; + //TODO: varchar, nchar + default: + qError("not support comparsion. %d, %d", val1->kind, val2->kind); + return (val1->kind - val2->kind); + } +} + +int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { + // val2 kind == uint64 + switch (val1->kind) { + case FLT_SCL_DATUM_KIND_UINT64: + return compareUint64Val(&val1->datum.val, &val2->datum.val); + case FLT_SCL_DATUM_KIND_INT64: + return compareInt64Uint64(&val1->datum.val, &val2->datum.val); + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareDoubleUint64(&val1->datum.val, &val2->datum.val); } + //TODO: varchar, nchar + default: + qError("not support comparsion. %d, %d", val1->kind, val2->kind); + return (val1->kind - val2->kind); + } +} + +int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { + if (val2->kind == FLT_SCL_DATUM_KIND_NULL || val2->kind == FLT_SCL_DATUM_KIND_MIN || + val2->kind == FLT_SCL_DATUM_KIND_MAX) { + if (val1->kind == FLT_SCL_DATUM_KIND_NULL || val1->kind == FLT_SCL_DATUM_KIND_MIN || + val1->kind == FLT_SCL_DATUM_KIND_MAX) { + return (val1->kind < val2->kind) ? -1 : ((val1->kind > val2->kind) ? 1 : 0); + } + } + + switch (val2->kind) { case FLT_SCL_DATUM_KIND_UINT64: { - return compareUint64Val(&val1->datum.val, &val1->datum.val); + return fltSclCompareWithUInt64(val1, val2); } case FLT_SCL_DATUM_KIND_INT64: { - return compareInt64Val(&val1->datum.val, &val2->datum.val); + return fltSclCompareWithInt64(val1, val2); } case FLT_SCL_DATUM_KIND_FLOAT64: { - return compareFloatDouble(&val1->datum.val, &val2->datum.val); - } - case FLT_SCL_DATUM_KIND_NCHAR: { - return compareLenPrefixedWStr(&val1->datum.pData, &val2->datum.pData); //TODO - } - case FLT_SCL_DATUM_KIND_VARCHAR: { - return compareLenPrefixedStr(&val1->datum.pData, &val2->datum.pData); //TODO + return fltSclCompareWithFloat64(val1, val2); } + //TODO: varchar/nchar default: fltError("not supported kind. just return 0"); return 0; @@ -4018,125 +4056,115 @@ int32_t fltSclCompareDatum(SFltSclDatum* val1, SFltSclDatum* val2) { return 0; } -bool fltSclLessPoint(SFltSclPoint* pt1, SFltSclPoint* pt2) { +bool fltSclLessPoint(SFltSclPoint *pt1, SFltSclPoint *pt2) { // first value compare int32_t cmp = fltSclCompareDatum(&pt1->val, &pt2->val); if (cmp != 0) { - return cmp < 0 ; + return cmp < 0; } if (pt1->start && pt2->start) { - return !pt1->excl && pt2->excl; - } else if (pt1->start) { - return pt1->excl && !pt2->excl; - } else if (pt2->start) { - return pt1->excl || pt2->excl; - } + return !pt1->excl && pt2->excl; + } else if (pt1->start) { + return pt1->excl && !pt2->excl; + } else if (pt2->start) { + return pt1->excl || pt2->excl; + } return pt1->excl && !pt2->excl; } -int32_t fltSclMergeSort(SArray* pts1, SArray* pts2, SArray* result) { +int32_t fltSclMergeSort(SArray *pts1, SArray *pts2, SArray *result) { size_t len1 = taosArrayGetSize(pts1); size_t len2 = taosArrayGetSize(pts2); size_t i = 0; size_t j = 0; while (i < len1 && j < len2) { - SFltSclPoint* pt1 = taosArrayGet(pts1, i); - SFltSclPoint* pt2 = taosArrayGet(pts2, j); - bool less = fltSclLessPoint(pt1, pt2); - if (less) { + SFltSclPoint *pt1 = taosArrayGet(pts1, i); + SFltSclPoint *pt2 = taosArrayGet(pts2, j); + bool less = fltSclLessPoint(pt1, pt2); + if (less) { taosArrayPush(result, pt1); ++i; - } else { + } else { taosArrayPush(result, pt2); ++j; - } + } } if (i < len1) { - for (; i < len1; ++i) { - SFltSclPoint* pt1 = taosArrayGet(pts1, i); + for (; i < len1; ++i) { + SFltSclPoint *pt1 = taosArrayGet(pts1, i); taosArrayPush(result, pt1); - } + } } if (j < len2) { - for (; j < len2; ++j) { + for (; j < len2; ++j) { SFltSclPoint *pt2 = taosArrayGet(pts2, j); taosArrayPush(result, pt2); - } + } } return 0; } // pts1 and pts2 must be ordered and de-duplicated and each range can not be a range of another range -int32_t fltSclMerge(SArray* pts1, SArray* pts2, bool isUnion, SArray* merged) { +int32_t fltSclMerge(SArray *pts1, SArray *pts2, bool isUnion, SArray *merged) { size_t len1 = taosArrayGetSize(pts1); size_t len2 = taosArrayGetSize(pts2); - //first merge sort pts1 and pts2 - SArray* all = taosArrayInit(len1 + len2, sizeof(SFltSclPoint)); + // first merge sort pts1 and pts2 + SArray *all = taosArrayInit(len1 + len2, sizeof(SFltSclPoint)); fltSclMergeSort(pts1, pts2, all); int32_t countRequired = (isUnion) ? 1 : 2; int32_t count = 0; for (int32_t i = 0; i < taosArrayGetSize(all); ++i) { - SFltSclPoint* pt = taosArrayGet(pts1, i); - if (pt->start) { - ++count; - if (count == countRequired) { - taosArrayPush(merged, pt); - } - } else { - if (count == countRequired) { - taosArrayPush(merged, pt); - } - --count; + SFltSclPoint *pt = taosArrayGet(pts1, i); + if (pt->start) { + ++count; + if (count == countRequired) { + taosArrayPush(merged, pt); } + } else { + if (count == countRequired) { + taosArrayPush(merged, pt); + } + --count; + } } taosArrayDestroy(all); return 0; } +int32_t fltSclIntersect(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, false, merged); } -int32_t fltSclIntersect(SArray* pts1, SArray* pts2, SArray* merged) { - return fltSclMerge(pts1, pts2, false, merged); -} - -int32_t fltSclUnion(SArray* pts1, SArray* pts2, SArray* merged) { - return fltSclMerge(pts1, pts2, true, merged); -} - +int32_t fltSclUnion(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, true, merged); } typedef struct { - SColumnNode* colNode; - SValueNode* valNode; + SColumnNode *colNode; + SValueNode *valNode; EOperatorType type; -} SFltSclScalarOp; +} SFltSclOperator; -// simple define it, TODO: implement later -typedef struct { - SColumnNode* colNode; -} SFltSclConstColumn; - -// simple define it, TODO: implement later -typedef struct { - SValueNode* value; -} SFltSclConstant; - - -int32_t fltSclBuildRangeFromScalarFunc(SFltSclScalarOp* func, SFltSclPoint* points) { +//TODO: column, constant, operator, and/or/not +int32_t fltSclBuildRangeFromOperator(SFltSclOperator *sclOper, SSHashObj* obj) { + switch (sclOper->type) { + case OP_TYPE_GREATER_THAN: { + } + break; + default: + break; + } + return TSDB_CODE_SUCCESS; } - typedef struct { - bool useRange; - SHashObj* colRanges; + bool useRange; + SHashObj *colRanges; } SFltSclOptCtx; -static EDealRes fltSclMayBeOptimed(SNode* pNode, void* pCtx) { - SFltSclOptCtx* ctx = (SFltSclOptCtx*)pCtx; +static EDealRes fltSclMayBeOptimed(SNode *pNode, void *pCtx) { + SFltSclOptCtx *ctx = (SFltSclOptCtx *)pCtx; return DEAL_RES_CONTINUE; } - int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) { SFltSclOptCtx ctx; nodesWalkExprPostOrder(*pNode, fltSclMayBeOptimed, &ctx); From 09837b7639190340f0d9a339a6da7db163b56ddd Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 18 May 2023 19:00:10 +0800 Subject: [PATCH 04/42] fix: save work --- source/libs/scalar/src/filter.c | 124 ++++++++++++++++++++++++++++---- 1 file changed, 109 insertions(+), 15 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 85974bcd3a..01bd53008e 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3960,7 +3960,7 @@ typedef struct { union { int64_t val; // for int64, uint64 and double and bool (1 true, 0 false) uint8_t *pData; // for varchar, nchar - } datum; + }; SDataType type; // TODO: original data type, may not be used? } SFltSclDatum; @@ -3981,11 +3981,11 @@ int32_t fltSclCompareWithFloat64(SFltSclDatum *val1, SFltSclDatum *val2) { // val2->kind == int64 switch (val1->kind) { case FLT_SCL_DATUM_KIND_UINT64: - return compareUint64Double(&val1->datum.val, &val2->datum.val); + return compareUint64Double(&val1->val, &val2->val); case FLT_SCL_DATUM_KIND_INT64: - return compareInt64Double(&val1->datum.val, &val2->datum.val); + return compareInt64Double(&val1->val, &val2->val); case FLT_SCL_DATUM_KIND_FLOAT64: { - return compareDoubleVal(&val1->datum.val, &val2->datum.val); + return compareDoubleVal(&val1->val, &val2->val); } //TODO: varchar, nchar default: @@ -3998,11 +3998,11 @@ int32_t fltSclCompareWithInt64(SFltSclDatum *val1, SFltSclDatum *val2) { // val2->kind == int64 switch (val1->kind) { case FLT_SCL_DATUM_KIND_UINT64: - return compareUint64Int64(&val1->datum.val, &val2->datum.val); + return compareUint64Int64(&val1->val, &val2->val); case FLT_SCL_DATUM_KIND_INT64: - return compareInt64Val(&val1->datum.val, &val2->datum.val); + return compareInt64Val(&val1->val, &val2->val); case FLT_SCL_DATUM_KIND_FLOAT64: { - return compareDoubleInt64(&val1->datum.val, &val2->datum.val); + return compareDoubleInt64(&val1->val, &val2->val); } //TODO: varchar, nchar default: @@ -4015,11 +4015,11 @@ int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { // val2 kind == uint64 switch (val1->kind) { case FLT_SCL_DATUM_KIND_UINT64: - return compareUint64Val(&val1->datum.val, &val2->datum.val); + return compareUint64Val(&val1->val, &val2->val); case FLT_SCL_DATUM_KIND_INT64: - return compareInt64Uint64(&val1->datum.val, &val2->datum.val); + return compareInt64Uint64(&val1->val, &val2->val); case FLT_SCL_DATUM_KIND_FLOAT64: { - return compareDoubleUint64(&val1->datum.val, &val2->datum.val); + return compareDoubleUint64(&val1->val, &val2->val); } //TODO: varchar, nchar default: @@ -4142,15 +4142,109 @@ typedef struct { EOperatorType type; } SFltSclOperator; -//TODO: column, constant, operator, and/or/not +//TODO: column, constant -int32_t fltSclBuildRangeFromOperator(SFltSclOperator *sclOper, SSHashObj* obj) { - switch (sclOper->type) { - case OP_TYPE_GREATER_THAN: { +typedef struct { + SColumnNode* colNode; + SArray* points; +} SFltSclColumnRange; + +SFltSclColumnRange* fltSclGetColumnRange(SColumnNode* colNode, SArray* colRangeList) { + for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) { + SFltSclColumnRange* colRange = taosArrayGet(colRangeList, i); + if (nodesEqualNode((SNode*)colRange->colNode, (SNode*)colNode)) { + return colRange; } + } + SFltSclColumnRange newColRange = {.colNode = colNode, .points = taosArrayInit(4, sizeof(SFltSclPoint))}; + taosArrayPush(colRangeList, &newColRange); + return taosArrayGetLast(colRangeList); +} + +int32_t fltSclBuildDatumFromValueNode(SFltSclDatum* datum, SValueNode* valNode) { + datum->type = valNode->node.resType; + + if (valNode->isNull) { + datum->kind = FLT_SCL_DATUM_KIND_NULL; + } else { + switch (valNode->node.resType.type) { + case TSDB_DATA_TYPE_NULL: { + datum->kind = FLT_SCL_DATUM_KIND_NULL; + break; + } + case TSDB_DATA_TYPE_BOOL: { + datum->kind = FLT_SCL_DATUM_KIND_INT64; + datum->val = (valNode->datum.b) ? 0 : 1; + break; + } + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: { + datum->kind = FLT_SCL_DATUM_KIND_INT64; + datum->val = valNode->datum.i; + break; + } + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: { + datum->kind = FLT_SCL_DATUM_KIND_UINT64; + *(uint64_t *)&datum->val = valNode->datum.u; + break; + } + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { + datum->kind = FLT_SCL_DATUM_KIND_FLOAT64; + *(double *)&datum->val = valNode->datum.d; + break; + } + //TODO:varchar/nchar/json + default: { + qError("not supported"); + break; + } + } + + } + return TSDB_CODE_SUCCESS; +} + +int32_t fltSclBuildRangePoints(SFltSclOperator* oper, SArray* points) { + switch (oper->type) { + case OP_TYPE_GREATER_THAN: { + SFltSclDatum start; + fltSclBuildDatumFromValueNode(&start, oper->valNode); + SFltSclPoint startPt = {.start = true, .excl = true, .val = start}; + SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; + taosArrayPush(points, &start); + taosArrayPush(points, &end); break; - default: + } + default: { + qError("not supported op"); break; + } + } + return TSDB_CODE_SUCCESS; +} + +//TODO: process DNF composed of CNF +int32_t fltSclProcessCNF(SArray* sclOpList, SArray* colRangeList) { + size_t sz = taosArrayGetSize(sclOpList); + for (int32_t i = 0; i < sz; ++i) { + SFltSclOperator* sclOper = taosArrayGet(sclOpList, i); + SFltSclColumnRange *colRange = fltSclGetColumnRange(sclOper->colNode, colRangeList); + SArray* points = taosArrayInit(4, sizeof(SFltSclPoint)); + fltSclBuildRangePoints(sclOper, points); + SArray* merged = taosArrayInit(4, sizeof(SFltSclPoint)); + int32_t code = fltSclIntersect(colRange->points, points, merged); + taosArrayDestroy(colRange->points); + taosArrayDestroy(points); + colRange->points = merged; + } return TSDB_CODE_SUCCESS; } From 3e92ec7a8a4c6f701c31a95a2c4d80e816e6817f Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 19 May 2023 14:09:37 +0800 Subject: [PATCH 05/42] fix: save work --- source/libs/scalar/src/filter.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 01bd53008e..ffc165cba9 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4223,6 +4223,16 @@ int32_t fltSclBuildRangePoints(SFltSclOperator* oper, SArray* points) { taosArrayPush(points, &end); break; } + case OP_TYPE_GREATER_EQUAL: { + SFltSclDatum start; + fltSclBuildDatumFromValueNode(&start, oper->valNode); + SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; + SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; + taosArrayPush(points, &start); + taosArrayPush(points, &end); + } + case default: { qError("not supported op"); break; From 1f6d894dfd12170afd9208e67fb0136faf30b420 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 19 May 2023 17:06:03 +0800 Subject: [PATCH 06/42] fix: save work --- source/libs/scalar/src/filter.c | 121 ++++++++++++++++++++++++-------- 1 file changed, 91 insertions(+), 30 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index ffc165cba9..fd6e79ed42 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3958,10 +3958,10 @@ typedef enum { typedef struct { SFltSclDatumKind kind; union { - int64_t val; // for int64, uint64 and double and bool (1 true, 0 false) - uint8_t *pData; // for varchar, nchar + int64_t val; // for int64, uint64 and double and bool (1 true, 0 false) + uint8_t *pData; // for varchar, nchar }; - SDataType type; // TODO: original data type, may not be used? + SDataType type; // TODO: original data type, may not be used? } SFltSclDatum; typedef struct { @@ -3987,7 +3987,7 @@ int32_t fltSclCompareWithFloat64(SFltSclDatum *val1, SFltSclDatum *val2) { case FLT_SCL_DATUM_KIND_FLOAT64: { return compareDoubleVal(&val1->val, &val2->val); } - //TODO: varchar, nchar + // TODO: varchar, nchar default: qError("not support comparsion. %d, %d", val1->kind, val2->kind); return (val1->kind - val2->kind); @@ -4004,7 +4004,7 @@ int32_t fltSclCompareWithInt64(SFltSclDatum *val1, SFltSclDatum *val2) { case FLT_SCL_DATUM_KIND_FLOAT64: { return compareDoubleInt64(&val1->val, &val2->val); } - //TODO: varchar, nchar + // TODO: varchar, nchar default: qError("not support comparsion. %d, %d", val1->kind, val2->kind); return (val1->kind - val2->kind); @@ -4021,7 +4021,7 @@ int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { case FLT_SCL_DATUM_KIND_FLOAT64: { return compareDoubleUint64(&val1->val, &val2->val); } - //TODO: varchar, nchar + // TODO: varchar, nchar default: qError("not support comparsion. %d, %d", val1->kind, val2->kind); return (val1->kind - val2->kind); @@ -4047,7 +4047,7 @@ int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { case FLT_SCL_DATUM_KIND_FLOAT64: { return fltSclCompareWithFloat64(val1, val2); } - //TODO: varchar/nchar + // TODO: varchar/nchar default: fltError("not supported kind. just return 0"); return 0; @@ -4142,17 +4142,17 @@ typedef struct { EOperatorType type; } SFltSclOperator; -//TODO: column, constant +// TODO: column, constant typedef struct { - SColumnNode* colNode; - SArray* points; + SColumnNode *colNode; + SArray *points; } SFltSclColumnRange; -SFltSclColumnRange* fltSclGetColumnRange(SColumnNode* colNode, SArray* colRangeList) { +SFltSclColumnRange *fltSclGetColumnRange(SColumnNode *colNode, SArray *colRangeList) { for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) { - SFltSclColumnRange* colRange = taosArrayGet(colRangeList, i); - if (nodesEqualNode((SNode*)colRange->colNode, (SNode*)colNode)) { + SFltSclColumnRange *colRange = taosArrayGet(colRangeList, i); + if (nodesEqualNode((SNode *)colRange->colNode, (SNode *)colNode)) { return colRange; } } @@ -4161,7 +4161,7 @@ SFltSclColumnRange* fltSclGetColumnRange(SColumnNode* colNode, SArray* colRangeL return taosArrayGetLast(colRangeList); } -int32_t fltSclBuildDatumFromValueNode(SFltSclDatum* datum, SValueNode* valNode) { +int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) { datum->type = valNode->node.resType; if (valNode->isNull) { @@ -4200,39 +4200,101 @@ int32_t fltSclBuildDatumFromValueNode(SFltSclDatum* datum, SValueNode* valNode) *(double *)&datum->val = valNode->datum.d; break; } - //TODO:varchar/nchar/json + // TODO:varchar/nchar/json default: { qError("not supported"); break; } } - } return TSDB_CODE_SUCCESS; } -int32_t fltSclBuildRangePoints(SFltSclOperator* oper, SArray* points) { +int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { + if (IS_UNSIGNED_NUMERIC_TYPE(oper->colNode->node.resType.type) && + ((IS_SIGNED_NUMERIC_TYPE(oper->valNode->node.resType.type) && oper->valNode->datum.i < 0) || + (IS_FLOAT_TYPE(oper->valNode->node.resType.type) && oper->valNode->datum.d < 0))) { + if (oper->type == OP_TYPE_GREATER_THAN || oper->type == OP_TYPE_GREATER_EQUAL || oper->type == OP_TYPE_NOT_EQUAL) { + SFltSclDatum start; + fltSclBuildDatumFromValueNode(&start, oper->valNode); + start.val = 0; + SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; + SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + } + return TSDB_CODE_SUCCESS; + } switch (oper->type) { case OP_TYPE_GREATER_THAN: { SFltSclDatum start; fltSclBuildDatumFromValueNode(&start, oper->valNode); SFltSclPoint startPt = {.start = true, .excl = true, .val = start}; - SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; - taosArrayPush(points, &start); - taosArrayPush(points, &end); + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); break; } case OP_TYPE_GREATER_EQUAL: { SFltSclDatum start; fltSclBuildDatumFromValueNode(&start, oper->valNode); SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; - SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; - taosArrayPush(points, &start); - taosArrayPush(points, &end); + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + case OP_TYPE_LOWER_THAN: { + SFltSclDatum end; + fltSclBuildDatumFromValueNode(&end, oper->valNode); + SFltSclPoint endPt = {.start = false, .excl = true, .val = end}; + SFltSclDatum start = {.kind = FLT_SCL_DATUM_KIND_MIN, .type = oper->colNode->node.resType}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + case OP_TYPE_LOWER_EQUAL: { + SFltSclDatum end; + fltSclBuildDatumFromValueNode(&end, oper->valNode); + SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; + SFltSclDatum start = {.kind = FLT_SCL_DATUM_KIND_MIN, .type = oper->colNode->node.resType}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + case OP_TYPE_EQUAL: { + SFltSclDatum valDatum; + fltSclBuildDatumFromValueNode(&valDatum, oper->valNode); + SFltSclPoint startPt = {.start = true, .excl = false, .val = valDatum}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = valDatum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + case OP_TYPE_NOT_EQUAL: { + SFltSclDatum valDatum; + fltSclBuildDatumFromValueNode(&valDatum, oper->valNode); + { + SFltSclDatum start = {.kind = FLT_SCL_DATUM_KIND_MIN, .type = oper->colNode->node.resType}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; + SFltSclPoint endPt = {.start = false, .excl = true, .val = valDatum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + } + { + SFltSclPoint startPt = {.start = true, .excl = true, .val = valDatum}; + SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + } + break; } - case default: { qError("not supported op"); break; @@ -4241,20 +4303,19 @@ int32_t fltSclBuildRangePoints(SFltSclOperator* oper, SArray* points) { return TSDB_CODE_SUCCESS; } -//TODO: process DNF composed of CNF -int32_t fltSclProcessCNF(SArray* sclOpList, SArray* colRangeList) { +// TODO: process DNF composed of CNF +int32_t fltSclProcessCNF(SArray *sclOpList, SArray *colRangeList) { size_t sz = taosArrayGetSize(sclOpList); for (int32_t i = 0; i < sz; ++i) { - SFltSclOperator* sclOper = taosArrayGet(sclOpList, i); + SFltSclOperator *sclOper = taosArrayGet(sclOpList, i); SFltSclColumnRange *colRange = fltSclGetColumnRange(sclOper->colNode, colRangeList); - SArray* points = taosArrayInit(4, sizeof(SFltSclPoint)); + SArray *points = taosArrayInit(4, sizeof(SFltSclPoint)); fltSclBuildRangePoints(sclOper, points); - SArray* merged = taosArrayInit(4, sizeof(SFltSclPoint)); + SArray *merged = taosArrayInit(4, sizeof(SFltSclPoint)); int32_t code = fltSclIntersect(colRange->points, points, merged); taosArrayDestroy(colRange->points); taosArrayDestroy(points); colRange->points = merged; - } return TSDB_CODE_SUCCESS; } From 6456d5dccfd11afe518fc3a282494eb50a0e8a38 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 22 May 2023 11:41:19 +0800 Subject: [PATCH 07/42] enhance: code finish --- source/libs/scalar/inc/filterInt.h | 7 + source/libs/scalar/src/filter.c | 700 ++++++++++++++++------------- 2 files changed, 407 insertions(+), 300 deletions(-) diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index 2023d38777..1ca8ac1d8c 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -227,8 +227,10 @@ typedef struct SFltTreeStat { SFilterInfo *info; } SFltTreeStat; + typedef struct SFltScalarCtx { SNode *node; + SArray* fltSclRange; } SFltScalarCtx; typedef struct SFltBuildGroupCtx { @@ -237,6 +239,11 @@ typedef struct SFltBuildGroupCtx { int32_t code; } SFltBuildGroupCtx; +typedef struct { + SColumnNode *colNode; + SArray *points; +} SFltSclColumnRange; + struct SFilterInfo { bool scalarMode; SFltScalarCtx sclCtx; diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index fd6e79ed42..33f37c722e 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1841,6 +1841,13 @@ void filterFreeInfo(SFilterInfo *info) { return; } + for (int32_t i = 0 ; i < taosArrayGetSize(info->sclCtx.fltSclRange); ++i) { + SFltSclColumnRange* colRange = taosArrayGet(info->sclCtx.fltSclRange, i); + nodesDestroyNode((SNode*)colRange->colNode); + taosArrayDestroy(colRange->points); + } + taosArrayDestroy(info->sclCtx.fltSclRange); + taosMemoryFreeClear(info->cunits); taosMemoryFreeClear(info->blkUnitRes); taosMemoryFreeClear(info->blkUnits); @@ -3424,8 +3431,335 @@ _return: return code; } + +// compare ranges, null < min < val < max. null=null, min=min, max=max +typedef enum { + FLT_SCL_DATUM_KIND_NULL, + FLT_SCL_DATUM_KIND_MIN, + FLT_SCL_DATUM_KIND_INT64, + FLT_SCL_DATUM_KIND_UINT64, + FLT_SCL_DATUM_KIND_FLOAT64, + FLT_SCL_DATUM_KIND_VARCHAR, + FLT_SCL_DATUM_KIND_NCHAR, + FLT_SCL_DATUM_KIND_MAX, +} SFltSclDatumKind; + +typedef struct { + SFltSclDatumKind kind; + union { + int64_t i; // for int64, uint64 and double and bool (1 true, 0 false) + uint64_t u; + double d; + uint8_t *pData; // for varchar, nchar + }; + SDataType type; // TODO: original data type, may not be used? +} SFltSclDatum; + +typedef struct { + SFltSclDatum val; + bool excl; + bool start; +} SFltSclPoint; + +int32_t fltSclCompareWithFloat64(SFltSclDatum *val1, SFltSclDatum *val2) { + // val2->kind == float64 + switch (val1->kind) { + case FLT_SCL_DATUM_KIND_UINT64: + return compareUint64Double(&val1->u, &val2->d); + case FLT_SCL_DATUM_KIND_INT64: + return compareInt64Double(&val1->i, &val2->d); + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareDoubleVal(&val1->d, &val2->d); + } + // TODO: varchar, nchar + default: + qError("not support comparsion. %d, %d", val1->kind, val2->kind); + return (val1->kind - val2->kind); + } +} + +int32_t fltSclCompareWithInt64(SFltSclDatum *val1, SFltSclDatum *val2) { + // val2->kind == int64 + switch (val1->kind) { + case FLT_SCL_DATUM_KIND_UINT64: + return compareUint64Int64(&val1->u, &val2->i); + case FLT_SCL_DATUM_KIND_INT64: + return compareInt64Val(&val1->i, &val2->i); + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareDoubleInt64(&val1->d, &val2->i); + } + // TODO: varchar, nchar + default: + qError("not support comparsion. %d, %d", val1->kind, val2->kind); + return (val1->kind - val2->kind); + } +} + +int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { + // val2 kind == uint64 + switch (val1->kind) { + case FLT_SCL_DATUM_KIND_UINT64: + return compareUint64Val(&val1->u, &val2->u); + case FLT_SCL_DATUM_KIND_INT64: + return compareInt64Uint64(&val1->i, &val2->u); + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareDoubleUint64(&val1->d, &val2->u); + } + // TODO: varchar, nchar + default: + qError("not support comparsion. %d, %d", val1->kind, val2->kind); + return (val1->kind - val2->kind); + } +} + +int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { + if (val2->kind == FLT_SCL_DATUM_KIND_NULL || val2->kind == FLT_SCL_DATUM_KIND_MIN || + val2->kind == FLT_SCL_DATUM_KIND_MAX) { + if (val1->kind == FLT_SCL_DATUM_KIND_NULL || val1->kind == FLT_SCL_DATUM_KIND_MIN || + val1->kind == FLT_SCL_DATUM_KIND_MAX) { + return (val1->kind < val2->kind) ? -1 : ((val1->kind > val2->kind) ? 1 : 0); + } + } + + switch (val2->kind) { + case FLT_SCL_DATUM_KIND_UINT64: { + return fltSclCompareWithUInt64(val1, val2); + } + case FLT_SCL_DATUM_KIND_INT64: { + return fltSclCompareWithInt64(val1, val2); + } + case FLT_SCL_DATUM_KIND_FLOAT64: { + return fltSclCompareWithFloat64(val1, val2); + } + // TODO: varchar/nchar + default: + fltError("not supported kind. just return 0"); + return 0; + break; + } + return 0; +} + +bool fltSclLessPoint(SFltSclPoint *pt1, SFltSclPoint *pt2) { + // first value compare + int32_t cmp = fltSclCompareDatum(&pt1->val, &pt2->val); + if (cmp != 0) { + return cmp < 0; + } + + if (pt1->start && pt2->start) { + return !pt1->excl && pt2->excl; + } else if (pt1->start) { + return pt1->excl && !pt2->excl; + } else if (pt2->start) { + return pt1->excl || pt2->excl; + } + return pt1->excl && !pt2->excl; +} + +int32_t fltSclMergeSort(SArray *pts1, SArray *pts2, SArray *result) { + size_t len1 = taosArrayGetSize(pts1); + size_t len2 = taosArrayGetSize(pts2); + size_t i = 0; + size_t j = 0; + while (i < len1 && j < len2) { + SFltSclPoint *pt1 = taosArrayGet(pts1, i); + SFltSclPoint *pt2 = taosArrayGet(pts2, j); + bool less = fltSclLessPoint(pt1, pt2); + if (less) { + taosArrayPush(result, pt1); + ++i; + } else { + taosArrayPush(result, pt2); + ++j; + } + } + if (i < len1) { + for (; i < len1; ++i) { + SFltSclPoint *pt1 = taosArrayGet(pts1, i); + taosArrayPush(result, pt1); + } + } + if (j < len2) { + for (; j < len2; ++j) { + SFltSclPoint *pt2 = taosArrayGet(pts2, j); + taosArrayPush(result, pt2); + } + } + return 0; +} + +// pts1 and pts2 must be ordered and de-duplicated and each range can not be a range of another range +int32_t fltSclMerge(SArray *pts1, SArray *pts2, bool isUnion, SArray *merged) { + size_t len1 = taosArrayGetSize(pts1); + size_t len2 = taosArrayGetSize(pts2); + // first merge sort pts1 and pts2 + SArray *all = taosArrayInit(len1 + len2, sizeof(SFltSclPoint)); + fltSclMergeSort(pts1, pts2, all); + int32_t countRequired = (isUnion) ? 1 : 2; + int32_t count = 0; + for (int32_t i = 0; i < taosArrayGetSize(all); ++i) { + SFltSclPoint *pt = taosArrayGet(pts1, i); + if (pt->start) { + ++count; + if (count == countRequired) { + taosArrayPush(merged, pt); + } + } else { + if (count == countRequired) { + taosArrayPush(merged, pt); + } + --count; + } + } + taosArrayDestroy(all); + return 0; +} + +int32_t fltSclIntersect(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, false, merged); } + +int32_t fltSclUnion(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, true, merged); } + +// TODO: column, constant +typedef struct { + SColumnNode *colNode; + SValueNode *valNode; + EOperatorType type; +} SFltSclOperator; + + +SFltSclColumnRange *fltSclGetOrCreateColumnRange(SColumnNode *colNode, SArray *colRangeList) { + for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) { + SFltSclColumnRange *colRange = taosArrayGet(colRangeList, i); + if (nodesEqualNode((SNode *)colRange->colNode, (SNode *)colNode)) { + return colRange; + } + } + SColumnNode* pColumnNode = (SColumnNode*)nodesCloneNode((SNode*)colNode); + SFltSclColumnRange newColRange = {.colNode = pColumnNode, .points = taosArrayInit(4, sizeof(SFltSclPoint))}; + taosArrayPush(colRangeList, &newColRange); + return taosArrayGetLast(colRangeList); +} + +int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) { + datum->type = valNode->node.resType; + + if (valNode->isNull) { + datum->kind = FLT_SCL_DATUM_KIND_NULL; + } else { + switch (valNode->node.resType.type) { + case TSDB_DATA_TYPE_NULL: { + datum->kind = FLT_SCL_DATUM_KIND_NULL; + break; + } + case TSDB_DATA_TYPE_BOOL: { + datum->kind = FLT_SCL_DATUM_KIND_INT64; + datum->i = (valNode->datum.b) ? 0 : 1; + break; + } + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: { + datum->kind = FLT_SCL_DATUM_KIND_INT64; + datum->i = valNode->datum.i; + break; + } + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: { + datum->kind = FLT_SCL_DATUM_KIND_UINT64; + datum->u = valNode->datum.u; + break; + } + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { + datum->kind = FLT_SCL_DATUM_KIND_FLOAT64; + datum->d = valNode->datum.d; + break; + } + // TODO:varchar/nchar/json + default: { + qError("not supported"); + break; + } + } + } + return TSDB_CODE_SUCCESS; +} + +int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum* datum, uint8_t type, int64_t val) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: { + datum->kind = FLT_SCL_DATUM_KIND_INT64; + datum->i = val; + break; + } + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: { + datum->kind = FLT_SCL_DATUM_KIND_UINT64; + datum->u = *(uint64_t*)&val; + break; + } + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { + datum->kind = FLT_SCL_DATUM_KIND_FLOAT64; + datum->d = *(double*)&val; + break; + } + // TODO:varchar/nchar/json + default: { + qError("not supported"); + break; + } + } + + + return TSDB_CODE_SUCCESS; +} + +int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange* colRange, SColumnDataAgg* pAgg, SArray* points) { + SFltSclDatum min; + fltSclBuildDatumFromBlockSmaValue(&min, colRange->colNode->node.resType.type, pAgg->min); + SFltSclPoint minPt = {.excl = false, .start = true, .val = min}; + SFltSclDatum max; + fltSclBuildDatumFromBlockSmaValue(&max, colRange->colNode->node.resType.type, pAgg->max); + SFltSclPoint maxPt = {.excl = false, .start = false, .val = max}; + taosArrayPush(points, &minPt); + taosArrayPush(points, &maxPt); + return TSDB_CODE_SUCCESS; +} + bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t numOfCols, int32_t numOfRows) { if (info->scalarMode) { + SArray* colRanges = info->sclCtx.fltSclRange; + for (int32_t i = 0; i < taosArrayGetSize(colRanges); ++i) { + SFltSclColumnRange* colRange = taosArrayGet(colRanges, i); + bool foundCol = false; + int32_t j = 0; + for (; j < numOfCols; ++j) { + if (pDataStatis[j] != NULL && pDataStatis[j]->colId == colRange->colNode->colId) { + foundCol = true; + } + } + if (foundCol) { + SColumnDataAgg* pAgg = pDataStatis[j]; + SArray* points = taosArrayInit(2, sizeof(SFltSclPoint)); + fltSclBuildRangeFromBlockSma(colRange, pAgg, points); + SArray* merged = taosArrayInit(8, sizeof(SFltSclPoint)); + fltSclIntersect(points, colRange->points, merged); + bool isIntersect = taosArrayGetSize(merged) != 0; + taosArrayDestroy(merged); + taosArrayDestroy(points); + if (!isIntersect) { + return false; + } + } + } return true; } @@ -3943,289 +4277,7 @@ _return: FLT_RET(code); } -// compare ranges, null < min < val < max. null=null, min=min, max=max -typedef enum { - FLT_SCL_DATUM_KIND_NULL, - FLT_SCL_DATUM_KIND_MIN, - FLT_SCL_DATUM_KIND_INT64, - FLT_SCL_DATUM_KIND_UINT64, - FLT_SCL_DATUM_KIND_FLOAT64, - FLT_SCL_DATUM_KIND_VARCHAR, - FLT_SCL_DATUM_KIND_NCHAR, - FLT_SCL_DATUM_KIND_MAX, -} SFltSclDatumKind; - -typedef struct { - SFltSclDatumKind kind; - union { - int64_t val; // for int64, uint64 and double and bool (1 true, 0 false) - uint8_t *pData; // for varchar, nchar - }; - SDataType type; // TODO: original data type, may not be used? -} SFltSclDatum; - -typedef struct { - SFltSclDatum val; - bool excl; - bool start; -} SFltSclPoint; - -typedef struct { - SFltSclDatum low; - SFltSclDatum high; - bool lowExcl; - bool highExcl; -} SFltSclRange; - -int32_t fltSclCompareWithFloat64(SFltSclDatum *val1, SFltSclDatum *val2) { - // val2->kind == int64 - switch (val1->kind) { - case FLT_SCL_DATUM_KIND_UINT64: - return compareUint64Double(&val1->val, &val2->val); - case FLT_SCL_DATUM_KIND_INT64: - return compareInt64Double(&val1->val, &val2->val); - case FLT_SCL_DATUM_KIND_FLOAT64: { - return compareDoubleVal(&val1->val, &val2->val); - } - // TODO: varchar, nchar - default: - qError("not support comparsion. %d, %d", val1->kind, val2->kind); - return (val1->kind - val2->kind); - } -} - -int32_t fltSclCompareWithInt64(SFltSclDatum *val1, SFltSclDatum *val2) { - // val2->kind == int64 - switch (val1->kind) { - case FLT_SCL_DATUM_KIND_UINT64: - return compareUint64Int64(&val1->val, &val2->val); - case FLT_SCL_DATUM_KIND_INT64: - return compareInt64Val(&val1->val, &val2->val); - case FLT_SCL_DATUM_KIND_FLOAT64: { - return compareDoubleInt64(&val1->val, &val2->val); - } - // TODO: varchar, nchar - default: - qError("not support comparsion. %d, %d", val1->kind, val2->kind); - return (val1->kind - val2->kind); - } -} - -int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { - // val2 kind == uint64 - switch (val1->kind) { - case FLT_SCL_DATUM_KIND_UINT64: - return compareUint64Val(&val1->val, &val2->val); - case FLT_SCL_DATUM_KIND_INT64: - return compareInt64Uint64(&val1->val, &val2->val); - case FLT_SCL_DATUM_KIND_FLOAT64: { - return compareDoubleUint64(&val1->val, &val2->val); - } - // TODO: varchar, nchar - default: - qError("not support comparsion. %d, %d", val1->kind, val2->kind); - return (val1->kind - val2->kind); - } -} - -int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { - if (val2->kind == FLT_SCL_DATUM_KIND_NULL || val2->kind == FLT_SCL_DATUM_KIND_MIN || - val2->kind == FLT_SCL_DATUM_KIND_MAX) { - if (val1->kind == FLT_SCL_DATUM_KIND_NULL || val1->kind == FLT_SCL_DATUM_KIND_MIN || - val1->kind == FLT_SCL_DATUM_KIND_MAX) { - return (val1->kind < val2->kind) ? -1 : ((val1->kind > val2->kind) ? 1 : 0); - } - } - - switch (val2->kind) { - case FLT_SCL_DATUM_KIND_UINT64: { - return fltSclCompareWithUInt64(val1, val2); - } - case FLT_SCL_DATUM_KIND_INT64: { - return fltSclCompareWithInt64(val1, val2); - } - case FLT_SCL_DATUM_KIND_FLOAT64: { - return fltSclCompareWithFloat64(val1, val2); - } - // TODO: varchar/nchar - default: - fltError("not supported kind. just return 0"); - return 0; - break; - } - return 0; -} - -bool fltSclLessPoint(SFltSclPoint *pt1, SFltSclPoint *pt2) { - // first value compare - int32_t cmp = fltSclCompareDatum(&pt1->val, &pt2->val); - if (cmp != 0) { - return cmp < 0; - } - - if (pt1->start && pt2->start) { - return !pt1->excl && pt2->excl; - } else if (pt1->start) { - return pt1->excl && !pt2->excl; - } else if (pt2->start) { - return pt1->excl || pt2->excl; - } - return pt1->excl && !pt2->excl; -} - -int32_t fltSclMergeSort(SArray *pts1, SArray *pts2, SArray *result) { - size_t len1 = taosArrayGetSize(pts1); - size_t len2 = taosArrayGetSize(pts2); - size_t i = 0; - size_t j = 0; - while (i < len1 && j < len2) { - SFltSclPoint *pt1 = taosArrayGet(pts1, i); - SFltSclPoint *pt2 = taosArrayGet(pts2, j); - bool less = fltSclLessPoint(pt1, pt2); - if (less) { - taosArrayPush(result, pt1); - ++i; - } else { - taosArrayPush(result, pt2); - ++j; - } - } - if (i < len1) { - for (; i < len1; ++i) { - SFltSclPoint *pt1 = taosArrayGet(pts1, i); - taosArrayPush(result, pt1); - } - } - if (j < len2) { - for (; j < len2; ++j) { - SFltSclPoint *pt2 = taosArrayGet(pts2, j); - taosArrayPush(result, pt2); - } - } - return 0; -} - -// pts1 and pts2 must be ordered and de-duplicated and each range can not be a range of another range -int32_t fltSclMerge(SArray *pts1, SArray *pts2, bool isUnion, SArray *merged) { - size_t len1 = taosArrayGetSize(pts1); - size_t len2 = taosArrayGetSize(pts2); - // first merge sort pts1 and pts2 - SArray *all = taosArrayInit(len1 + len2, sizeof(SFltSclPoint)); - fltSclMergeSort(pts1, pts2, all); - int32_t countRequired = (isUnion) ? 1 : 2; - int32_t count = 0; - for (int32_t i = 0; i < taosArrayGetSize(all); ++i) { - SFltSclPoint *pt = taosArrayGet(pts1, i); - if (pt->start) { - ++count; - if (count == countRequired) { - taosArrayPush(merged, pt); - } - } else { - if (count == countRequired) { - taosArrayPush(merged, pt); - } - --count; - } - } - taosArrayDestroy(all); - return 0; -} - -int32_t fltSclIntersect(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, false, merged); } - -int32_t fltSclUnion(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, true, merged); } - -typedef struct { - SColumnNode *colNode; - SValueNode *valNode; - EOperatorType type; -} SFltSclOperator; - -// TODO: column, constant - -typedef struct { - SColumnNode *colNode; - SArray *points; -} SFltSclColumnRange; - -SFltSclColumnRange *fltSclGetColumnRange(SColumnNode *colNode, SArray *colRangeList) { - for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) { - SFltSclColumnRange *colRange = taosArrayGet(colRangeList, i); - if (nodesEqualNode((SNode *)colRange->colNode, (SNode *)colNode)) { - return colRange; - } - } - SFltSclColumnRange newColRange = {.colNode = colNode, .points = taosArrayInit(4, sizeof(SFltSclPoint))}; - taosArrayPush(colRangeList, &newColRange); - return taosArrayGetLast(colRangeList); -} - -int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) { - datum->type = valNode->node.resType; - - if (valNode->isNull) { - datum->kind = FLT_SCL_DATUM_KIND_NULL; - } else { - switch (valNode->node.resType.type) { - case TSDB_DATA_TYPE_NULL: { - datum->kind = FLT_SCL_DATUM_KIND_NULL; - break; - } - case TSDB_DATA_TYPE_BOOL: { - datum->kind = FLT_SCL_DATUM_KIND_INT64; - datum->val = (valNode->datum.b) ? 0 : 1; - break; - } - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: { - datum->kind = FLT_SCL_DATUM_KIND_INT64; - datum->val = valNode->datum.i; - break; - } - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_UBIGINT: { - datum->kind = FLT_SCL_DATUM_KIND_UINT64; - *(uint64_t *)&datum->val = valNode->datum.u; - break; - } - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: { - datum->kind = FLT_SCL_DATUM_KIND_FLOAT64; - *(double *)&datum->val = valNode->datum.d; - break; - } - // TODO:varchar/nchar/json - default: { - qError("not supported"); - break; - } - } - } - return TSDB_CODE_SUCCESS; -} - int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { - if (IS_UNSIGNED_NUMERIC_TYPE(oper->colNode->node.resType.type) && - ((IS_SIGNED_NUMERIC_TYPE(oper->valNode->node.resType.type) && oper->valNode->datum.i < 0) || - (IS_FLOAT_TYPE(oper->valNode->node.resType.type) && oper->valNode->datum.d < 0))) { - if (oper->type == OP_TYPE_GREATER_THAN || oper->type == OP_TYPE_GREATER_EQUAL || oper->type == OP_TYPE_NOT_EQUAL) { - SFltSclDatum start; - fltSclBuildDatumFromValueNode(&start, oper->valNode); - start.val = 0; - SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; - SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; - SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; - taosArrayPush(points, &startPt); - taosArrayPush(points, &endPt); - } - return TSDB_CODE_SUCCESS; - } switch (oper->type) { case OP_TYPE_GREATER_THAN: { SFltSclDatum start; @@ -4304,36 +4356,84 @@ int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { } // TODO: process DNF composed of CNF -int32_t fltSclProcessCNF(SArray *sclOpList, SArray *colRangeList) { - size_t sz = taosArrayGetSize(sclOpList); +int32_t fltSclProcessCNF(SArray *sclOpListCNF, SArray *colRangeList) { + size_t sz = taosArrayGetSize(sclOpListCNF); for (int32_t i = 0; i < sz; ++i) { - SFltSclOperator *sclOper = taosArrayGet(sclOpList, i); - SFltSclColumnRange *colRange = fltSclGetColumnRange(sclOper->colNode, colRangeList); + SFltSclOperator *sclOper = taosArrayGet(sclOpListCNF, i); + SFltSclColumnRange *colRange = fltSclGetOrCreateColumnRange(sclOper->colNode, colRangeList); SArray *points = taosArrayInit(4, sizeof(SFltSclPoint)); fltSclBuildRangePoints(sclOper, points); - SArray *merged = taosArrayInit(4, sizeof(SFltSclPoint)); - int32_t code = fltSclIntersect(colRange->points, points, merged); - taosArrayDestroy(colRange->points); - taosArrayDestroy(points); - colRange->points = merged; + if (taosArrayGetSize(colRange->points) != 0) { + SArray *merged = taosArrayInit(4, sizeof(SFltSclPoint)); + int32_t code = fltSclIntersect(colRange->points, points, merged); + taosArrayDestroy(colRange->points); + taosArrayDestroy(points); + colRange->points = merged; + } else { + colRange->points = points; + } + } return TSDB_CODE_SUCCESS; } -typedef struct { - bool useRange; - SHashObj *colRanges; -} SFltSclOptCtx; +static int32_t fltSclCollectOperatorFromNode(SNode* pNode, SArray* sclOpList) { + if (nodeType(pNode) != QUERY_NODE_OPERATOR) { + return TSDB_CODE_SUCCESS; + } + SOperatorNode* pOper = (SOperatorNode*)pNode; + //TODO: left value node, right column node + //TODO: datatype + //TODO: operator + if (nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_VALUE && + (pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL || + pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL || + pOper->opType == OP_TYPE_NOT_EQUAL || pOper->opType == OP_TYPE_EQUAL)) { + SValueNode* valNode = (SValueNode*)pOper->pRight; + if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { + SNode *p = nodesCloneNode(pNode); + taosArrayPush(sclOpList, &p); + } + } + return TSDB_CODE_SUCCESS; +} -static EDealRes fltSclMayBeOptimed(SNode *pNode, void *pCtx) { - SFltSclOptCtx *ctx = (SFltSclOptCtx *)pCtx; - return DEAL_RES_CONTINUE; +static int32_t fltSclCollectOperatorsFromLogicCond(SNode* pNode, SArray* sclOpList) { + if (nodeType(pNode) != QUERY_NODE_LOGIC_CONDITION) { + return TSDB_CODE_SUCCESS; + } + SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pNode; + //TODO: support LOGIC_COND_TYPE_OR + if (pLogicCond->condType != LOGIC_COND_TYPE_AND) { + return TSDB_CODE_SUCCESS; + } + SNode* pExpr = NULL; + FOREACH(pExpr, pLogicCond->pParameterList) { fltSclCollectOperatorFromNode(pExpr, sclOpList); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t fltSclCollectOperators(SNode *pNode, SArray* sclOpList) { + if (nodeType(pNode) == QUERY_NODE_OPERATOR) { + fltSclCollectOperatorFromNode(pNode, sclOpList); + } else if (nodeType(pNode) == QUERY_NODE_LOGIC_CONDITION) { + fltSclCollectOperatorsFromLogicCond(pNode, sclOpList); + } + return TSDB_CODE_SUCCESS; } int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) { - SFltSclOptCtx ctx; - nodesWalkExprPostOrder(*pNode, fltSclMayBeOptimed, &ctx); + SArray* sclOpList = taosArrayInit(16, POINTER_BYTES); + fltSclCollectOperators(*pNode, sclOpList); + SArray* colRangeList = taosArrayInit(16, sizeof(SFltSclColumnRange)); + fltSclProcessCNF(sclOpList, colRangeList); + pInfo->sclCtx.fltSclRange = colRangeList; + for (int32_t i = 0; i < taosArrayGetSize(sclOpList); ++i) { + SNode* p = taosArrayGetP(sclOpList, i); + nodesDestroyNode(p); + } + taosArrayDestroy(sclOpList); return TSDB_CODE_SUCCESS; } From 43ac6fb1fd532c96f5deba1b21e46abd5b2e0762 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 23 May 2023 08:43:31 +0800 Subject: [PATCH 08/42] fix: all null from block sma --- source/libs/scalar/src/filter.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 33f37c722e..4475a07dcd 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3722,7 +3722,15 @@ int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum* datum, uint8_t type, int return TSDB_CODE_SUCCESS; } -int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange* colRange, SColumnDataAgg* pAgg, SArray* points) { +int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange* colRange, SColumnDataAgg* pAgg, int32_t numOfRows, SArray* points) { + if (pAgg->numOfNull == numOfRows) { + SFltSclDatum datum = {.kind = FLT_SCL_DATUM_KIND_NULL}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = datum}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = datum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + return TSDB_CODE_SUCCESS; + } SFltSclDatum min; fltSclBuildDatumFromBlockSmaValue(&min, colRange->colNode->node.resType.type, pAgg->min); SFltSclPoint minPt = {.excl = false, .start = true, .val = min}; @@ -3749,7 +3757,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t if (foundCol) { SColumnDataAgg* pAgg = pDataStatis[j]; SArray* points = taosArrayInit(2, sizeof(SFltSclPoint)); - fltSclBuildRangeFromBlockSma(colRange, pAgg, points); + fltSclBuildRangeFromBlockSma(colRange, pAgg, numOfRows, points); SArray* merged = taosArrayInit(8, sizeof(SFltSclPoint)); fltSclIntersect(points, colRange->points, merged); bool isIntersect = taosArrayGetSize(merged) != 0; From 24ae8d7ecb870852c9979083eddb5686b32bb278 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 23 May 2023 09:49:42 +0800 Subject: [PATCH 09/42] fix: fix ci memory leak error --- source/libs/scalar/src/filter.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 4475a07dcd..c21abc1380 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4378,6 +4378,7 @@ int32_t fltSclProcessCNF(SArray *sclOpListCNF, SArray *colRangeList) { taosArrayDestroy(points); colRange->points = merged; } else { + taosArrayDestroy(colRange->points); colRange->points = points; } From 77a1de444c24bd68d0db36c2d7820feb4b1a8c4b Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 23 May 2023 10:47:58 +0800 Subject: [PATCH 10/42] fix: where c7 means c7 is true, the right child is null --- source/libs/scalar/src/filter.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index c21abc1380..4a9ebb16c1 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4394,6 +4394,9 @@ static int32_t fltSclCollectOperatorFromNode(SNode* pNode, SArray* sclOpList) { //TODO: left value node, right column node //TODO: datatype //TODO: operator + if (pOper->pLeft == NULL || pOper->pRight == NULL) { + return TSDB_CODE_SUCCESS; + } if (nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_VALUE && (pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL || pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL || From c9140e547e652e7e241abeee7b5fc175bcbeeb4c Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 23 May 2023 14:08:20 +0800 Subject: [PATCH 11/42] fix: fix bugs of wrongly use sfltscloperator --- source/libs/scalar/src/filter.c | 76 ++++++++++++++++----------------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 4a9ebb16c1..3b61e3a97a 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1841,9 +1841,9 @@ void filterFreeInfo(SFilterInfo *info) { return; } - for (int32_t i = 0 ; i < taosArrayGetSize(info->sclCtx.fltSclRange); ++i) { - SFltSclColumnRange* colRange = taosArrayGet(info->sclCtx.fltSclRange, i); - nodesDestroyNode((SNode*)colRange->colNode); + for (int32_t i = 0; i < taosArrayGetSize(info->sclCtx.fltSclRange); ++i) { + SFltSclColumnRange *colRange = taosArrayGet(info->sclCtx.fltSclRange, i); + nodesDestroyNode((SNode *)colRange->colNode); taosArrayDestroy(colRange->points); } taosArrayDestroy(info->sclCtx.fltSclRange); @@ -3431,7 +3431,6 @@ _return: return code; } - // compare ranges, null < min < val < max. null=null, min=min, max=max typedef enum { FLT_SCL_DATUM_KIND_NULL, @@ -3447,7 +3446,7 @@ typedef enum { typedef struct { SFltSclDatumKind kind; union { - int64_t i; // for int64, uint64 and double and bool (1 true, 0 false) + int64_t i; // for int64, uint64 and double and bool (1 true, 0 false) uint64_t u; double d; uint8_t *pData; // for varchar, nchar @@ -3627,7 +3626,6 @@ typedef struct { EOperatorType type; } SFltSclOperator; - SFltSclColumnRange *fltSclGetOrCreateColumnRange(SColumnNode *colNode, SArray *colRangeList) { for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) { SFltSclColumnRange *colRange = taosArrayGet(colRangeList, i); @@ -3635,7 +3633,7 @@ SFltSclColumnRange *fltSclGetOrCreateColumnRange(SColumnNode *colNode, SArray *c return colRange; } } - SColumnNode* pColumnNode = (SColumnNode*)nodesCloneNode((SNode*)colNode); + SColumnNode *pColumnNode = (SColumnNode *)nodesCloneNode((SNode *)colNode); SFltSclColumnRange newColRange = {.colNode = pColumnNode, .points = taosArrayInit(4, sizeof(SFltSclPoint))}; taosArrayPush(colRangeList, &newColRange); return taosArrayGetLast(colRangeList); @@ -3690,7 +3688,7 @@ int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) return TSDB_CODE_SUCCESS; } -int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum* datum, uint8_t type, int64_t val) { +int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum *datum, uint8_t type, int64_t val) { switch (type) { case TSDB_DATA_TYPE_BOOL: { datum->kind = FLT_SCL_DATUM_KIND_INT64; @@ -3702,13 +3700,13 @@ int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum* datum, uint8_t type, int case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UBIGINT: { datum->kind = FLT_SCL_DATUM_KIND_UINT64; - datum->u = *(uint64_t*)&val; + datum->u = *(uint64_t *)&val; break; } case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: { datum->kind = FLT_SCL_DATUM_KIND_FLOAT64; - datum->d = *(double*)&val; + datum->d = *(double *)&val; break; } // TODO:varchar/nchar/json @@ -3718,11 +3716,11 @@ int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum* datum, uint8_t type, int } } - return TSDB_CODE_SUCCESS; } -int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange* colRange, SColumnDataAgg* pAgg, int32_t numOfRows, SArray* points) { +int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange *colRange, SColumnDataAgg *pAgg, int32_t numOfRows, + SArray *points) { if (pAgg->numOfNull == numOfRows) { SFltSclDatum datum = {.kind = FLT_SCL_DATUM_KIND_NULL}; SFltSclPoint startPt = {.start = true, .excl = false, .val = datum}; @@ -3744,21 +3742,21 @@ int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange* colRange, SColumnDataAg bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t numOfCols, int32_t numOfRows) { if (info->scalarMode) { - SArray* colRanges = info->sclCtx.fltSclRange; + SArray *colRanges = info->sclCtx.fltSclRange; for (int32_t i = 0; i < taosArrayGetSize(colRanges); ++i) { - SFltSclColumnRange* colRange = taosArrayGet(colRanges, i); - bool foundCol = false; - int32_t j = 0; + SFltSclColumnRange *colRange = taosArrayGet(colRanges, i); + bool foundCol = false; + int32_t j = 0; for (; j < numOfCols; ++j) { if (pDataStatis[j] != NULL && pDataStatis[j]->colId == colRange->colNode->colId) { foundCol = true; } } if (foundCol) { - SColumnDataAgg* pAgg = pDataStatis[j]; - SArray* points = taosArrayInit(2, sizeof(SFltSclPoint)); + SColumnDataAgg *pAgg = pDataStatis[j]; + SArray *points = taosArrayInit(2, sizeof(SFltSclPoint)); fltSclBuildRangeFromBlockSma(colRange, pAgg, numOfRows, points); - SArray* merged = taosArrayInit(8, sizeof(SFltSclPoint)); + SArray *merged = taosArrayInit(8, sizeof(SFltSclPoint)); fltSclIntersect(points, colRange->points, merged); bool isIntersect = taosArrayGetSize(merged) != 0; taosArrayDestroy(merged); @@ -4381,19 +4379,18 @@ int32_t fltSclProcessCNF(SArray *sclOpListCNF, SArray *colRangeList) { taosArrayDestroy(colRange->points); colRange->points = points; } - } return TSDB_CODE_SUCCESS; } -static int32_t fltSclCollectOperatorFromNode(SNode* pNode, SArray* sclOpList) { +static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) { if (nodeType(pNode) != QUERY_NODE_OPERATOR) { return TSDB_CODE_SUCCESS; } - SOperatorNode* pOper = (SOperatorNode*)pNode; - //TODO: left value node, right column node - //TODO: datatype - //TODO: operator + SOperatorNode *pOper = (SOperatorNode *)pNode; + // TODO: left value node, right column node + // TODO: datatype + // TODO: operator if (pOper->pLeft == NULL || pOper->pRight == NULL) { return TSDB_CODE_SUCCESS; } @@ -4401,31 +4398,31 @@ static int32_t fltSclCollectOperatorFromNode(SNode* pNode, SArray* sclOpList) { (pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL || pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL || pOper->opType == OP_TYPE_NOT_EQUAL || pOper->opType == OP_TYPE_EQUAL)) { - SValueNode* valNode = (SValueNode*)pOper->pRight; + SValueNode *valNode = (SValueNode *)pOper->pRight; if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { - SNode *p = nodesCloneNode(pNode); - taosArrayPush(sclOpList, &p); + SFltSclOperator sclOp = { + .colNode = nodesCloneNode(pOper->pLeft), .valNode = nodesCloneNode(pOper->pRight), .type = pOper->opType}; + taosArrayPush(sclOpList, &sclOp); } } return TSDB_CODE_SUCCESS; } -static int32_t fltSclCollectOperatorsFromLogicCond(SNode* pNode, SArray* sclOpList) { +static int32_t fltSclCollectOperatorsFromLogicCond(SNode *pNode, SArray *sclOpList) { if (nodeType(pNode) != QUERY_NODE_LOGIC_CONDITION) { return TSDB_CODE_SUCCESS; } - SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pNode; - //TODO: support LOGIC_COND_TYPE_OR + SLogicConditionNode *pLogicCond = (SLogicConditionNode *)pNode; + // TODO: support LOGIC_COND_TYPE_OR if (pLogicCond->condType != LOGIC_COND_TYPE_AND) { return TSDB_CODE_SUCCESS; } - SNode* pExpr = NULL; - FOREACH(pExpr, pLogicCond->pParameterList) { fltSclCollectOperatorFromNode(pExpr, sclOpList); - } + SNode *pExpr = NULL; + FOREACH(pExpr, pLogicCond->pParameterList) { fltSclCollectOperatorFromNode(pExpr, sclOpList); } return TSDB_CODE_SUCCESS; } -static int32_t fltSclCollectOperators(SNode *pNode, SArray* sclOpList) { +static int32_t fltSclCollectOperators(SNode *pNode, SArray *sclOpList) { if (nodeType(pNode) == QUERY_NODE_OPERATOR) { fltSclCollectOperatorFromNode(pNode, sclOpList); } else if (nodeType(pNode) == QUERY_NODE_LOGIC_CONDITION) { @@ -4435,15 +4432,16 @@ static int32_t fltSclCollectOperators(SNode *pNode, SArray* sclOpList) { } int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) { - SArray* sclOpList = taosArrayInit(16, POINTER_BYTES); + SArray *sclOpList = taosArrayInit(16, POINTER_BYTES); fltSclCollectOperators(*pNode, sclOpList); - SArray* colRangeList = taosArrayInit(16, sizeof(SFltSclColumnRange)); + SArray *colRangeList = taosArrayInit(16, sizeof(SFltSclColumnRange)); fltSclProcessCNF(sclOpList, colRangeList); pInfo->sclCtx.fltSclRange = colRangeList; for (int32_t i = 0; i < taosArrayGetSize(sclOpList); ++i) { - SNode* p = taosArrayGetP(sclOpList, i); - nodesDestroyNode(p); + SFltSclOperator *sclOp = taosArrayGet(sclOpList, i); + nodesDestroyNode(sclOp->colNode); + nodesDestroyNode(sclOp->valNode); } taosArrayDestroy(sclOpList); return TSDB_CODE_SUCCESS; From 99620d434da9fdf995a923380416e5138ce5a32a Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 23 May 2023 14:11:52 +0800 Subject: [PATCH 12/42] fix: fix bugs --- source/libs/scalar/src/filter.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 3b61e3a97a..c1de62a407 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4401,7 +4401,7 @@ static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) { SValueNode *valNode = (SValueNode *)pOper->pRight; if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { SFltSclOperator sclOp = { - .colNode = nodesCloneNode(pOper->pLeft), .valNode = nodesCloneNode(pOper->pRight), .type = pOper->opType}; + .colNode = (SColumnNode*)nodesCloneNode(pOper->pLeft), .valNode = (SValueNode*)nodesCloneNode(pOper->pRight), .type = pOper->opType}; taosArrayPush(sclOpList, &sclOp); } } @@ -4440,8 +4440,8 @@ int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) for (int32_t i = 0; i < taosArrayGetSize(sclOpList); ++i) { SFltSclOperator *sclOp = taosArrayGet(sclOpList, i); - nodesDestroyNode(sclOp->colNode); - nodesDestroyNode(sclOp->valNode); + nodesDestroyNode((SNode*)sclOp->colNode); + nodesDestroyNode((SNode*)sclOp->valNode); } taosArrayDestroy(sclOpList); return TSDB_CODE_SUCCESS; From 857cf3d269952a1f4a3010f01f67c2a8d77b9af4 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 23 May 2023 14:29:33 +0800 Subject: [PATCH 13/42] fix: fix bugs --- source/libs/scalar/src/filter.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index c1de62a407..2e0750359f 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3750,6 +3750,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t for (; j < numOfCols; ++j) { if (pDataStatis[j] != NULL && pDataStatis[j]->colId == colRange->colNode->colId) { foundCol = true; + break; } } if (foundCol) { From 22616d235ae120362973ab67cdbe3da0fabf2a0b Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 23 May 2023 16:36:27 +0800 Subject: [PATCH 14/42] fix: fix minior bugs --- source/libs/scalar/src/filter.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 2e0750359f..e7f88908e5 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3598,7 +3598,7 @@ int32_t fltSclMerge(SArray *pts1, SArray *pts2, bool isUnion, SArray *merged) { int32_t countRequired = (isUnion) ? 1 : 2; int32_t count = 0; for (int32_t i = 0; i < taosArrayGetSize(all); ++i) { - SFltSclPoint *pt = taosArrayGet(pts1, i); + SFltSclPoint *pt = taosArrayGet(all, i); if (pt->start) { ++count; if (count == countRequired) { @@ -4402,7 +4402,9 @@ static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) { SValueNode *valNode = (SValueNode *)pOper->pRight; if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { SFltSclOperator sclOp = { - .colNode = (SColumnNode*)nodesCloneNode(pOper->pLeft), .valNode = (SValueNode*)nodesCloneNode(pOper->pRight), .type = pOper->opType}; + .colNode = (SColumnNode*)nodesCloneNode(pOper->pLeft), + .valNode = (SValueNode*)nodesCloneNode(pOper->pRight), + .type = pOper->opType}; taosArrayPush(sclOpList, &sclOp); } } @@ -4433,7 +4435,7 @@ static int32_t fltSclCollectOperators(SNode *pNode, SArray *sclOpList) { } int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) { - SArray *sclOpList = taosArrayInit(16, POINTER_BYTES); + SArray *sclOpList = taosArrayInit(16, sizeof(SFltSclOperator)); fltSclCollectOperators(*pNode, sclOpList); SArray *colRangeList = taosArrayInit(16, sizeof(SFltSclColumnRange)); fltSclProcessCNF(sclOpList, colRangeList); From 2e4c14c18ce9a87f38eabe8c249fa26d04f70dbe Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 23 May 2023 21:30:41 +0800 Subject: [PATCH 15/42] fix: fix bugs --- source/libs/scalar/src/filter.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index e7f88908e5..264e41b006 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3514,10 +3514,7 @@ int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { if (val2->kind == FLT_SCL_DATUM_KIND_NULL || val2->kind == FLT_SCL_DATUM_KIND_MIN || val2->kind == FLT_SCL_DATUM_KIND_MAX) { - if (val1->kind == FLT_SCL_DATUM_KIND_NULL || val1->kind == FLT_SCL_DATUM_KIND_MIN || - val1->kind == FLT_SCL_DATUM_KIND_MAX) { return (val1->kind < val2->kind) ? -1 : ((val1->kind > val2->kind) ? 1 : 0); - } } switch (val2->kind) { @@ -3690,7 +3687,12 @@ int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum *datum, uint8_t type, int64_t val) { switch (type) { - case TSDB_DATA_TYPE_BOOL: { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: { datum->kind = FLT_SCL_DATUM_KIND_INT64; datum->i = val; break; @@ -3711,7 +3713,8 @@ int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum *datum, uint8_t type, int } // TODO:varchar/nchar/json default: { - qError("not supported"); + datum->kind = FLT_SCL_DATUM_KIND_NULL; + qError("not supported type when build datum from block sma value"); break; } } From 383c828c856df00a752b9843a615c468993f9ef3 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 24 May 2023 11:18:44 +0800 Subject: [PATCH 16/42] fix: before review --- source/libs/scalar/src/filter.c | 90 +++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 32 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 264e41b006..ecf621a726 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3472,7 +3472,7 @@ int32_t fltSclCompareWithFloat64(SFltSclDatum *val1, SFltSclDatum *val2) { } // TODO: varchar, nchar default: - qError("not support comparsion. %d, %d", val1->kind, val2->kind); + qError("not supported comparsion. kind1 %d, kind2 %d", val1->kind, val2->kind); return (val1->kind - val2->kind); } } @@ -3489,7 +3489,7 @@ int32_t fltSclCompareWithInt64(SFltSclDatum *val1, SFltSclDatum *val2) { } // TODO: varchar, nchar default: - qError("not support comparsion. %d, %d", val1->kind, val2->kind); + qError("not supported comparsion. kind1 %d, kind2 %d", val1->kind, val2->kind); return (val1->kind - val2->kind); } } @@ -3506,7 +3506,7 @@ int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { } // TODO: varchar, nchar default: - qError("not support comparsion. %d, %d", val1->kind, val2->kind); + qError("not supported comparsion. kind1 %d, kind2 %d", val1->kind, val2->kind); return (val1->kind - val2->kind); } } @@ -3514,7 +3514,7 @@ int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { if (val2->kind == FLT_SCL_DATUM_KIND_NULL || val2->kind == FLT_SCL_DATUM_KIND_MIN || val2->kind == FLT_SCL_DATUM_KIND_MAX) { - return (val1->kind < val2->kind) ? -1 : ((val1->kind > val2->kind) ? 1 : 0); + return (val1->kind < val2->kind) ? -1 : ((val1->kind > val2->kind) ? 1 : 0); } switch (val2->kind) { @@ -3529,7 +3529,7 @@ int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { } // TODO: varchar/nchar default: - fltError("not supported kind. just return 0"); + qError("not supported kind when compare datum. kind2 : %d", val2->kind); return 0; break; } @@ -3585,7 +3585,6 @@ int32_t fltSclMergeSort(SArray *pts1, SArray *pts2, SArray *result) { return 0; } -// pts1 and pts2 must be ordered and de-duplicated and each range can not be a range of another range int32_t fltSclMerge(SArray *pts1, SArray *pts2, bool isUnion, SArray *merged) { size_t len1 = taosArrayGetSize(pts1); size_t len2 = taosArrayGetSize(pts2); @@ -3616,7 +3615,6 @@ int32_t fltSclIntersect(SArray *pts1, SArray *pts2, SArray *merged) { return flt int32_t fltSclUnion(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, true, merged); } -// TODO: column, constant typedef struct { SColumnNode *colNode; SValueNode *valNode; @@ -3675,9 +3673,9 @@ int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) datum->d = valNode->datum.d; break; } - // TODO:varchar/nchar/json + // TODO:varchar/nchar/json default: { - qError("not supported"); + qError("not supported type %d when build datum from value node", valNode->node.resType.type); break; } } @@ -3687,11 +3685,11 @@ int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum *datum, uint8_t type, int64_t val) { switch (type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { datum->kind = FLT_SCL_DATUM_KIND_INT64; datum->i = val; @@ -3711,10 +3709,10 @@ int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum *datum, uint8_t type, int datum->d = *(double *)&val; break; } - // TODO:varchar/nchar/json + // TODO:varchar/nchar/json default: { datum->kind = FLT_SCL_DATUM_KIND_NULL; - qError("not supported type when build datum from block sma value"); + qError("not supported type %d when build datum from block sma value", type); break; } } @@ -3732,6 +3730,13 @@ int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange *colRange, SColumnDataAg taosArrayPush(points, &endPt); return TSDB_CODE_SUCCESS; } + if (pAgg->numOfNull > 0) { + SFltSclDatum nullDatum = {.kind = FLT_SCL_DATUM_KIND_NULL}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = nullDatum}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = nullDatum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + } SFltSclDatum min; fltSclBuildDatumFromBlockSmaValue(&min, colRange->colNode->node.resType.type, pAgg->min); SFltSclPoint minPt = {.excl = false, .start = true, .val = min}; @@ -4357,8 +4362,25 @@ int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { } break; } + case OP_TYPE_IS_NULL: { + SFltSclDatum nullDatum = {.kind = FLT_SCL_DATUM_KIND_NULL}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = nullDatum}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = nullDatum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + case OP_TYPE_IS_NOT_NULL: { + SFltSclDatum minDatum = {.kind = FLT_SCL_DATUM_KIND_MIN, .type = oper->colNode->node.resType}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = minDatum}; + SFltSclDatum maxDatum = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = maxDatum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } default: { - qError("not supported op"); + qError("not supported operator type : %d when build range points", oper->type); break; } } @@ -4391,26 +4413,30 @@ static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) { if (nodeType(pNode) != QUERY_NODE_OPERATOR) { return TSDB_CODE_SUCCESS; } + SOperatorNode *pOper = (SOperatorNode *)pNode; - // TODO: left value node, right column node - // TODO: datatype - // TODO: operator if (pOper->pLeft == NULL || pOper->pRight == NULL) { return TSDB_CODE_SUCCESS; } - if (nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_VALUE && - (pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL || - pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL || - pOper->opType == OP_TYPE_NOT_EQUAL || pOper->opType == OP_TYPE_EQUAL)) { - SValueNode *valNode = (SValueNode *)pOper->pRight; - if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { - SFltSclOperator sclOp = { - .colNode = (SColumnNode*)nodesCloneNode(pOper->pLeft), - .valNode = (SValueNode*)nodesCloneNode(pOper->pRight), - .type = pOper->opType}; - taosArrayPush(sclOpList, &sclOp); - } + + if (!(pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL || + pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL || + pOper->opType == OP_TYPE_NOT_EQUAL || pOper->opType == OP_TYPE_EQUAL)) { + return TSDB_CODE_SUCCESS; } + + if (!(nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_VALUE)) { + return TSDB_CODE_SUCCESS; + } + + SValueNode *valNode = (SValueNode *)pOper->pRight; + if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { + SFltSclOperator sclOp = {.colNode = (SColumnNode *)nodesCloneNode(pOper->pLeft), + .valNode = (SValueNode *)nodesCloneNode(pOper->pRight), + .type = pOper->opType}; + taosArrayPush(sclOpList, &sclOp); + } + return TSDB_CODE_SUCCESS; } @@ -4446,8 +4472,8 @@ int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) for (int32_t i = 0; i < taosArrayGetSize(sclOpList); ++i) { SFltSclOperator *sclOp = taosArrayGet(sclOpList, i); - nodesDestroyNode((SNode*)sclOp->colNode); - nodesDestroyNode((SNode*)sclOp->valNode); + nodesDestroyNode((SNode *)sclOp->colNode); + nodesDestroyNode((SNode *)sclOp->valNode); } taosArrayDestroy(sclOpList); return TSDB_CODE_SUCCESS; From 5b1e0be7560ebc658aec864e6db2b7fabcce15bb Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 24 May 2023 13:41:21 +0800 Subject: [PATCH 17/42] enhance: add log --- source/libs/scalar/src/filter.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index ecf621a726..3c01edf857 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3446,10 +3446,10 @@ typedef enum { typedef struct { SFltSclDatumKind kind; union { - int64_t i; // for int64, uint64 and double and bool (1 true, 0 false) - uint64_t u; - double d; - uint8_t *pData; // for varchar, nchar + int64_t i; // for int and bool (1 true, 0 false) and ts + uint64_t u; // for uint + double d; // for double + uint8_t *pData; // for varchar, nchar, len prefixed }; SDataType type; // TODO: original data type, may not be used? } SFltSclDatum; @@ -3765,9 +3765,15 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t SColumnDataAgg *pAgg = pDataStatis[j]; SArray *points = taosArrayInit(2, sizeof(SFltSclPoint)); fltSclBuildRangeFromBlockSma(colRange, pAgg, numOfRows, points); + qDebug("column data agg: nulls %d, rows %d, max %" PRId64 " min " PRId64, pAgg->numOfNull, numOfRows, pAgg->max, + pAgg->min); + SArray *merged = taosArrayInit(8, sizeof(SFltSclPoint)); fltSclIntersect(points, colRange->points, merged); bool isIntersect = taosArrayGetSize(merged) != 0; + qDebug("filter range execute, scalar mode, column range found. colId: %d colName: %s has overlap: %d", + colRange->colNode->colId, colRange->colNode->colName, isIntersect); + taosArrayDestroy(merged); taosArrayDestroy(points); if (!isIntersect) { From e5f9d78b6d0df0c0fa9970a29ce9f1d2617ae52a Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 24 May 2023 16:39:38 +0800 Subject: [PATCH 18/42] fix: rocksdb so --- cmake/cmake.options | 5 -- contrib/CMakeLists.txt | 56 --------------------- contrib/test/CMakeLists.txt | 3 -- source/dnode/mgmt/mgmt_dnode/CMakeLists.txt | 2 +- source/dnode/vnode/CMakeLists.txt | 2 - source/libs/stream/CMakeLists.txt | 2 - 6 files changed, 1 insertion(+), 69 deletions(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index a7e62350df..064b1f7c6f 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -106,11 +106,6 @@ option( OFF ) -option( - BUILD_WITH_ROCKSDB - "If build with rocksdb" - ON -) option( BUILD_WITH_SQLITE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 536d4eae8e..ad3d8bc326 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -77,11 +77,6 @@ if(${BUILD_WITH_LEVELDB}) cat("${TD_SUPPORT_DIR}/leveldb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_WITH_LEVELDB}) -# rocksdb -if(${BUILD_WITH_ROCKSDB}) - cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) - add_definitions(-DUSE_ROCKSDB) -endif(${BUILD_WITH_ROCKSDB}) # canonical-raft if(${BUILD_WITH_CRAFT}) @@ -220,57 +215,6 @@ if(${BUILD_WITH_LEVELDB}) ) endif(${BUILD_WITH_LEVELDB}) -# rocksdb -# To support rocksdb build on ubuntu: sudo apt-get install libgflags-dev -if(${BUILD_WITH_ROCKSDB}) - if(${TD_LINUX}) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized -Wno-error=unused-but-set-variable -Wno-error=unused-variable -Wno-error=unused-function -Wno-errno=unused-private-field -Wno-error=unused-result") - endif(${TD_LINUX}) - - if(${TD_DARWIN}) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized") - endif(${TD_DARWIN}) - - if (${TD_WINDOWS}) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819") - endif(${TD_WINDOWS}) - - - if(${TD_DARWIN}) - option(HAVE_THREAD_LOCAL "" OFF) - option(WITH_IOSTATS_CONTEXT "" OFF) - option(WITH_PERF_CONTEXT "" OFF) - endif(${TD_DARWIN}) - - if(${TD_WINDOWS}) - option(WITH_JNI "" ON) - endif(${TD_WINDOWS}) - - if(${TD_WINDOWS}) - option(WITH_MD_LIBRARY "build with MD" OFF) - set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib) - endif(${TD_WINDOWS}) - - - option(WITH_FALLOCATE "" OFF) - option(WITH_JEMALLOC "" OFF) - option(WITH_GFLAGS "" OFF) - option(PORTABLE "" ON) - option(WITH_LIBURING "" OFF) - option(FAIL_ON_WARNINGS OFF) - - option(WITH_TESTS "" OFF) - option(WITH_BENCHMARK_TOOLS "" OFF) - option(WITH_TOOLS "" OFF) - option(WITH_LIBURING "" OFF) - - option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF) - add_subdirectory(rocksdb EXCLUDE_FROM_ALL) - target_include_directories( - rocksdb - PUBLIC $ - ) -endif(${BUILD_WITH_ROCKSDB}) # lucene # To support build on ubuntu: sudo apt-get install libboost-all-dev diff --git a/contrib/test/CMakeLists.txt b/contrib/test/CMakeLists.txt index f35cf0d13d..8d095518e2 100644 --- a/contrib/test/CMakeLists.txt +++ b/contrib/test/CMakeLists.txt @@ -1,7 +1,4 @@ # rocksdb -if(${BUILD_WITH_ROCKSDB}) - add_subdirectory(rocksdb) -endif(${BUILD_WITH_ROCKSDB}) if(${BUILD_WITH_LUCENE}) add_subdirectory(lucene) diff --git a/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt b/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt index fdd0830a58..7d0b6d617d 100644 --- a/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt +++ b/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt @@ -9,5 +9,5 @@ target_include_directories( PUBLIC "${GRANT_CFG_INCLUDE_DIR}" ) target_link_libraries( - mgmt_dnode node_util + mgmt_dnode node_util rocksdb ) \ No newline at end of file diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index e8660cd6ad..79a7d55198 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -117,9 +117,7 @@ if(${BUILD_WITH_INVERTEDINDEX}) add_definitions(-DUSE_INVERTED_INDEX) endif(${BUILD_WITH_INVERTEDINDEX}) -if(${BUILD_WITH_ROCKSDB}) add_definitions(-DUSE_ROCKSDB) -endif(${BUILD_WITH_ROCKSDB}) diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt index 2edbc44aae..5952250b3e 100644 --- a/source/libs/stream/CMakeLists.txt +++ b/source/libs/stream/CMakeLists.txt @@ -6,7 +6,6 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -if(${BUILD_WITH_ROCKSDB}) target_link_libraries( stream PUBLIC rocksdb tdb @@ -19,7 +18,6 @@ if(${BUILD_WITH_ROCKSDB}) ) add_definitions(-DUSE_ROCKSDB) -endif(${BUILD_WITH_ROCKSDB}) #target_link_libraries( From d7b1613d850eed2d65faf94c84f58a07320082b8 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 24 May 2023 16:42:50 +0800 Subject: [PATCH 19/42] Revert "fix: rocksdb so" This reverts commit e5f9d78b6d0df0c0fa9970a29ce9f1d2617ae52a. --- cmake/cmake.options | 5 ++ contrib/CMakeLists.txt | 56 +++++++++++++++++++++ contrib/test/CMakeLists.txt | 3 ++ source/dnode/mgmt/mgmt_dnode/CMakeLists.txt | 2 +- source/dnode/vnode/CMakeLists.txt | 2 + source/libs/stream/CMakeLists.txt | 2 + 6 files changed, 69 insertions(+), 1 deletion(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index 064b1f7c6f..a7e62350df 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -106,6 +106,11 @@ option( OFF ) +option( + BUILD_WITH_ROCKSDB + "If build with rocksdb" + ON +) option( BUILD_WITH_SQLITE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index ad3d8bc326..536d4eae8e 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -77,6 +77,11 @@ if(${BUILD_WITH_LEVELDB}) cat("${TD_SUPPORT_DIR}/leveldb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_WITH_LEVELDB}) +# rocksdb +if(${BUILD_WITH_ROCKSDB}) + cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + add_definitions(-DUSE_ROCKSDB) +endif(${BUILD_WITH_ROCKSDB}) # canonical-raft if(${BUILD_WITH_CRAFT}) @@ -215,6 +220,57 @@ if(${BUILD_WITH_LEVELDB}) ) endif(${BUILD_WITH_LEVELDB}) +# rocksdb +# To support rocksdb build on ubuntu: sudo apt-get install libgflags-dev +if(${BUILD_WITH_ROCKSDB}) + if(${TD_LINUX}) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized -Wno-error=unused-but-set-variable -Wno-error=unused-variable -Wno-error=unused-function -Wno-errno=unused-private-field -Wno-error=unused-result") + endif(${TD_LINUX}) + + if(${TD_DARWIN}) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized") + endif(${TD_DARWIN}) + + if (${TD_WINDOWS}) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819") + endif(${TD_WINDOWS}) + + + if(${TD_DARWIN}) + option(HAVE_THREAD_LOCAL "" OFF) + option(WITH_IOSTATS_CONTEXT "" OFF) + option(WITH_PERF_CONTEXT "" OFF) + endif(${TD_DARWIN}) + + if(${TD_WINDOWS}) + option(WITH_JNI "" ON) + endif(${TD_WINDOWS}) + + if(${TD_WINDOWS}) + option(WITH_MD_LIBRARY "build with MD" OFF) + set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib) + endif(${TD_WINDOWS}) + + + option(WITH_FALLOCATE "" OFF) + option(WITH_JEMALLOC "" OFF) + option(WITH_GFLAGS "" OFF) + option(PORTABLE "" ON) + option(WITH_LIBURING "" OFF) + option(FAIL_ON_WARNINGS OFF) + + option(WITH_TESTS "" OFF) + option(WITH_BENCHMARK_TOOLS "" OFF) + option(WITH_TOOLS "" OFF) + option(WITH_LIBURING "" OFF) + + option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF) + add_subdirectory(rocksdb EXCLUDE_FROM_ALL) + target_include_directories( + rocksdb + PUBLIC $ + ) +endif(${BUILD_WITH_ROCKSDB}) # lucene # To support build on ubuntu: sudo apt-get install libboost-all-dev diff --git a/contrib/test/CMakeLists.txt b/contrib/test/CMakeLists.txt index 8d095518e2..f35cf0d13d 100644 --- a/contrib/test/CMakeLists.txt +++ b/contrib/test/CMakeLists.txt @@ -1,4 +1,7 @@ # rocksdb +if(${BUILD_WITH_ROCKSDB}) + add_subdirectory(rocksdb) +endif(${BUILD_WITH_ROCKSDB}) if(${BUILD_WITH_LUCENE}) add_subdirectory(lucene) diff --git a/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt b/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt index 7d0b6d617d..fdd0830a58 100644 --- a/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt +++ b/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt @@ -9,5 +9,5 @@ target_include_directories( PUBLIC "${GRANT_CFG_INCLUDE_DIR}" ) target_link_libraries( - mgmt_dnode node_util rocksdb + mgmt_dnode node_util ) \ No newline at end of file diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 79a7d55198..e8660cd6ad 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -117,7 +117,9 @@ if(${BUILD_WITH_INVERTEDINDEX}) add_definitions(-DUSE_INVERTED_INDEX) endif(${BUILD_WITH_INVERTEDINDEX}) +if(${BUILD_WITH_ROCKSDB}) add_definitions(-DUSE_ROCKSDB) +endif(${BUILD_WITH_ROCKSDB}) diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt index 5952250b3e..2edbc44aae 100644 --- a/source/libs/stream/CMakeLists.txt +++ b/source/libs/stream/CMakeLists.txt @@ -6,6 +6,7 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) +if(${BUILD_WITH_ROCKSDB}) target_link_libraries( stream PUBLIC rocksdb tdb @@ -18,6 +19,7 @@ target_include_directories( ) add_definitions(-DUSE_ROCKSDB) +endif(${BUILD_WITH_ROCKSDB}) #target_link_libraries( From 768372fc7bca72b43b47a4dc29ebc3b96dab4c6d Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 24 May 2023 16:45:06 +0800 Subject: [PATCH 20/42] fix: fix compilation warning --- source/libs/scalar/src/filter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 3c01edf857..27cf9da1d5 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3765,7 +3765,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t SColumnDataAgg *pAgg = pDataStatis[j]; SArray *points = taosArrayInit(2, sizeof(SFltSclPoint)); fltSclBuildRangeFromBlockSma(colRange, pAgg, numOfRows, points); - qDebug("column data agg: nulls %d, rows %d, max %" PRId64 " min " PRId64, pAgg->numOfNull, numOfRows, pAgg->max, + qDebug("column data agg: nulls %d, rows %d, max %" PRId64 " min %" PRId64, pAgg->numOfNull, numOfRows, pAgg->max, pAgg->min); SArray *merged = taosArrayInit(8, sizeof(SFltSclPoint)); From de1848cb08726936958a51f6667ca33e49eb6995 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 25 May 2023 14:51:53 +0800 Subject: [PATCH 21/42] enhance: use scalar mode to run ci test --- source/libs/function/src/udfd.c | 2 +- source/libs/scalar/src/filter.c | 2 +- source/libs/scalar/src/sclvector.c | 17 ++++++++++++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index b6ab7bd6f1..3b827a2f99 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -600,9 +600,9 @@ SUdf *udfdGetOrCreateUdf(const char *udfName) { return udf; } else { (*pUdfHash)->expired = true; - taosHashRemove(global.udfsHash, udfName, strlen(udfName)); fnInfo("udfd expired, check for new version. existing udf %s udf version %d, udf created time %" PRIx64, (*pUdfHash)->name, (*pUdfHash)->version, (*pUdfHash)->createdTime); + taosHashRemove(global.udfsHash, udfName, strlen(udfName)); } } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 27cf9da1d5..464d388db4 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4556,7 +4556,7 @@ int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pInfo, uint32_t options) FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat)); - info->scalarMode = stat.scalarMode; + info->scalarMode = true; fltDebug("scalar mode: %d", info->scalarMode); if (!info->scalarMode) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 4d803cb638..6983f6313b 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1784,6 +1784,9 @@ void vectorNotMatch(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOu void vectorIsNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) { for (int32_t i = 0; i < pLeft->numOfRows; ++i) { int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 1 : 0; + if (v) { + ++pOut->numOfQualified; + } colDataSetInt8(pOut->columnData, i, &v); } pOut->numOfRows = pLeft->numOfRows; @@ -1792,6 +1795,9 @@ void vectorIsNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, void vectorNotNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) { for (int32_t i = 0; i < pLeft->numOfRows; ++i) { int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 0 : 1; + if (v) { + ++pOut->numOfQualified; + } colDataSetInt8(pOut->columnData, i, &v); } pOut->numOfRows = pLeft->numOfRows; @@ -1805,6 +1811,13 @@ void vectorIsTrue(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, colDataSetInt8(pOut->columnData, i, &v); colDataClearNull_f(pOut->columnData->nullbitmap, i); } + { + bool v = false; + GET_TYPED_DATA(v, bool, pOut->columnData->info.type, colDataGetData(pOut->columnData, i)); + if (v) { + ++pOut->numOfQualified; + } + } } pOut->columnData->hasNull = false; } @@ -1844,7 +1857,9 @@ void vectorJsonContains(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam char *pLeftData = colDataGetVarData(pLeft->columnData, i); getJsonValue(pLeftData, jsonKey, &isExist); } - + if (isExist) { + ++pOut->numOfQualified; + } colDataSetVal(pOutputCol, i, (const char *)(&isExist), false); } taosMemoryFree(jsonKey); From 61b063529384019a954f05fdcb247dc8623537ed Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 25 May 2023 16:27:48 +0800 Subject: [PATCH 22/42] fix: get time range using new scalar mode range --- source/libs/scalar/src/filter.c | 46 +++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 464d388db4..ae4712b9ba 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3765,8 +3765,8 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t SColumnDataAgg *pAgg = pDataStatis[j]; SArray *points = taosArrayInit(2, sizeof(SFltSclPoint)); fltSclBuildRangeFromBlockSma(colRange, pAgg, numOfRows, points); - qDebug("column data agg: nulls %d, rows %d, max %" PRId64 " min %" PRId64, pAgg->numOfNull, numOfRows, pAgg->max, - pAgg->min); + qDebug("column data agg: nulls %d, rows %d, max %" PRId64 " min %" PRId64, pAgg->numOfNull, numOfRows, + pAgg->max, pAgg->min); SArray *merged = taosArrayInit(8, sizeof(SFltSclPoint)); fltSclIntersect(points, colRange->points, merged); @@ -3960,6 +3960,31 @@ _return: return code; } +static int32_t fltSclGetDatumValueFromPoint(SFltSclPoint *point, SFltSclDatum *d) { + *d = point->val; + if (point->val.kind == FLT_SCL_DATUM_KIND_NULL) { + return TSDB_CODE_SUCCESS; + } + if (point->val.kind == FLT_SCL_DATUM_KIND_MAX) { + getDataMax(d->type.type, &(d->i)); + } else if (point->val.kind == FLT_SCL_DATUM_KIND_MIN) { + getDataMin(d->type.type, &(d->i)); + } + + if ((point->val.kind == FLT_SCL_DATUM_KIND_INT64) || (point->val.kind = FLT_SCL_DATUM_KIND_UINT64)) { + if (point->excl) { + if (point->start) { + ++d->i; + } else { + --d->i; + } + } + } else { + qError("not supported kind %d when get datum from point", point->val.kind); + } + return TSDB_CODE_SUCCESS; +} + int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) { SFilterInfo *info = NULL; int32_t code = 0; @@ -3969,6 +3994,23 @@ int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) { FLT_ERR_RET(filterInitFromNode(pNode, &info, FLT_OPTION_NO_REWRITE | FLT_OPTION_TIMESTAMP)); if (info->scalarMode) { + SArray *colRanges = info->sclCtx.fltSclRange; + if (taosArrayGetSize(colRanges) == 1) { + SFltSclColumnRange *colRange = taosArrayGet(colRanges, 0); + SArray *points = colRange->points; + if (taosArrayGetSize(points) == 2) { + SFltSclPoint *startPt = taosArrayGet(points, 0); + SFltSclPoint *endPt = taosArrayGet(points, 1); + SFltSclDatum start; + SFltSclDatum end; + fltSclGetDatumValueFromPoint(startPt, &start); + fltSclGetDatumValueFromPoint(endPt, &end); + win->skey = start.i; + win->ekey = end.i; + *isStrict = true; + goto _return; + } + } *win = TSWINDOW_INITIALIZER; *isStrict = false; goto _return; From d9042f5ed38bdd9529709d685f555d49b43b558e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 25 May 2023 17:24:11 +0800 Subject: [PATCH 23/42] fix:memory leak --- source/client/inc/clientSml.h | 2 +- source/client/src/clientSml.c | 69 ++++++++++++++++------------- source/client/src/clientSmlJson.c | 15 +++++-- source/client/src/clientSmlLine.c | 8 +++- source/client/src/clientSmlTelnet.c | 5 ++- tests/parallel_test/cases.task | 4 +- 6 files changed, 64 insertions(+), 39 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index f9cbffa748..c9eb951014 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -251,7 +251,7 @@ int32_t smlClearForRerun(SSmlHandle *info); int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg); uint8_t smlGetTimestampLen(int64_t num); void clearColValArray(SArray* pCols); -void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag); +void smlDestroyTableInfo(void *para); void freeSSmlKv(void* data); int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 92e4d0f8da..5748c06e8b 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -230,6 +230,16 @@ void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tin } } +static void smlDestroySTableMeta(void *para) { + SSmlSTableMeta *meta = *(SSmlSTableMeta**)para; + taosHashCleanup(meta->tagHash); + taosHashCleanup(meta->colHash); + taosArrayDestroy(meta->tags); + taosArrayDestroy(meta->cols); + taosMemoryFree(meta->tableMeta); + taosMemoryFree(meta); +} + SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); if (!meta) { @@ -264,7 +274,7 @@ SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { return meta; cleanup: - taosMemoryFree(meta); + smlDestroySTableMeta(meta); return NULL; } @@ -1093,15 +1103,6 @@ static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) { } } -static void smlDestroySTableMeta(SSmlSTableMeta *meta) { - taosHashCleanup(meta->tagHash); - taosHashCleanup(meta->colHash); - taosArrayDestroy(meta->tags); - taosArrayDestroy(meta->cols); - taosMemoryFree(meta->tableMeta); - taosMemoryFree(meta); -} - static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg) { for (int i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); @@ -1141,7 +1142,8 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols return TSDB_CODE_SUCCESS; } -void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) { +void smlDestroyTableInfo(void *para) { + SSmlTableInfo *tag = *(SSmlTableInfo**)para; for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i); taosHashCleanup(kvHash); @@ -1178,18 +1180,18 @@ void smlDestroyInfo(SSmlHandle *info) { qDestroyQuery(info->pQuery); // destroy info->childTables - SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); - while (oneTable) { - smlDestroyTableInfo(info, *oneTable); - oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); - } +// SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); +// while (oneTable) { +// smlDestroyTableInfo(oneTable); +// oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); +// } // destroy info->superTables - SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); - while (oneSTable) { - smlDestroySTableMeta(*oneSTable); - oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); - } +// SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); +// while (oneSTable) { +// smlDestroySTableMeta(*oneSTable); +// oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); +// } // destroy info->pVgHash taosHashCleanup(info->pVgHash); @@ -1248,6 +1250,8 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) { info->childTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); info->tableUids = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); info->superTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + taosHashSetFreeFp(info->superTables, smlDestroySTableMeta); + taosHashSetFreeFp(info->childTables, smlDestroyTableInfo); info->id = smlGenId(); info->pQuery = smlInitHandle(); @@ -1354,6 +1358,9 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); + if(meta == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); terrno = 0; smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); @@ -1473,18 +1480,18 @@ static void smlPrintStatisticInfo(SSmlHandle *info) { int32_t smlClearForRerun(SSmlHandle *info) { info->reRun = false; // clear info->childTables - SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); - while (oneTable) { - smlDestroyTableInfo(info, *oneTable); - oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); - } +// SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); +// while (oneTable) { +// smlDestroyTableInfo(info, *oneTable); +// oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); +// } // clear info->superTables - SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); - while (oneSTable) { - smlDestroySTableMeta(*oneSTable); - oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); - } +// SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); +// while (oneSTable) { +// smlDestroySTableMeta(*oneSTable); +// oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); +// } taosHashClear(info->childTables); taosHashClear(info->superTables); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 7ccf930964..ad5b199c4e 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -695,6 +695,9 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo return TSDB_CODE_SUCCESS; } sMeta = smlBuildSTableMeta(info->dataFormat); + if(sMeta == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } sMeta->tableMeta = pTableMeta; taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES); for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){ @@ -784,7 +787,7 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); if (tinfo->tableDataCtx == NULL) { smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); - smlDestroyTableInfo(info, tinfo); + smlDestroyTableInfo(&tinfo); return TSDB_CODE_SML_INVALID_DATA; } } @@ -1048,12 +1051,18 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { return TSDB_CODE_TSC_INVALID_JSON; } - info->lineNum = payloadNum; - info->dataFormat = true; + if (unlikely(info->lines != NULL)) { + for (int i = 0; i < info->lineNum; i++) { + taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv); + if (info->lines[i].measureTagsLen != 0) taosMemoryFree(info->lines[i].measureTag); + } taosMemoryFree(info->lines); info->lines = NULL; } + info->lineNum = payloadNum; + info->dataFormat = true; + ret = smlClearForRerun(info); if (ret != TSDB_CODE_SUCCESS) { return ret; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index e79093398e..4bf4ae091f 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -168,6 +168,9 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_SUCCESS; } sMeta = smlBuildSTableMeta(info->dataFormat); + if(sMeta == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } sMeta->tableMeta = pTableMeta; taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES); for (int i = pTableMeta->tableInfo.numOfColumns; @@ -326,7 +329,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin info->currSTableMeta->uid = tinfo->uid; tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); if (tinfo->tableDataCtx == NULL) { - smlDestroyTableInfo(info, tinfo); + smlDestroyTableInfo(&tinfo); smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); return TSDB_CODE_SML_INVALID_DATA; } @@ -372,6 +375,9 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_SUCCESS; } *tmp = smlBuildSTableMeta(info->dataFormat); + if(*tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } (*tmp)->tableMeta = pTableMeta; taosHashPut(info->superTables, currElement->measure, currElement->measureLen, tmp, POINTER_BYTES); diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index 42b8001e59..6dd8842797 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -91,6 +91,9 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS return TSDB_CODE_SUCCESS; } sMeta = smlBuildSTableMeta(info->dataFormat); + if(sMeta == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } sMeta->tableMeta = pTableMeta; taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES); for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){ @@ -212,7 +215,7 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); if (tinfo->tableDataCtx == NULL) { smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); - smlDestroyTableInfo(info, tinfo); + smlDestroyTableInfo(&tinfo); return TSDB_CODE_SML_INVALID_DATA; } } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ab4663e5b3..b09d8f68ec 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -571,7 +571,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py -#,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_stable.py @@ -1089,7 +1089,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 4 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 4 From 8ee1288611ce3a922e296a6c1519df7a95aca43d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 25 May 2023 17:47:55 +0800 Subject: [PATCH 24/42] fix(cache): remove deleted data --- source/dnode/vnode/src/tsdb/tsdbCache.c | 62 ++++++++++++---------- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 6 +-- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index d33f04c145..830f44b411 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -937,14 +937,14 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE } // build keys & multi get from rocks - int num_keys = pTSchema->numOfCols; - char **keys_list = taosMemoryCalloc(num_keys * 2, sizeof(char *)); - size_t *keys_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t)); + int num_keys = pTSchema->numOfCols; + char **keys_list = taosMemoryCalloc(num_keys * 2, sizeof(char *)); + size_t *keys_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t)); + const size_t klen = ROCKS_KEY_LEN; for (int i = 0; i < num_keys; ++i) { int16_t cid = pTSchema->columns[i].colId; - size_t klen = ROCKS_KEY_LEN; - char *keys = taosMemoryCalloc(2, sizeof(SLastKey)); + char *keys = taosMemoryCalloc(2, sizeof(SLastKey)); ((SLastKey *)keys)[0] = (SLastKey){.ltype = 1, .uid = uid, .cid = cid}; ((SLastKey *)keys)[1] = (SLastKey){.ltype = 0, .uid = uid, .cid = cid}; @@ -960,39 +960,35 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE rocksMayWrite(pTsdb, true, false, false); rocksdb_multi_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, num_keys * 2, (const char *const *)keys_list, keys_list_sizes, values_list, values_list_sizes, errs); - for (int i = 0; i < num_keys; ++i) { - taosMemoryFree(keys_list[i]); - } for (int i = 0; i < num_keys * 2; ++i) { - rocksdb_free(errs[i]); + if (errs[i]) { + rocksdb_free(errs[i]); + } } - taosMemoryFree(keys_list); - taosMemoryFree(keys_list_sizes); taosMemoryFree(errs); rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; for (int i = 0; i < num_keys; ++i) { SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i]); if (NULL != pLastCol && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) { - SLastKey *key = &(SLastKey){.ltype = 1, .uid = uid, .cid = pLastCol->colVal.cid}; - size_t klen = ROCKS_KEY_LEN; - - rocksdb_writebatch_delete(wb, (char *)key, klen); - taosLRUCacheErase(pTsdb->lruCache, key, klen); + rocksdb_writebatch_delete(wb, keys_list[i], klen); } + taosLRUCacheErase(pTsdb->lruCache, keys_list[i], klen); pLastCol = tsdbCacheDeserialize(values_list[i + num_keys]); if (NULL != pLastCol && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) { - SLastKey *key = &(SLastKey){.ltype = 0, .uid = uid, .cid = pLastCol->colVal.cid}; - size_t klen = ROCKS_KEY_LEN; - - rocksdb_writebatch_delete(wb, (char *)key, klen); - taosLRUCacheErase(pTsdb->lruCache, key, klen); + rocksdb_writebatch_delete(wb, keys_list[num_keys + i], klen); } + taosLRUCacheErase(pTsdb->lruCache, keys_list[num_keys + i], klen); rocksdb_free(values_list[i]); rocksdb_free(values_list[i + num_keys]); } + for (int i = 0; i < num_keys; ++i) { + taosMemoryFree(keys_list[i]); + } + taosMemoryFree(keys_list); + taosMemoryFree(keys_list_sizes); taosMemoryFree(values_list); taosMemoryFree(values_list_sizes); @@ -1871,10 +1867,14 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie if (isLast && (pColData->flag & HAS_VALUE)) { skipBlock = false; break; - } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { + } /*else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { skipBlock = false; break; - } + }*/ + } + + if (!isLast) { + skipBlock = false; } if (skipBlock) { @@ -1908,6 +1908,9 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie if (checkRemainingRow) { bool skipBlock = true; int inputColIndex = 0; + if (aCols[0] == PRIMARYKEY_TIMESTAMP_COL_ID) { + ++inputColIndex; + } for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) { SColData *pColData = &state->pBlockData->aColData[colIndex]; int16_t cid = pColData->cid; @@ -1916,15 +1919,19 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie if (isLast && (pColData->flag & HAS_VALUE)) { skipBlock = false; break; - } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { + } /*else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { skipBlock = false; break; - } + }*/ ++inputColIndex; } } + if (!isLast) { + skipBlock = false; + } + if (skipBlock) { if (--state->iBlock < 0) { tsdbDataFReaderClose(state->pDataFReader); @@ -2144,7 +2151,8 @@ static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) { if (key->ts > pItemBack->ts) { return false; } else if (key->ts >= pItemFront->ts && key->ts <= pItemBack->ts) { - if (key->version <= pItemFront->version || (key->ts == pItemBack->ts && key->version <= pItemBack->version)) { + // if (key->version <= pItemFront->version || (key->ts == pItemBack->ts && key->version <= pItemBack->version)) { + if (key->version <= pItemFront->version || key->version <= pItemBack->version) { return true; } else { return false; @@ -2959,7 +2967,7 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, do { TSDBROW *pRow = NULL; - nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, true, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray)); + nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, false, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray)); if (!pRow) { break; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 80967a906f..6d223e00c5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -190,9 +190,9 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid tsdbCacheDeleteLast(pTsdb->lruCache, pTbData->uid, eKey); } */ - if (eKey >= pTbData->maxKey && sKey <= pTbData->maxKey) { - tsdbCacheDel(pTsdb, suid, uid, sKey, eKey); - } + // if (eKey >= pTbData->maxKey && sKey <= pTbData->maxKey) { + tsdbCacheDel(pTsdb, suid, uid, sKey, eKey); + //} tsdbTrace("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64 " at version %" PRId64, From 40fa0e06201153504d06d1bc0ff4aef896d3cdc9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 25 May 2023 18:07:36 +0800 Subject: [PATCH 25/42] fix:memory leak --- source/client/src/clientSml.c | 3 ++- source/client/src/clientSmlJson.c | 1 + source/client/src/clientSmlLine.c | 2 ++ source/client/src/clientSmlTelnet.c | 1 + 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 5748c06e8b..85cba9cda6 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -236,7 +236,7 @@ static void smlDestroySTableMeta(void *para) { taosHashCleanup(meta->colHash); taosArrayDestroy(meta->tags); taosArrayDestroy(meta->cols); - taosMemoryFree(meta->tableMeta); + taosMemoryFreeClear(meta->tableMeta); taosMemoryFree(meta); } @@ -1045,6 +1045,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { } } + taosMemoryFreeClear(sTableData->tableMeta); sTableData->tableMeta = pTableMeta; uDebug("SML:0x%" PRIx64 "modify schema uid:%" PRIu64 ", sversion:%d, tversion:%d", info->id, pTableMeta->uid, pTableMeta->sversion, pTableMeta->tversion) tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index ad5b199c4e..0f59505f8c 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -696,6 +696,7 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo } sMeta = smlBuildSTableMeta(info->dataFormat); if(sMeta == NULL){ + taosMemoryFreeClear(pTableMeta); return TSDB_CODE_OUT_OF_MEMORY; } sMeta->tableMeta = pTableMeta; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 4bf4ae091f..40e014458e 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -169,6 +169,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin } sMeta = smlBuildSTableMeta(info->dataFormat); if(sMeta == NULL){ + taosMemoryFreeClear(pTableMeta); return TSDB_CODE_OUT_OF_MEMORY; } sMeta->tableMeta = pTableMeta; @@ -376,6 +377,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin } *tmp = smlBuildSTableMeta(info->dataFormat); if(*tmp == NULL){ + taosMemoryFreeClear(pTableMeta); return TSDB_CODE_OUT_OF_MEMORY; } (*tmp)->tableMeta = pTableMeta; diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index 6dd8842797..b3f45a3107 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -92,6 +92,7 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS } sMeta = smlBuildSTableMeta(info->dataFormat); if(sMeta == NULL){ + taosMemoryFreeClear(pTableMeta); return TSDB_CODE_OUT_OF_MEMORY; } sMeta->tableMeta = pTableMeta; From 469b2b573c1ac673cbe54307d3613eafa133319c Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 25 May 2023 21:49:28 +0800 Subject: [PATCH 26/42] fix: fix error during emptyTsRange.sim --- source/libs/scalar/src/filter.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index ae4712b9ba..8a97f5fae2 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3546,7 +3546,7 @@ bool fltSclLessPoint(SFltSclPoint *pt1, SFltSclPoint *pt2) { if (pt1->start && pt2->start) { return !pt1->excl && pt2->excl; } else if (pt1->start) { - return pt1->excl && !pt2->excl; + return !pt1->excl && !pt2->excl; } else if (pt2->start) { return pt1->excl || pt2->excl; } @@ -3971,7 +3971,7 @@ static int32_t fltSclGetDatumValueFromPoint(SFltSclPoint *point, SFltSclDatum *d getDataMin(d->type.type, &(d->i)); } - if ((point->val.kind == FLT_SCL_DATUM_KIND_INT64) || (point->val.kind = FLT_SCL_DATUM_KIND_UINT64)) { + if (IS_NUMERIC_TYPE(d->type.type)) { if (point->excl) { if (point->start) { ++d->i; @@ -3980,7 +3980,7 @@ static int32_t fltSclGetDatumValueFromPoint(SFltSclPoint *point, SFltSclDatum *d } } } else { - qError("not supported kind %d when get datum from point", point->val.kind); + qError("not supported type %d when get datum from point", d->type.type); } return TSDB_CODE_SUCCESS; } @@ -4001,14 +4001,17 @@ int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) { if (taosArrayGetSize(points) == 2) { SFltSclPoint *startPt = taosArrayGet(points, 0); SFltSclPoint *endPt = taosArrayGet(points, 1); - SFltSclDatum start; - SFltSclDatum end; + SFltSclDatum start; + SFltSclDatum end; fltSclGetDatumValueFromPoint(startPt, &start); fltSclGetDatumValueFromPoint(endPt, &end); win->skey = start.i; win->ekey = end.i; *isStrict = true; goto _return; + } else if (taosArrayGetSize(points) == 0) { + *win = TSWINDOW_DESC_INITIALIZER; + goto _return; } } *win = TSWINDOW_INITIALIZER; From cbbd09b85c0fbe7038cfbfd3402888b217638acf Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 25 May 2023 22:37:49 +0800 Subject: [PATCH 27/42] fix: for logical and condition, each element shall be collectable operator --- source/libs/scalar/src/filter.c | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 8a97f5fae2..d45abb1b87 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3971,7 +3971,7 @@ static int32_t fltSclGetDatumValueFromPoint(SFltSclPoint *point, SFltSclDatum *d getDataMin(d->type.type, &(d->i)); } - if (IS_NUMERIC_TYPE(d->type.type)) { + if (IS_INTEGER_TYPE(d->type.type) || IS_TIMESTAMP_TYPE(d->type.type)) { if (point->excl) { if (point->start) { ++d->i; @@ -4460,26 +4460,35 @@ int32_t fltSclProcessCNF(SArray *sclOpListCNF, SArray *colRangeList) { return TSDB_CODE_SUCCESS; } -static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) { +static bool fltSclIsCollectableNode(SNode *pNode) { if (nodeType(pNode) != QUERY_NODE_OPERATOR) { - return TSDB_CODE_SUCCESS; + return false; } SOperatorNode *pOper = (SOperatorNode *)pNode; if (pOper->pLeft == NULL || pOper->pRight == NULL) { - return TSDB_CODE_SUCCESS; + return false; } if (!(pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL || pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL || pOper->opType == OP_TYPE_NOT_EQUAL || pOper->opType == OP_TYPE_EQUAL)) { - return TSDB_CODE_SUCCESS; + return false; } if (!(nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_VALUE)) { + return false; + } + return true; +} + +static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) { + if (!fltSclIsCollectableNode(pNode)) { return TSDB_CODE_SUCCESS; } + SOperatorNode *pOper = (SOperatorNode *)pNode; + SValueNode *valNode = (SValueNode *)pOper->pRight; if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { SFltSclOperator sclOp = {.colNode = (SColumnNode *)nodesCloneNode(pOper->pLeft), @@ -4501,6 +4510,11 @@ static int32_t fltSclCollectOperatorsFromLogicCond(SNode *pNode, SArray *sclOpLi return TSDB_CODE_SUCCESS; } SNode *pExpr = NULL; + FOREACH(pExpr, pLogicCond->pParameterList) { + if (!fltSclIsCollectableNode(pExpr)) { + return TSDB_CODE_SUCCESS; + } + } FOREACH(pExpr, pLogicCond->pParameterList) { fltSclCollectOperatorFromNode(pExpr, sclOpList); } return TSDB_CODE_SUCCESS; } From 75eb724c37297c6010b013914b09ea0e5c31065c Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Thu, 25 May 2023 17:28:50 +0800 Subject: [PATCH 28/42] fix tsdb read error version --- source/dnode/vnode/src/tq/tq.c | 1 + source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 0a3173b3cb..25df80479d 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1083,6 +1083,7 @@ int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t sversion, char* msg, int32_t qDebug("s-task:%s set the start wal offset to be:%"PRId64, pTask->id.idStr, sversion); walReaderSeekVer(pTask->exec.pWalReader, sversion); + pTask->chkInfo.currentVer = sversion; if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { streamMetaReleaseTask(pTq->pStreamMeta, pTask); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b4b090249e..2fb86db8b8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1848,7 +1848,7 @@ static bool isCleanFileDataBlock(STsdbReader* pReader, SFileDataBlockInfo* pBloc SDataBlockToLoadInfo info = {0}; getBlockToLoadInfo(&info, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader, pReader); bool isCleanFileBlock = !(info.overlapWithNeighborBlock || info.hasDupTs || info.overlapWithKeyInBuf || - info.overlapWithDelInfo || info.overlapWithLastBlock); + info.overlapWithDelInfo || info.overlapWithLastBlock || info.partiallyRequired); return isCleanFileBlock; } From 208ba2ef4d507aec918d7001b399b9b68f5255ff Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 26 May 2023 10:45:37 +0800 Subject: [PATCH 29/42] fix:memory leak --- source/client/src/clientImpl.c | 1 + source/libs/parser/src/parTranslater.c | 11 ++++------- source/libs/parser/src/parUtil.c | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 82177302c7..5963e419e1 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1757,6 +1757,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int return TSDB_CODE_TSC_INTERNAL_ERROR; } + taosMemoryFreeClear(pResultInfo->convertJson); pResultInfo->convertJson = taosMemoryCalloc(1, dataLen); if (pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; char* p1 = pResultInfo->convertJson; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 864513f15d..39dab556f3 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -8008,6 +8008,9 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { isJson = true; code = buildJsonTagVal(pCxt, pTagSchema, pVal, pTagArray, ppTag); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode((SNode*)pVal); + } taosArrayPush(tagName, pTagSchema->name); } else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL && !pVal->isNull) { char* tmpVal = nodesGetValueFromNode(pVal); @@ -8328,13 +8331,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS SArray* pTagVals = taosArrayInit(1, sizeof(STagVal)); int32_t code = TSDB_CODE_SUCCESS; STag* pTag = NULL; - do { - code = parseJsontoTagData(pStmt->pVal->literal, pTagVals, &pTag, &pCxt->msgBuf); - if (TSDB_CODE_SUCCESS != code) { - break; - } - } while (0); - + code = parseJsontoTagData(pStmt->pVal->literal, pTagVals, &pTag, &pCxt->msgBuf); taosArrayDestroy(pTagVals); if (code != TSDB_CODE_SUCCESS) { return code; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 5597bd3df8..8fa92a9d2e 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -416,7 +416,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi end: taosHashCleanup(keyHash); if (retCode == TSDB_CODE_SUCCESS) { - tTagNew(pTagVals, 1, true, ppTag); + retCode = tTagNew(pTagVals, 1, true, ppTag); } for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { STagVal* p = (STagVal*)taosArrayGet(pTagVals, i); From 81c796953e5233d1a2603eab13b71e7f25d12a0b Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 26 May 2023 10:55:52 +0800 Subject: [PATCH 30/42] fix: address sanitizer error --- source/libs/executor/src/executorInt.c | 7 ++++++- source/libs/scalar/src/sclvector.c | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index d229f7d0ee..66ad3ca892 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -573,8 +573,13 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD if (colDataIsNull_var(pDst, j)) { colDataSetNull_var(pDst, numOfRows); } else { + // fix address sanitizer error. p1 may point to memory that will change during realloc of colDataSetVal, first copy it to p2 char* p1 = colDataGetVarData(pDst, j); - colDataSetVal(pDst, numOfRows, p1, false); + int32_t len = varDataTLen(p1); + char* p2 = taosMemoryMalloc(len); + varDataCopy(p2, p1); + colDataSetVal(pDst, numOfRows, p2, false); + taosMemoryFree(p2); } numOfRows += 1; j += 1; diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 6983f6313b..e2496e15c3 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1788,6 +1788,7 @@ void vectorIsNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, ++pOut->numOfQualified; } colDataSetInt8(pOut->columnData, i, &v); + colDataClearNull_f(pOut->columnData->nullbitmap, i); } pOut->numOfRows = pLeft->numOfRows; } @@ -1799,6 +1800,7 @@ void vectorNotNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut ++pOut->numOfQualified; } colDataSetInt8(pOut->columnData, i, &v); + colDataClearNull_f(pOut->columnData->nullbitmap, i); } pOut->numOfRows = pLeft->numOfRows; } From bb481c52b6c03c8b0c8f27877a7b93401e5c2eb3 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 26 May 2023 11:26:55 +0800 Subject: [PATCH 31/42] cache/skyline: fix skyline false condition --- source/dnode/vnode/src/tsdb/tsdbCache.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 830f44b411..8757661216 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -825,7 +825,7 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pCache, h); SLastCol lastCol = *pLastCol; - // reallocVarData(&lastCol.colVal); + reallocVarData(&lastCol.colVal); taosArrayPush(pLastArray, &lastCol); if (h) { @@ -2151,11 +2151,15 @@ static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) { if (key->ts > pItemBack->ts) { return false; } else if (key->ts >= pItemFront->ts && key->ts <= pItemBack->ts) { - // if (key->version <= pItemFront->version || (key->ts == pItemBack->ts && key->version <= pItemBack->version)) { - if (key->version <= pItemFront->version || key->version <= pItemBack->version) { + if (key->version <= pItemFront->version || (key->ts == pItemBack->ts && key->version <= pItemBack->version)) { + // if (key->version <= pItemFront->version || key->version <= pItemBack->version) { return true; } else { - return false; + if (*iSkyline > 1) { + --*iSkyline; + } else { + return false; + } } } else { if (*iSkyline > 1) { From fcf2b58aad3010b10dbb9f24465ce7d6faf61eff Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 26 May 2023 12:48:37 +0800 Subject: [PATCH 32/42] check offline when force --- include/util/taoserror.h | 1 + source/dnode/mnode/impl/src/mndDnode.c | 11 +++++++++-- source/util/src/terror.c | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 636a5e63a7..9e5229870e 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -409,6 +409,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MNODE_ALREADY_IS_VOTER TAOS_DEF_ERROR_CODE(0, 0x0413) // internal #define TSDB_CODE_MNODE_ONLY_TWO_MNODE TAOS_DEF_ERROR_CODE(0, 0x0414) // internal #define TSDB_CODE_MNODE_NO_NEED_RESTORE TAOS_DEF_ERROR_CODE(0, 0x0415) // internal +#define TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0416) // vnode // #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index cee3b3c61d..99694ae4b8 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -986,8 +986,15 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { } int32_t numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id); - if ((numOfVnodes > 0 || pMObj != NULL || pSObj != NULL || pQObj != NULL) && !force) { - if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) { + if (numOfVnodes > 0 || pMObj != NULL || pSObj != NULL || pQObj != NULL) { + bool isonline = mndIsDnodeOnline(pDnode, taosGetTimestampMs()); + if (isonline && force) { + terrno = TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE; + mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), + numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); + goto _OVER; + } + if (!isonline && !force) { terrno = TSDB_CODE_DNODE_OFFLINE; mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 31727f7535..e28e67f83a 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -325,6 +325,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_NOT_CATCH_UP, "Mnode didn't catch th TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_ALREADY_IS_VOTER, "Mnode already is a leader") TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_ONLY_TWO_MNODE, "Only two mnodes exist") TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_NO_NEED_RESTORE, "No need to restore on this dnode") +TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE, "Please use this command when the dnode is offline") // vnode TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode is closed or removed") From 70bcbafbb8b3ee3adde1bd1813857fec9c9732da Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 26 May 2023 13:36:56 +0800 Subject: [PATCH 33/42] fix: json value length is not same as binary/nchar length --- source/libs/executor/src/executorInt.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 66ad3ca892..ecd1874fa5 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -575,9 +575,14 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD } else { // fix address sanitizer error. p1 may point to memory that will change during realloc of colDataSetVal, first copy it to p2 char* p1 = colDataGetVarData(pDst, j); - int32_t len = varDataTLen(p1); + int32_t len = 0; + if (pDst->info.type == TSDB_DATA_TYPE_JSON) { + len = getJsonValueLen(p1); + } else { + len = varDataTLen(p1); + } char* p2 = taosMemoryMalloc(len); - varDataCopy(p2, p1); + memcpy(p2, p1, len); colDataSetVal(pDst, numOfRows, p2, false); taosMemoryFree(p2); } From 6e29d7879f9b9dec98f9d7faef4ac78aa0fafdeb Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 26 May 2023 14:13:01 +0800 Subject: [PATCH 34/42] cache/var-data: realloc var data --- source/dnode/vnode/src/tsdb/tsdbCache.c | 3 ++- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 24 ++++++++++----------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 8757661216..aea7c45bd6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -703,6 +703,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr *pTmpLastCol = *pLastCol; pLastCol = pTmpLastCol; + reallocVarData(&pLastCol->colVal); size_t charge = sizeof(*pLastCol); if (IS_VAR_DATA_TYPE(pLastCol->colVal.type)) { charge += pLastCol->colVal.value.nData; @@ -853,8 +854,8 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache SLastCol lastCol = *pLastCol; reallocVarData(&lastCol.colVal); - taosArraySet(pLastArray, idxKey->idx, &lastCol); + if (h) { taosLRUCacheRelease(pCache, h, false); } diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 0e0b7a2ffa..5df1ea0672 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -315,14 +315,14 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 tsdbCacheGetBatch(pr->pTsdb, pKeyInfo->uid, pRow, pr, ltype); // tsdbCacheGet(pr->pTsdb, pKeyInfo->uid, pRow, pr, ltype); if (TARRAY_SIZE(pRow) <= 0) { - // taosArrayClearEx(pRow, freeItem); - taosArrayClear(pRow); + taosArrayClearEx(pRow, freeItem); + // taosArrayClear(pRow); continue; } SLastCol* pColVal = taosArrayGet(pRow, 0); if (COL_VAL_IS_NONE(&pColVal->colVal)) { - // taosArrayClearEx(pRow, freeItem); - taosArrayClear(pRow); + taosArrayClearEx(pRow, freeItem); + // taosArrayClear(pRow); continue; } @@ -381,8 +381,8 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } } - // taosArrayClearEx(pRow, freeItem); - taosArrayClear(pRow); + taosArrayClearEx(pRow, freeItem); + // taosArrayClear(pRow); } if (hasRes) { @@ -394,20 +394,20 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype); if (TARRAY_SIZE(pRow) <= 0) { - // taosArrayClearEx(pRow, freeItem); - taosArrayClear(pRow); + taosArrayClearEx(pRow, freeItem); + // taosArrayClear(pRow); continue; } SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, 0); if (COL_VAL_IS_NONE(&pColVal->colVal)) { - // taosArrayClearEx(pRow, freeItem); - taosArrayClear(pRow); + taosArrayClearEx(pRow, freeItem); + // taosArrayClear(pRow); continue; } saveOneRow(pRow, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr); - // taosArrayClearEx(pRow, freeItem); - taosArrayClear(pRow); + taosArrayClearEx(pRow, freeItem); + // taosArrayClear(pRow); taosArrayPush(pTableUidList, &uid); From f61f37c936b53ea89d5a6ce3c1229fda8f3465ed Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 26 May 2023 14:20:01 +0800 Subject: [PATCH 35/42] fix(tmq): set the correct offset rsp when no poll occuring. --- source/client/test/clientTests.cpp | 15 ++++++++++++--- source/dnode/vnode/src/tq/tq.c | 9 ++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index b04727bfc0..ccc17289b0 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -1100,7 +1100,7 @@ TEST(clientCase, sub_tb_test) { // 创建订阅 topics 列表 tmq_list_t* topicList = tmq_list_new(); - tmq_list_append(topicList, "topic_t1"); + tmq_list_append(topicList, "t1"); // 启动订阅 tmq_subscribe(tmq, topicList); @@ -1118,7 +1118,7 @@ TEST(clientCase, sub_tb_test) { tmq_topic_assignment* pAssign = NULL; int32_t numOfAssign = 0; - int32_t code = tmq_get_topic_assignment(tmq, "topic_t1", &pAssign, &numOfAssign); + int32_t code = tmq_get_topic_assignment(tmq, "t1", &pAssign, &numOfAssign); if (code != 0) { printf("error occurs:%s\n", tmq_err2str(code)); tmq_consumer_close(tmq); @@ -1127,7 +1127,16 @@ TEST(clientCase, sub_tb_test) { return; } - tmq_offset_seek(tmq, "topic_t1", pAssign[0].vgId, 0); + tmq_offset_seek(tmq, "t1", pAssign[0].vgId, 4); + + code = tmq_get_topic_assignment(tmq, "t1", &pAssign, &numOfAssign); + if (code != 0) { + printf("error occurs:%s\n", tmq_err2str(code)); + tmq_consumer_close(tmq); + taos_close(pConn); + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); + return; + } while (1) { TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 14deff7502..c6077a3f8f 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -510,8 +510,6 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { int64_t sver = 0, ever = 0; walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); - int64_t currentVer = walReaderGetCurrentVer(pHandle->execHandle.pTqReader->pWalReader); - SMqDataRsp dataRsp = {0}; tqInitDataRsp(&dataRsp, &req); @@ -537,7 +535,12 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { dataRsp.rspOffset.type = TMQ_OFFSET__LOG; if (reqOffset.type == TMQ_OFFSET__LOG) { - dataRsp.rspOffset.version = currentVer; // return current consume offset value + int64_t currentVer = walReaderGetCurrentVer(pHandle->execHandle.pTqReader->pWalReader); + if (currentVer == -1) { // not start to read data from wal yet, return req offset directly + dataRsp.rspOffset.version = reqOffset.version; + } else { + dataRsp.rspOffset.version = currentVer; // return current consume offset value + } } else if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { dataRsp.rspOffset.version = sver; // not consume yet, set the earliest position } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { From b83bf9163ec854d2fb857666fd50693df16572fa Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 26 May 2023 14:46:32 +0800 Subject: [PATCH 36/42] fix: restore scalar mode computation and add filterScalarMode config variable and add test case --- include/common/tglobal.h | 1 + source/common/src/tglobal.c | 5 + source/libs/scalar/src/filter.c | 10 +- tests/parallel_test/cases.task | 2 + tests/script/tsim/parser/condition_scl.sim | 136 +++++++++++++++++++ tests/script/tsim/query/emptyTsRange_scl.sim | 21 +++ 6 files changed, 172 insertions(+), 3 deletions(-) create mode 100644 tests/script/tsim/parser/condition_scl.sim create mode 100644 tests/script/tsim/query/emptyTsRange_scl.sim diff --git a/include/common/tglobal.h b/include/common/tglobal.h index abee37d122..157e37f080 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -180,6 +180,7 @@ extern int32_t tsRpcRetryInterval; extern bool tsDisableStream; extern int64_t tsStreamBufferSize; extern int64_t tsCheckpointInterval; +extern bool tsFilterScalarMode; // #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 9ff297c5b4..c648f8551a 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -208,6 +208,7 @@ char tsUdfdLdLibPath[512] = ""; bool tsDisableStream = false; int64_t tsStreamBufferSize = 128 * 1024 * 1024; int64_t tsCheckpointInterval = 3 * 60 * 60 * 1000; +bool tsFilterScalarMode = false; #ifndef _STORAGE int32_t taosSetTfsCfg(SConfig *pCfg) { @@ -522,6 +523,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, 0) != 0) return -1; + if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, 0) != 0) return -1; + GRANT_CFG_ADD; return 0; } @@ -898,6 +901,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsStreamBufferSize = cfgGetItem(pCfg, "streamBufferSize")->i64; tsCheckpointInterval = cfgGetItem(pCfg, "checkpointInterval")->i64; + tsFilterScalarMode = cfgGetItem(pCfg, "filterScalarMode")->bval; + GRANT_CFG_GET; return 0; } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index d45abb1b87..9a488edd67 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -14,6 +14,7 @@ */ #include #include "os.h" +#include "tglobal.h" #include "thash.h" // #include "queryLog.h" #include "filter.h" @@ -4510,7 +4511,7 @@ static int32_t fltSclCollectOperatorsFromLogicCond(SNode *pNode, SArray *sclOpLi return TSDB_CODE_SUCCESS; } SNode *pExpr = NULL; - FOREACH(pExpr, pLogicCond->pParameterList) { + FOREACH(pExpr, pLogicCond->pParameterList) { if (!fltSclIsCollectableNode(pExpr)) { return TSDB_CODE_SUCCESS; } @@ -4614,8 +4615,11 @@ int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pInfo, uint32_t options) stat.info = info; FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat)); - - info->scalarMode = true; + if (tsFilterScalarMode) { + info->scalarMode = true; + } else { + info->scalarMode = stat.scalarMode; + } fltDebug("scalar mode: %d", info->scalarMode); if (!info->scalarMode) { diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ab4663e5b3..6037a14c8a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1274,6 +1274,7 @@ ,,y,script,./test.sh -f tsim/parser/columnValue_tinyint.sim ,,y,script,./test.sh -f tsim/parser/columnValue_unsign.sim ,,y,script,./test.sh -f tsim/parser/condition.sim +,,y,script,./test.sh -f tsim/parser/condition_scl.sim ,,y,script,./test.sh -f tsim/parser/constCol.sim ,,y,script,./test.sh -f tsim/parser/create_db.sim ,,y,script,./test.sh -f tsim/parser/create_mt.sim @@ -1353,6 +1354,7 @@ ,,y,script,./test.sh -f tsim/query/event.sim ,,y,script,./test.sh -f tsim/query/forceFill.sim ,,y,script,./test.sh -f tsim/query/emptyTsRange.sim +,,y,script,./test.sh -f tsim/query/emptyTsRange_scl.sim ,,y,script,./test.sh -f tsim/query/partitionby.sim ,,y,script,./test.sh -f tsim/query/tableCount.sim ,,y,script,./test.sh -f tsim/query/tag_scan.sim diff --git a/tests/script/tsim/parser/condition_scl.sim b/tests/script/tsim/parser/condition_scl.sim new file mode 100644 index 0000000000..f377988006 --- /dev/null +++ b/tests/script/tsim/parser/condition_scl.sim @@ -0,0 +1,136 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c filterScalarMode -v 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql drop database if exists cdb +sql create database if not exists cdb +sql use cdb +sql create table stb1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(10), t3 double) +sql create table tb1 using stb1 tags(1,'1',1.0) +sql create table tb2 using stb1 tags(2,'2',2.0) +sql create table tb3 using stb1 tags(3,'3',3.0) +sql create table tb4 using stb1 tags(4,'4',4.0) +sql create table tb5 using stb1 tags(5,'5',5.0) +sql create table tb6 using stb1 tags(6,'6',6.0) + +sql insert into tb1 values ('2021-05-05 18:19:00',1,1.0,1,1,1,1.0,true ,'1','1') +sql insert into tb1 values ('2021-05-05 18:19:01',2,2.0,2,2,2,2.0,true ,'2','2') +sql insert into tb1 values ('2021-05-05 18:19:02',3,3.0,3,3,3,3.0,false,'3','3') +sql insert into tb1 values ('2021-05-05 18:19:03',4,4.0,4,4,4,4.0,false,'4','4') +sql insert into tb1 values ('2021-05-05 18:19:04',11,11.0,11,11,11,11.0,true ,'11','11') +sql insert into tb1 values ('2021-05-05 18:19:05',12,12.0,12,12,12,12.0,true ,'12','12') +sql insert into tb1 values ('2021-05-05 18:19:06',13,13.0,13,13,13,13.0,false,'13','13') +sql insert into tb1 values ('2021-05-05 18:19:07',14,14.0,14,14,14,14.0,false,'14','14') +sql insert into tb2 values ('2021-05-05 18:19:08',21,21.0,21,21,21,21.0,true ,'21','21') +sql insert into tb2 values ('2021-05-05 18:19:09',22,22.0,22,22,22,22.0,true ,'22','22') +sql insert into tb2 values ('2021-05-05 18:19:10',23,23.0,23,23,23,23.0,false,'23','23') +sql insert into tb2 values ('2021-05-05 18:19:11',24,24.0,24,24,24,24.0,false,'24','24') +sql insert into tb3 values ('2021-05-05 18:19:12',31,31.0,31,31,31,31.0,true ,'31','31') +sql insert into tb3 values ('2021-05-05 18:19:13',32,32.0,32,32,32,32.0,true ,'32','32') +sql insert into tb3 values ('2021-05-05 18:19:14',33,33.0,33,33,33,33.0,false,'33','33') +sql insert into tb3 values ('2021-05-05 18:19:15',34,34.0,34,34,34,34.0,false,'34','34') +sql insert into tb4 values ('2021-05-05 18:19:16',41,41.0,41,41,41,41.0,true ,'41','41') +sql insert into tb4 values ('2021-05-05 18:19:17',42,42.0,42,42,42,42.0,true ,'42','42') +sql insert into tb4 values ('2021-05-05 18:19:18',43,43.0,43,43,43,43.0,false,'43','43') +sql insert into tb4 values ('2021-05-05 18:19:19',44,44.0,44,44,44,44.0,false,'44','44') +sql insert into tb5 values ('2021-05-05 18:19:20',51,51.0,51,51,51,51.0,true ,'51','51') +sql insert into tb5 values ('2021-05-05 18:19:21',52,52.0,52,52,52,52.0,true ,'52','52') +sql insert into tb5 values ('2021-05-05 18:19:22',53,53.0,53,53,53,53.0,false,'53','53') +sql insert into tb5 values ('2021-05-05 18:19:23',54,54.0,54,54,54,54.0,false,'54','54') +sql insert into tb6 values ('2021-05-05 18:19:24',61,61.0,61,61,61,61.0,true ,'61','61') +sql insert into tb6 values ('2021-05-05 18:19:25',62,62.0,62,62,62,62.0,true ,'62','62') +sql insert into tb6 values ('2021-05-05 18:19:26',63,63.0,63,63,63,63.0,false,'63','63') +sql insert into tb6 values ('2021-05-05 18:19:27',64,64.0,64,64,64,64.0,false,'64','64') +sql insert into tb6 values ('2021-05-05 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) + +sql create table stb2 (ts timestamp, u1 int unsigned, u2 bigint unsigned, u3 smallint unsigned, u4 tinyint unsigned, ts2 timestamp) TAGS(t1 int unsigned, t2 bigint unsigned, t3 timestamp, t4 int) +sql create table tb2_1 using stb2 tags(1,1,'2021-05-05 18:38:38',1) +sql create table tb2_2 using stb2 tags(2,2,'2021-05-05 18:58:58',2) + +sql insert into tb2_1 values ('2021-05-05 18:19:00',1,2,3,4,'2021-05-05 18:28:01') +sql insert into tb2_1 values ('2021-05-05 18:19:01',5,6,7,8,'2021-05-05 18:28:02') +sql insert into tb2_1 values ('2021-05-05 18:19:02',2,2,3,4,'2021-05-05 18:28:03') +sql insert into tb2_1 values ('2021-05-05 18:19:03',5,6,7,8,'2021-05-05 18:28:04') +sql insert into tb2_1 values ('2021-05-05 18:19:04',3,2,3,4,'2021-05-05 18:28:05') +sql insert into tb2_1 values ('2021-05-05 18:19:05',5,6,7,8,'2021-05-05 18:28:06') +sql insert into tb2_1 values ('2021-05-05 18:19:06',4,2,3,4,'2021-05-05 18:28:07') +sql insert into tb2_1 values ('2021-05-05 18:19:07',5,6,7,8,'2021-05-05 18:28:08') +sql insert into tb2_1 values ('2021-05-05 18:19:08',5,2,3,4,'2021-05-05 18:28:09') +sql insert into tb2_1 values ('2021-05-05 18:19:09',5,6,7,8,'2021-05-05 18:28:10') +sql insert into tb2_1 values ('2021-05-05 18:19:10',6,2,3,4,'2021-05-05 18:28:11') +sql insert into tb2_2 values ('2021-05-05 18:19:11',5,6,7,8,'2021-05-05 18:28:12') +sql insert into tb2_2 values ('2021-05-05 18:19:12',7,2,3,4,'2021-05-05 18:28:13') +sql insert into tb2_2 values ('2021-05-05 18:19:13',5,6,7,8,'2021-05-05 18:28:14') +sql insert into tb2_2 values ('2021-05-05 18:19:14',8,2,3,4,'2021-05-05 18:28:15') +sql insert into tb2_2 values ('2021-05-05 18:19:15',5,6,7,8,'2021-05-05 18:28:16') + +sql create table stb3 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(10), t3 double) +sql create table tb3_1 using stb3 tags(1,'1',1.0) +sql create table tb3_2 using stb3 tags(2,'2',2.0) + +sql insert into tb3_1 values ('2021-01-05 18:19:00',1,1.0,1,1,1,1.0,true ,'1','1') +sql insert into tb3_1 values ('2021-02-05 18:19:01',2,2.0,2,2,2,2.0,true ,'2','2') +sql insert into tb3_1 values ('2021-03-05 18:19:02',3,3.0,3,3,3,3.0,false,'3','3') +sql insert into tb3_1 values ('2021-04-05 18:19:03',4,4.0,4,4,4,4.0,false,'4','4') +sql insert into tb3_1 values ('2021-05-05 18:19:28',5,NULL,5,NULL,5,NULL,true,NULL,'5') +sql insert into tb3_1 values ('2021-06-05 18:19:28',NULL,6.0,NULL,6,NULL,6.0,NULL,'6',NULL) +sql insert into tb3_1 values ('2021-07-05 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) +sql insert into tb3_2 values ('2021-01-06 18:19:00',11,11.0,11,11,11,11.0,true ,'11','11') +sql insert into tb3_2 values ('2021-02-06 18:19:01',12,12.0,12,12,12,12.0,true ,'12','12') +sql insert into tb3_2 values ('2021-03-06 18:19:02',13,13.0,13,13,13,13.0,false,'13','13') +sql insert into tb3_2 values ('2021-04-06 18:19:03',14,14.0,14,14,14,14.0,false,'14','14') +sql insert into tb3_2 values ('2021-05-06 18:19:28',15,NULL,15,NULL,15,NULL,true,NULL,'15') +sql insert into tb3_2 values ('2021-06-06 18:19:28',NULL,16.0,NULL,16,NULL,16.0,NULL,'16',NULL) +sql insert into tb3_2 values ('2021-07-06 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) + +sql create table stb4 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9),c10 binary(16300)) TAGS(t1 int, t2 binary(10), t3 double) +sql create table tb4_0 using stb4 tags(0,'0',0.0) +sql create table tb4_1 using stb4 tags(1,'1',1.0) +sql create table tb4_2 using stb4 tags(2,'2',2.0) +sql create table tb4_3 using stb4 tags(3,'3',3.0) +sql create table tb4_4 using stb4 tags(4,'4',4.0) + +$i = 0 +$ts0 = 1625850000000 +$blockNum = 5 +$delta = 0 +$tbname0 = tb4_ +$a = 0 +$b = 200 +$c = 400 +while $i < $blockNum + $x = 0 + $rowNum = 1200 + while $x < $rowNum + $ts = $ts0 + $x + $a = $a + 1 + $b = $b + 1 + $c = $c + 1 + $d = $x / 10 + $tin = $rowNum + $binary = 'binary . $c + $binary = $binary . ' + $nchar = 'nchar . $c + $nchar = $nchar . ' + $tbname = 'tb4_ . $i + $tbname = $tbname . ' + sql insert into $tbname values ( $ts , $a , $b , $c , $d , $d , $c , true, $binary , $nchar , $binary ) + $x = $x + 1 + endw + + $i = $i + 1 + $ts0 = $ts0 + 259200000 +endw + +run tsim/parser/condition_query.sim + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sql connect + +run tsim/parser/condition_query.sim + diff --git a/tests/script/tsim/query/emptyTsRange_scl.sim b/tests/script/tsim/query/emptyTsRange_scl.sim new file mode 100644 index 0000000000..43734b047d --- /dev/null +++ b/tests/script/tsim/query/emptyTsRange_scl.sim @@ -0,0 +1,21 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c filterScalarMode -v 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql drop database if exists db1; +sql create database if not exists db1; +sql use db1; +sql create stable sta (ts timestamp, f1 double, f2 binary(200)) tags(t1 int); +sql create table tba1 using sta tags(1); +sql insert into tba1 values ('2022-04-26 15:15:01', 1.0, "a"); +sql insert into tba1 values ('2022-04-26 15:15:02', 2.0, "b"); +sql insert into tba1 values ('2022-04-26 15:15:04', 4.0, "b"); +sql insert into tba1 values ('2022-04-26 15:15:05', 5.0, "b"); +sql select last_row(*) from sta where ts >= 1678901803783 and ts <= 1678901803783 and _c0 <= 1678901803782 interval(10d,8d) fill(linear) order by _wstart desc; +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From b89f1652fcc16487de8e4fc5194c08316bc376dd Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 26 May 2023 14:52:14 +0800 Subject: [PATCH 37/42] fix: remove tbl count scan modification from the PR --- source/libs/planner/src/planOptimizer.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 4a803065ed..72e5081ab9 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2632,13 +2632,11 @@ static bool tbCntScanOptIsEligibleConds(STbCntScanOptInfo* pInfo, SNode* pCondit return false; } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pConditions)) { - return tbCntScanOptIsEligibleLogicCond(pInfo, (SLogicConditionNode*)pConditions) && - LIST_LENGTH(pInfo->pAgg->pGroupKeys) == 0; + return tbCntScanOptIsEligibleLogicCond(pInfo, (SLogicConditionNode*)pConditions); } if (QUERY_NODE_OPERATOR == nodeType(pConditions)) { - return tbCntScanOptIsEligibleOpCond((SOperatorNode*)pConditions) && - LIST_LENGTH(pInfo->pAgg->pGroupKeys) == 0; + return tbCntScanOptIsEligibleOpCond((SOperatorNode*)pConditions); } return false; From 5ebe75599e4a7f853726acd14661599c3f56d74e Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Fri, 26 May 2023 10:09:05 +0800 Subject: [PATCH 38/42] fix pause issue --- source/libs/stream/src/streamExec.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 637d661343..95b97e080a 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -327,7 +327,11 @@ int32_t streamExecForAll(SStreamTask* pTask) { while (1) { if (streamTaskShouldPause(&pTask->status)) { - return 0; + if (batchSize > 1) { + break; + } else { + return 0; + } } SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); From 34b9b322cdcbd17dcb83a4e9a4d343257d64945b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 26 May 2023 15:01:57 +0800 Subject: [PATCH 39/42] cache/var-data: realloc for cache reading --- source/dnode/vnode/src/tsdb/tsdbCache.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index aea7c45bd6..63629f8623 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -790,7 +790,9 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA code = -1; } - taosArraySet(pLastArray, idxKey->idx, pLastCol); + SLastCol lastCol = *pLastCol; + reallocVarData(&lastCol.colVal); + taosArraySet(pLastArray, idxKey->idx, &lastCol); taosArrayRemove(remainCols, j); taosMemoryFree(values_list[i]); From b38e2530191715ec0e3db9e2ff97347d5c7c349a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 26 May 2023 22:51:58 +0800 Subject: [PATCH 40/42] fix(stream): set the skip offset when resume stream task. --- include/libs/wal/wal.h | 3 +++ source/dnode/vnode/src/tq/tq.c | 27 ++++++++++++++++----------- source/dnode/vnode/src/tq/tqRestore.c | 10 ++++++++++ source/libs/wal/src/walRead.c | 8 ++++++++ 4 files changed, 37 insertions(+), 11 deletions(-) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 35a6838b2e..a081527144 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -149,6 +149,7 @@ struct SWalReader { TdFilePtr pIdxFile; int64_t curFileFirstVer; int64_t curVersion; + int64_t skipToVersion; // skip data and jump to destination version, usually used by stream resume ignoring untreated data int64_t capacity; TdThreadMutex mutex; SWalFilterCond cond; @@ -200,6 +201,8 @@ int32_t walReaderSeekVer(SWalReader *pRead, int64_t ver); int32_t walNextValidMsg(SWalReader *pRead); int64_t walReaderGetCurrentVer(const SWalReader *pReader); int64_t walReaderGetValidFirstVer(const SWalReader *pReader); +int64_t walReaderGetSkipToVersion(SWalReader *pReader); +void walReaderSetSkipToVersion(SWalReader *pReader, int64_t ver); void walReaderValidVersionRange(SWalReader *pReader, int64_t *sver, int64_t *ever); void walReaderVerifyOffset(SWalReader *pWalReader, STqOffsetVal* pOffset); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c6077a3f8f..633b15a1fb 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1303,19 +1303,22 @@ int32_t tqProcessTaskPauseReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg int32_t tqProcessTaskResumeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { SVResumeStreamTaskReq* pReq = (SVResumeStreamTaskReq*)msg; + + int32_t vgId = pTq->pStreamMeta->vgId; SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->taskId); if (pTask) { atomic_store_8(&pTask->status.taskStatus, pTask->status.keepTaskStatus); // no lock needs to secure the access of the version - if (pReq->igUntreated && pTask->taskLevel == TASK_LEVEL__SOURCE) { // discard all the data when the stream task is suspended. - pTask->chkInfo.currentVer = sversion; - walReaderSeekVer(pTask->exec.pWalReader, sversion); - tqDebug("vgId:%d s-task:%s resume to normal from the latest version:%" PRId64 ", vnode ver:%" PRId64 ", schedStatus:%d", pTq->pStreamMeta->vgId, - pTask->id.idStr, pTask->chkInfo.currentVer, sversion, pTask->status.schedStatus); + if (pReq->igUntreated && pTask->taskLevel == TASK_LEVEL__SOURCE) { + // discard all the data when the stream task is suspended. + walReaderSetSkipToVersion(pTask->exec.pWalReader, sversion); + tqDebug("vgId:%d s-task:%s resume to exec, prev paused version:%" PRId64 ", start from vnode ver:%" PRId64 + ", schedStatus:%d", + vgId, pTask->id.idStr, pTask->chkInfo.currentVer, sversion, pTask->status.schedStatus); } else { // from the previous paused version and go on - tqDebug("vgId:%d s-task:%s resume to normal from paused ver:%" PRId64 ", vnode ver:%" PRId64 ", schedStatus:%d", pTq->pStreamMeta->vgId, - pTask->id.idStr, pTask->chkInfo.currentVer, sversion, pTask->status.schedStatus); + tqDebug("vgId:%d s-task:%s resume to exec, from paused ver:%" PRId64 ", vnode ver:%" PRId64 ", schedStatus:%d", + vgId, pTask->id.idStr, pTask->chkInfo.currentVer, sversion, pTask->status.schedStatus); } if (pTask->taskLevel == TASK_LEVEL__SOURCE && taosQueueItemSize(pTask->inputQueue->queue) == 0) { @@ -1324,6 +1327,8 @@ int32_t tqProcessTaskResumeReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms streamSchedExec(pTask); } streamMetaReleaseTask(pTq->pStreamMeta, pTask); + } else { + tqError("vgId:%d failed to find the s-task:0x%x for resume stream task", vgId, pReq->taskId); } return 0; @@ -1432,7 +1437,7 @@ int32_t tqStartStreamTasks(STQ* pTq) { int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); if (numOfTasks == 0) { tqInfo("vgId:%d no stream tasks exist", vgId); - taosWUnLockLatch(&pTq->pStreamMeta->lock); + taosWUnLockLatch(&pMeta->lock); return 0; } @@ -1440,7 +1445,7 @@ int32_t tqStartStreamTasks(STQ* pTq) { if (pMeta->walScanCounter > 1) { tqDebug("vgId:%d wal read task has been launched, remain scan times:%d", vgId, pMeta->walScanCounter); - taosWUnLockLatch(&pTq->pStreamMeta->lock); + taosWUnLockLatch(&pMeta->lock); return 0; } @@ -1448,7 +1453,7 @@ int32_t tqStartStreamTasks(STQ* pTq) { if (pRunReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("vgId:%d failed to create msg to start wal scanning to launch stream tasks, code:%s", vgId, terrstr()); - taosWUnLockLatch(&pTq->pStreamMeta->lock); + taosWUnLockLatch(&pMeta->lock); return -1; } @@ -1459,7 +1464,7 @@ int32_t tqStartStreamTasks(STQ* pTq) { SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &msg); - taosWUnLockLatch(&pTq->pStreamMeta->lock); + taosWUnLockLatch(&pMeta->lock); return 0; } diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index 6e3036a66b..fe80f48691 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -87,6 +87,16 @@ static int32_t doSetOffsetForWalReader(SStreamTask *pTask, int32_t vgId) { } } + int64_t skipToVer = walReaderGetSkipToVersion(pTask->exec.pWalReader); + if (skipToVer != 0 && skipToVer > pTask->chkInfo.currentVer) { + int32_t code = walReaderSeekVer(pTask->exec.pWalReader, skipToVer); + if (code != TSDB_CODE_SUCCESS) { // no data in wal, quit + return code; + } + + tqDebug("vgId:%d s-task:%s wal reader jump to ver:%" PRId64, vgId, pTask->id.idStr, skipToVer); + } + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index c29d82bcf3..0eb3d6ef09 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -108,6 +108,14 @@ int32_t walNextValidMsg(SWalReader *pReader) { int64_t walReaderGetCurrentVer(const SWalReader *pReader) { return pReader->curVersion; } int64_t walReaderGetValidFirstVer(const SWalReader *pReader) { return walGetFirstVer(pReader->pWal); } +void walReaderSetSkipToVersion(SWalReader *pReader, int64_t ver) { atomic_store_64(&pReader->skipToVersion, ver); } + +// this function is NOT multi-thread safe, and no need to be. +int64_t walReaderGetSkipToVersion(SWalReader *pReader) { + int64_t newVersion = pReader->skipToVersion; + pReader->skipToVersion = 0; + return newVersion; +} void walReaderValidVersionRange(SWalReader *pReader, int64_t *sver, int64_t *ever) { *sver = walGetFirstVer(pReader->pWal); From c91029f92fcb369633a564efc0583a99c91cac3f Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 27 May 2023 19:28:47 +0800 Subject: [PATCH 41/42] docs: fix typos (#21503) --- docs/en/07-develop/09-udf.md | 28 ++++++++++++------------- docs/en/14-reference/12-config/index.md | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md index b0124b9c3f..5a98ff2789 100644 --- a/docs/en/07-develop/09-udf.md +++ b/docs/en/07-develop/09-udf.md @@ -10,7 +10,7 @@ User-defined functions can be scalar functions or aggregate functions. Scalar fu TDengine supports user-defined functions written in C or Python. This document describes the usage of user-defined functions. -## Implement a UDF in C +## Implement a UDF in C When you create a user-defined function, you must implement standard interface functions: - For scalar functions, implement the `scalarfn` interface function. @@ -111,13 +111,13 @@ Interface functions return a value that indicates whether the operation was succ For information about the parameters for interface functions, see Data Model #### Scalar Interface - `int32_t scalarfn(SUdfDataBlock* inputDataBlock, SUdfColumn *resultColumn)` - + `int32_t scalarfn(SUdfDataBlock* inputDataBlock, SUdfColumn *resultColumn)` + Replace `scalarfn` with the name of your function. This function performs scalar calculations on data blocks. You can configure a value through the parameters in the `resultColumn` structure. The parameters in the function are defined as follows: - inputDataBlock: The data block to input. - - resultColumn: The column to output. The column to output. + - resultColumn: The column to output. The column to output. #### Aggregate Interface @@ -197,7 +197,7 @@ The data structure is described as follows: - The SUdfDataBlock block includes the number of rows (numOfRows) and the number of columns (numCols). udfCols[i] (0 <= i <= numCols-1) indicates that each column is of type SUdfColumn. - SUdfColumn includes the definition of the data type of the column (colMeta) and the data in the column (colData). - The member definitions of SUdfColumnMeta are the same as the data type definitions in `taos.h`. -- The data in SUdfColumnData can become longer. varLenCol indicates variable-length data, and fixLenCol indicates fixed-length data. +- The data in SUdfColumnData can become longer. varLenCol indicates variable-length data, and fixLenCol indicates fixed-length data. - SUdfInterBuf defines the intermediate structure `buffer` and the number of results in the buffer `numOfResult`. Additional functions are defined in `taosudf.h` to make it easier to work with these structures. @@ -285,9 +285,9 @@ def init(): def destroy(): # destroy def process(input: datablock) -> tuple[output_type]: - # process input datablock, + # process input datablock, # datablock.data(row, col) is to access the python object in location(row,col) - # return tuple object consisted of object of type outputtype + # return tuple object consisted of object of type outputtype ``` ### Implement an Aggregate UDF in Python @@ -303,12 +303,12 @@ def start() -> bytes: #return serialize(init_state) def reduce(inputs: datablock, buf: bytes) -> bytes # deserialize buf to state - # reduce the inputs and state into new_state. - # use inputs.data(i,j) to access python ojbect of location(i,j) + # reduce the inputs and state into new_state. + # use inputs.data(i,j) to access python object of location(i,j) # serialize new_state into new_state_bytes - return new_state_bytes + return new_state_bytes def finish(buf: bytes) -> output_type: - #return obj of type outputtype + #return obj of type outputtype ``` ### Python UDF Interface Definition @@ -328,7 +328,7 @@ def finish(buf: bytes) -> output_type: ``` - first `start()` is called to return the initial result in type `bytes` -- then the input data are divided into multiple data blocks and for each block `input`, `reduce` is called with the data block `input` and the current result `buf` bytes and generates a new intermediate result buffer. +- then the input data are divided into multiple data blocks and for each block `input`, `reduce` is called with the data block `input` and the current result `buf` bytes and generates a new intermediate result buffer. - finally, the `finish` function is called on the intermediate result `buf` and outputs 0 or 1 data of type `output_type` @@ -337,7 +337,7 @@ def finish(buf: bytes) -> output_type: def init() def destroy() ``` -Implement `init` for initialization and `destroy` for termination. +Implement `init` for initialization and `destroy` for termination. ### Data Mapping between TDengine SQL and Python UDF @@ -360,7 +360,7 @@ sudo pip install taospyudf ldconfig ``` 2. If PYTHONPATH is needed to find Python packages when the Python UDF executes, include the PYTHONPATH contents into the udfdLdLibPath variable of the taos.cfg configuration file - + ### Python UDF Sample Code #### Scalar Function [pybitand](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pybitand.py) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 3ce63fb6cc..52ded6208a 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -111,7 +111,7 @@ The parameters described in this document by the effect that they have on the sy | Attribute | Description | | ------------- | ---------------------------------------------- | | Applicable | Client/Server | -| Meaning | The maximum waiting time to get avaliable conn | +| Meaning | The maximum waiting time to get available conn | | Value Range | 10-50000000(ms) | | Default Value | 500000 | From 884e7d0db669c60458af1287e0b4d003ab3fd9d5 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 28 May 2023 00:16:16 +0800 Subject: [PATCH 42/42] docs: fix udf zh typos (#21504) * docs: fix typos * docs: fix udf typos --- docs/zh/07-develop/09-udf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index 92f5d2a857..e43275d5e0 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -303,7 +303,7 @@ def start() -> bytes: def reduce(inputs: datablock, buf: bytes) -> bytes # deserialize buf to state # reduce the inputs and state into new_state. - # use inputs.data(i,j) to access python ojbect of location(i,j) + # use inputs.data(i,j) to access python object of location(i,j) # serialize new_state into new_state_bytes return new_state_bytes def finish(buf: bytes) -> output_type: