From bdee8ab3c11b360c3d4f57d52e297acc906f0f36 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 12 May 2023 19:00:26 +0800 Subject: [PATCH 01/97] enhance: save work and sync home/office --- source/libs/planner/src/planOptimizer.c | 6 +- source/libs/scalar/src/filter.c | 156 +++++++++++++++++++++++- 2 files changed, 159 insertions(+), 3 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 4f8b57de5f..a6d4898bc8 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2630,11 +2630,13 @@ static bool tbCntScanOptIsEligibleConds(STbCntScanOptInfo* pInfo, SNode* pCondit } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pConditions)) { - return tbCntScanOptIsEligibleLogicCond(pInfo, (SLogicConditionNode*)pConditions); + return tbCntScanOptIsEligibleLogicCond(pInfo, (SLogicConditionNode*)pConditions) && + LIST_LENGTH(pInfo->pAgg->pGroupKeys) == 0; } if (QUERY_NODE_OPERATOR == nodeType(pConditions)) { - return tbCntScanOptIsEligibleOpCond((SOperatorNode*)pConditions); + return tbCntScanOptIsEligibleOpCond((SOperatorNode*)pConditions) && + LIST_LENGTH(pInfo->pAgg->pGroupKeys) == 0; } return false; diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 02a21b66ed..77534db411 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3942,8 +3942,162 @@ _return: FLT_RET(code); } +// compare ranges, null < min < val < max. null=null, min=min, max=max +typedef enum { + FLT_SCL_DATUM_KIND_NULL, + FLT_SCL_DATUM_KIND_MIN, + FLT_SCL_DATUM_KIND_INT64, + FLT_SCL_DATUM_KIND_UINT64, + FLT_SCL_DATUM_KIND_FLOAT64, + FLT_SCL_DATUM_KIND_VARCHAR, + FLT_SCL_DATUM_KIND_NCHAR, + FLT_SCL_DATUM_KIND_MAX, +} SFltSclDatumKind; + +typedef struct { + SFltSclDatumKind kind; + union { + int64_t val; // may be int64, uint64 and double + struct { + uint32_t nData; + uint8_t *pData; + }; // maybe varchar, nchar + } datum; +} SFltSclDatum; + +typedef struct { + SFltSclDatum val; + bool excl; + bool start; +} SFltSclPoint; + +typedef struct { + SFltSclDatum low; + SFltSclDatum high; + bool lowExcl; + bool highExcl; +} SFltSclRange; + +int32_t fltSclCompareDatum(SFltSclDatum* val1, SFltSclDatum* val2) { +} + +bool fltSclLessPoint(SFltSclPoint* pt1, SFltSclPoint* pt2) { + // first value compare + int32_t cmp = fltSclCompareDatum(&pt1->val, &pt2->val); + if (cmp != 0) { + return cmp < 0 ; + } + + if (pt1->start && pt2->start) { + return !pt1->excl && pt2->excl; + } else if (pt1->start) { + return pt1->excl && !pt2->excl; + } else if (pt2->start) { + return pt1->excl || pt2->excl; + } + return pt1->excl && !pt2->excl; +} + +int32_t fltSclMergeSort(SArray* pts1, SArray* pts2, SArray* result) { + size_t len1 = taosArrayGetSize(pts1); + size_t len2 = taosArrayGetSize(pts2); + size_t i = 0; + size_t j = 0; + while (i < len1 && j < len2) { + SFltSclPoint* pt1 = taosArrayGet(pts1, i); + SFltSclPoint* pt2 = taosArrayGet(pts2, j); + bool less = fltSclLessPoint(pt1, pt2); + if (less) { + taosArrayPush(result, pt1); + ++i; + } else { + taosArrayPush(result, pt2); + ++j; + } + } + if (i < len1) { + for (; i < len1; ++i) { + SFltSclPoint* pt1 = taosArrayGet(pts1, i); + taosArrayPush(result, pt1); + } + } + if (j < len2) { + for (; j < len2; ++j) { + SFltSclPoint *pt2 = taosArrayGet(pts2, j); + taosArrayPush(result, pt2); + } + } + return 0; +} + +// pts1 and pts2 must be ordered and de-duplicated and each range can not be a range of another range +int32_t fltSclMerge(SArray* pts1, SArray* pts2, bool isUnion, SArray* merged) { + size_t len1 = taosArrayGetSize(pts1); + size_t len2 = taosArrayGetSize(pts2); + //first merge sort pts1 and pts2 + SArray* all = taosArrayInit(len1 + len2, sizeof(SFltSclPoint)); + fltSclMergeSort(pts1, pts2, all); + int32_t countRequired = (isUnion) ? 1 : 2; + int32_t count = 0; + for (int32_t i = 0; i < taosArrayGetSize(all); ++i) { + SFltSclPoint* pt = taosArrayGet(pts1, i); + if (pt->start) { + ++count; + if (count == countRequired) { + taosArrayPush(merged, pt); + } + } else { + if (count == countRequired) { + taosArrayPush(merged, pt); + } + --count; + } + } + taosArrayDestroy(all); + return 0; +} + + +int32_t fltSclIntersect(SArray* pts1, SArray* pts2, SArray* merged) { + return fltSclMerge(pts1, pts2, false, merged); +} + +int32_t fltSclUnion(SArray* pts1, SArray* pts2, SArray* merged) { + return fltSclMerge(pts1, pts2, true, merged); +} + + +typedef struct { + SColumnNode* colNode; + SValueNode* valNode; + EOperatorType type; +} SFltSclScalarFunc; + +typedef struct { + SColumnNode* colNode; + +} SFltSclConstColumn; + +typedef struct { + SValueNode* value; +} SFltSclConstant; + +typedef struct { + bool useRange; + SHashObj* colRanges; + +} SFltSclOptCtx; + +static EDealRes fltSclMayBeOptimed(SNode* pNode, void* pCtx) { + SFltSclOptCtx* ctx = (SFltSclOptCtx*)pCtx; + +} + + int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) { - // TODO + SFltSclOptCtx ctx; + nodesWalkExprPostOrder(*pNode, fltSclMayBeOptimed, &ctx); + return TSDB_CODE_SUCCESS; } From 51fa363fd50e148ec730f4b1814e1957caa41a4b Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 15 May 2023 11:34:05 +0800 Subject: [PATCH 02/97] feature: save work --- source/libs/scalar/src/filter.c | 57 +++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 77534db411..8c3e97ccbc 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3957,12 +3957,13 @@ typedef enum { typedef struct { SFltSclDatumKind kind; union { - int64_t val; // may be int64, uint64 and double + int64_t val; // for int64, uint64 and double and bool (1 true, 0 false) struct { - uint32_t nData; + uint32_t nData; // TODO maybe remove this and make pData len prefixed? uint8_t *pData; - }; // maybe varchar, nchar + }; // for varchar, nchar } datum; + SDataType type; // TODO: original data type, may not be used? } SFltSclDatum; typedef struct { @@ -3979,6 +3980,42 @@ typedef struct { } SFltSclRange; int32_t fltSclCompareDatum(SFltSclDatum* val1, SFltSclDatum* val2) { + switch (val1->kind) { + case FLT_SCL_DATUM_KIND_NULL: { + if (val2->kind == FLT_SCL_DATUM_KIND_NULL) return 0 ; else return -1; + break; + } + case FLT_SCL_DATUM_KIND_MIN: { + if (val2->kind == FLT_SCL_DATUM_KIND_NULL) return 1; + else { + if (val2->kind == FLT_SCL_DATUM_KIND_MIN) return 0; else return -1; + } + break; + } + case FLT_SCL_DATUM_KIND_MAX: { + if (val2->kind == FLT_SCL_DATUM_KIND_MAX) return 0; else return 1; + } + case FLT_SCL_DATUM_KIND_UINT64: { + return compareUint64Val(&val1->datum.val, &val1->datum.val); + } + case FLT_SCL_DATUM_KIND_INT64: { + return compareInt64Val(&val1->datum.val, &val2->datum.val); + } + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareFloatDouble(&val1->datum.val, &val2->datum.val); + } + case FLT_SCL_DATUM_KIND_NCHAR: { + return compareLenPrefixedWStr(&val1->datum.pData, &val2->datum.pData); //TODO + } + case FLT_SCL_DATUM_KIND_VARCHAR: { + return compareLenPrefixedStr(&val1->datum.pData, &val2->datum.pData); //TODO + } + default: + fltError("not supported kind. just return 0"); + return 0; + break; + } + return 0; } bool fltSclLessPoint(SFltSclPoint* pt1, SFltSclPoint* pt2) { @@ -4071,26 +4108,32 @@ typedef struct { SColumnNode* colNode; SValueNode* valNode; EOperatorType type; -} SFltSclScalarFunc; +} SFltSclScalarOp; +// simple define it, TODO: implement later typedef struct { SColumnNode* colNode; - } SFltSclConstColumn; +// simple define it, TODO: implement later typedef struct { SValueNode* value; } SFltSclConstant; + +int32_t fltSclBuildRangeFromScalarFunc(SFltSclScalarOp* func, SFltSclPoint* points) { + +} + + typedef struct { bool useRange; SHashObj* colRanges; - } SFltSclOptCtx; static EDealRes fltSclMayBeOptimed(SNode* pNode, void* pCtx) { SFltSclOptCtx* ctx = (SFltSclOptCtx*)pCtx; - + return DEAL_RES_CONTINUE; } From e16f4c7c440ae02905e18b397eb985aa3b91f668 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 17 May 2023 16:57:09 +0800 Subject: [PATCH 03/97] fix: refactor and save work --- source/libs/scalar/src/filter.c | 220 ++++++++++++++++++-------------- 1 file changed, 124 insertions(+), 96 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 8c3e97ccbc..85974bcd3a 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -22,6 +22,7 @@ #include "sclInt.h" #include "tcompare.h" #include "tdatablock.h" +#include "tsimplehash.h" #include "ttime.h" bool filterRangeCompGi(const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { @@ -260,7 +261,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { comparFn = 19; } else if (optr == OP_TYPE_NMATCH) { comparFn = 20; - } else if (optr == OP_TYPE_LIKE) { /* wildcard query using like operator */ + } else if (optr == OP_TYPE_LIKE) { /* wildcard query using like operator */ comparFn = 7; } else if (optr == OP_TYPE_NOT_LIKE) { /* wildcard query using like operator */ comparFn = 26; @@ -1633,7 +1634,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) SDataType *dType = &var->node.resType; qDebug("VAL%d => [type:%d][val:%" PRIx64 "]", i, dType->type, var->datum.i); // TODO } else if (field->data) { - qDebug("VAL%d => [type:NIL][val:NIL]", i); // TODO + qDebug("VAL%d => [type:NIL][val:NIL]", i); // TODO } } @@ -3957,59 +3958,96 @@ typedef enum { typedef struct { SFltSclDatumKind kind; union { - int64_t val; // for int64, uint64 and double and bool (1 true, 0 false) - struct { - uint32_t nData; // TODO maybe remove this and make pData len prefixed? - uint8_t *pData; - }; // for varchar, nchar + int64_t val; // for int64, uint64 and double and bool (1 true, 0 false) + uint8_t *pData; // for varchar, nchar } datum; - SDataType type; // TODO: original data type, may not be used? + SDataType type; // TODO: original data type, may not be used? } SFltSclDatum; typedef struct { SFltSclDatum val; - bool excl; - bool start; + bool excl; + bool start; } SFltSclPoint; typedef struct { SFltSclDatum low; SFltSclDatum high; - bool lowExcl; - bool highExcl; + bool lowExcl; + bool highExcl; } SFltSclRange; -int32_t fltSclCompareDatum(SFltSclDatum* val1, SFltSclDatum* val2) { +int32_t fltSclCompareWithFloat64(SFltSclDatum *val1, SFltSclDatum *val2) { + // val2->kind == int64 switch (val1->kind) { - case FLT_SCL_DATUM_KIND_NULL: { - if (val2->kind == FLT_SCL_DATUM_KIND_NULL) return 0 ; else return -1; - break; + case FLT_SCL_DATUM_KIND_UINT64: + return compareUint64Double(&val1->datum.val, &val2->datum.val); + case FLT_SCL_DATUM_KIND_INT64: + return compareInt64Double(&val1->datum.val, &val2->datum.val); + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareDoubleVal(&val1->datum.val, &val2->datum.val); } - case FLT_SCL_DATUM_KIND_MIN: { - if (val2->kind == FLT_SCL_DATUM_KIND_NULL) return 1; - else { - if (val2->kind == FLT_SCL_DATUM_KIND_MIN) return 0; else return -1; - } - break; + //TODO: varchar, nchar + default: + qError("not support comparsion. %d, %d", val1->kind, val2->kind); + return (val1->kind - val2->kind); + } +} + +int32_t fltSclCompareWithInt64(SFltSclDatum *val1, SFltSclDatum *val2) { + // val2->kind == int64 + switch (val1->kind) { + case FLT_SCL_DATUM_KIND_UINT64: + return compareUint64Int64(&val1->datum.val, &val2->datum.val); + case FLT_SCL_DATUM_KIND_INT64: + return compareInt64Val(&val1->datum.val, &val2->datum.val); + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareDoubleInt64(&val1->datum.val, &val2->datum.val); } - case FLT_SCL_DATUM_KIND_MAX: { - if (val2->kind == FLT_SCL_DATUM_KIND_MAX) return 0; else return 1; + //TODO: varchar, nchar + default: + qError("not support comparsion. %d, %d", val1->kind, val2->kind); + return (val1->kind - val2->kind); + } +} + +int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { + // val2 kind == uint64 + switch (val1->kind) { + case FLT_SCL_DATUM_KIND_UINT64: + return compareUint64Val(&val1->datum.val, &val2->datum.val); + case FLT_SCL_DATUM_KIND_INT64: + return compareInt64Uint64(&val1->datum.val, &val2->datum.val); + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareDoubleUint64(&val1->datum.val, &val2->datum.val); } + //TODO: varchar, nchar + default: + qError("not support comparsion. %d, %d", val1->kind, val2->kind); + return (val1->kind - val2->kind); + } +} + +int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { + if (val2->kind == FLT_SCL_DATUM_KIND_NULL || val2->kind == FLT_SCL_DATUM_KIND_MIN || + val2->kind == FLT_SCL_DATUM_KIND_MAX) { + if (val1->kind == FLT_SCL_DATUM_KIND_NULL || val1->kind == FLT_SCL_DATUM_KIND_MIN || + val1->kind == FLT_SCL_DATUM_KIND_MAX) { + return (val1->kind < val2->kind) ? -1 : ((val1->kind > val2->kind) ? 1 : 0); + } + } + + switch (val2->kind) { case FLT_SCL_DATUM_KIND_UINT64: { - return compareUint64Val(&val1->datum.val, &val1->datum.val); + return fltSclCompareWithUInt64(val1, val2); } case FLT_SCL_DATUM_KIND_INT64: { - return compareInt64Val(&val1->datum.val, &val2->datum.val); + return fltSclCompareWithInt64(val1, val2); } case FLT_SCL_DATUM_KIND_FLOAT64: { - return compareFloatDouble(&val1->datum.val, &val2->datum.val); - } - case FLT_SCL_DATUM_KIND_NCHAR: { - return compareLenPrefixedWStr(&val1->datum.pData, &val2->datum.pData); //TODO - } - case FLT_SCL_DATUM_KIND_VARCHAR: { - return compareLenPrefixedStr(&val1->datum.pData, &val2->datum.pData); //TODO + return fltSclCompareWithFloat64(val1, val2); } + //TODO: varchar/nchar default: fltError("not supported kind. just return 0"); return 0; @@ -4018,125 +4056,115 @@ int32_t fltSclCompareDatum(SFltSclDatum* val1, SFltSclDatum* val2) { return 0; } -bool fltSclLessPoint(SFltSclPoint* pt1, SFltSclPoint* pt2) { +bool fltSclLessPoint(SFltSclPoint *pt1, SFltSclPoint *pt2) { // first value compare int32_t cmp = fltSclCompareDatum(&pt1->val, &pt2->val); if (cmp != 0) { - return cmp < 0 ; + return cmp < 0; } if (pt1->start && pt2->start) { - return !pt1->excl && pt2->excl; - } else if (pt1->start) { - return pt1->excl && !pt2->excl; - } else if (pt2->start) { - return pt1->excl || pt2->excl; - } + return !pt1->excl && pt2->excl; + } else if (pt1->start) { + return pt1->excl && !pt2->excl; + } else if (pt2->start) { + return pt1->excl || pt2->excl; + } return pt1->excl && !pt2->excl; } -int32_t fltSclMergeSort(SArray* pts1, SArray* pts2, SArray* result) { +int32_t fltSclMergeSort(SArray *pts1, SArray *pts2, SArray *result) { size_t len1 = taosArrayGetSize(pts1); size_t len2 = taosArrayGetSize(pts2); size_t i = 0; size_t j = 0; while (i < len1 && j < len2) { - SFltSclPoint* pt1 = taosArrayGet(pts1, i); - SFltSclPoint* pt2 = taosArrayGet(pts2, j); - bool less = fltSclLessPoint(pt1, pt2); - if (less) { + SFltSclPoint *pt1 = taosArrayGet(pts1, i); + SFltSclPoint *pt2 = taosArrayGet(pts2, j); + bool less = fltSclLessPoint(pt1, pt2); + if (less) { taosArrayPush(result, pt1); ++i; - } else { + } else { taosArrayPush(result, pt2); ++j; - } + } } if (i < len1) { - for (; i < len1; ++i) { - SFltSclPoint* pt1 = taosArrayGet(pts1, i); + for (; i < len1; ++i) { + SFltSclPoint *pt1 = taosArrayGet(pts1, i); taosArrayPush(result, pt1); - } + } } if (j < len2) { - for (; j < len2; ++j) { + for (; j < len2; ++j) { SFltSclPoint *pt2 = taosArrayGet(pts2, j); taosArrayPush(result, pt2); - } + } } return 0; } // pts1 and pts2 must be ordered and de-duplicated and each range can not be a range of another range -int32_t fltSclMerge(SArray* pts1, SArray* pts2, bool isUnion, SArray* merged) { +int32_t fltSclMerge(SArray *pts1, SArray *pts2, bool isUnion, SArray *merged) { size_t len1 = taosArrayGetSize(pts1); size_t len2 = taosArrayGetSize(pts2); - //first merge sort pts1 and pts2 - SArray* all = taosArrayInit(len1 + len2, sizeof(SFltSclPoint)); + // first merge sort pts1 and pts2 + SArray *all = taosArrayInit(len1 + len2, sizeof(SFltSclPoint)); fltSclMergeSort(pts1, pts2, all); int32_t countRequired = (isUnion) ? 1 : 2; int32_t count = 0; for (int32_t i = 0; i < taosArrayGetSize(all); ++i) { - SFltSclPoint* pt = taosArrayGet(pts1, i); - if (pt->start) { - ++count; - if (count == countRequired) { - taosArrayPush(merged, pt); - } - } else { - if (count == countRequired) { - taosArrayPush(merged, pt); - } - --count; + SFltSclPoint *pt = taosArrayGet(pts1, i); + if (pt->start) { + ++count; + if (count == countRequired) { + taosArrayPush(merged, pt); } + } else { + if (count == countRequired) { + taosArrayPush(merged, pt); + } + --count; + } } taosArrayDestroy(all); return 0; } +int32_t fltSclIntersect(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, false, merged); } -int32_t fltSclIntersect(SArray* pts1, SArray* pts2, SArray* merged) { - return fltSclMerge(pts1, pts2, false, merged); -} - -int32_t fltSclUnion(SArray* pts1, SArray* pts2, SArray* merged) { - return fltSclMerge(pts1, pts2, true, merged); -} - +int32_t fltSclUnion(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, true, merged); } typedef struct { - SColumnNode* colNode; - SValueNode* valNode; + SColumnNode *colNode; + SValueNode *valNode; EOperatorType type; -} SFltSclScalarOp; +} SFltSclOperator; -// simple define it, TODO: implement later -typedef struct { - SColumnNode* colNode; -} SFltSclConstColumn; - -// simple define it, TODO: implement later -typedef struct { - SValueNode* value; -} SFltSclConstant; - - -int32_t fltSclBuildRangeFromScalarFunc(SFltSclScalarOp* func, SFltSclPoint* points) { +//TODO: column, constant, operator, and/or/not +int32_t fltSclBuildRangeFromOperator(SFltSclOperator *sclOper, SSHashObj* obj) { + switch (sclOper->type) { + case OP_TYPE_GREATER_THAN: { + } + break; + default: + break; + } + return TSDB_CODE_SUCCESS; } - typedef struct { - bool useRange; - SHashObj* colRanges; + bool useRange; + SHashObj *colRanges; } SFltSclOptCtx; -static EDealRes fltSclMayBeOptimed(SNode* pNode, void* pCtx) { - SFltSclOptCtx* ctx = (SFltSclOptCtx*)pCtx; +static EDealRes fltSclMayBeOptimed(SNode *pNode, void *pCtx) { + SFltSclOptCtx *ctx = (SFltSclOptCtx *)pCtx; return DEAL_RES_CONTINUE; } - int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) { SFltSclOptCtx ctx; nodesWalkExprPostOrder(*pNode, fltSclMayBeOptimed, &ctx); From 09837b7639190340f0d9a339a6da7db163b56ddd Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 18 May 2023 19:00:10 +0800 Subject: [PATCH 04/97] fix: save work --- source/libs/scalar/src/filter.c | 124 ++++++++++++++++++++++++++++---- 1 file changed, 109 insertions(+), 15 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 85974bcd3a..01bd53008e 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3960,7 +3960,7 @@ typedef struct { union { int64_t val; // for int64, uint64 and double and bool (1 true, 0 false) uint8_t *pData; // for varchar, nchar - } datum; + }; SDataType type; // TODO: original data type, may not be used? } SFltSclDatum; @@ -3981,11 +3981,11 @@ int32_t fltSclCompareWithFloat64(SFltSclDatum *val1, SFltSclDatum *val2) { // val2->kind == int64 switch (val1->kind) { case FLT_SCL_DATUM_KIND_UINT64: - return compareUint64Double(&val1->datum.val, &val2->datum.val); + return compareUint64Double(&val1->val, &val2->val); case FLT_SCL_DATUM_KIND_INT64: - return compareInt64Double(&val1->datum.val, &val2->datum.val); + return compareInt64Double(&val1->val, &val2->val); case FLT_SCL_DATUM_KIND_FLOAT64: { - return compareDoubleVal(&val1->datum.val, &val2->datum.val); + return compareDoubleVal(&val1->val, &val2->val); } //TODO: varchar, nchar default: @@ -3998,11 +3998,11 @@ int32_t fltSclCompareWithInt64(SFltSclDatum *val1, SFltSclDatum *val2) { // val2->kind == int64 switch (val1->kind) { case FLT_SCL_DATUM_KIND_UINT64: - return compareUint64Int64(&val1->datum.val, &val2->datum.val); + return compareUint64Int64(&val1->val, &val2->val); case FLT_SCL_DATUM_KIND_INT64: - return compareInt64Val(&val1->datum.val, &val2->datum.val); + return compareInt64Val(&val1->val, &val2->val); case FLT_SCL_DATUM_KIND_FLOAT64: { - return compareDoubleInt64(&val1->datum.val, &val2->datum.val); + return compareDoubleInt64(&val1->val, &val2->val); } //TODO: varchar, nchar default: @@ -4015,11 +4015,11 @@ int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { // val2 kind == uint64 switch (val1->kind) { case FLT_SCL_DATUM_KIND_UINT64: - return compareUint64Val(&val1->datum.val, &val2->datum.val); + return compareUint64Val(&val1->val, &val2->val); case FLT_SCL_DATUM_KIND_INT64: - return compareInt64Uint64(&val1->datum.val, &val2->datum.val); + return compareInt64Uint64(&val1->val, &val2->val); case FLT_SCL_DATUM_KIND_FLOAT64: { - return compareDoubleUint64(&val1->datum.val, &val2->datum.val); + return compareDoubleUint64(&val1->val, &val2->val); } //TODO: varchar, nchar default: @@ -4142,15 +4142,109 @@ typedef struct { EOperatorType type; } SFltSclOperator; -//TODO: column, constant, operator, and/or/not +//TODO: column, constant -int32_t fltSclBuildRangeFromOperator(SFltSclOperator *sclOper, SSHashObj* obj) { - switch (sclOper->type) { - case OP_TYPE_GREATER_THAN: { +typedef struct { + SColumnNode* colNode; + SArray* points; +} SFltSclColumnRange; + +SFltSclColumnRange* fltSclGetColumnRange(SColumnNode* colNode, SArray* colRangeList) { + for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) { + SFltSclColumnRange* colRange = taosArrayGet(colRangeList, i); + if (nodesEqualNode((SNode*)colRange->colNode, (SNode*)colNode)) { + return colRange; } + } + SFltSclColumnRange newColRange = {.colNode = colNode, .points = taosArrayInit(4, sizeof(SFltSclPoint))}; + taosArrayPush(colRangeList, &newColRange); + return taosArrayGetLast(colRangeList); +} + +int32_t fltSclBuildDatumFromValueNode(SFltSclDatum* datum, SValueNode* valNode) { + datum->type = valNode->node.resType; + + if (valNode->isNull) { + datum->kind = FLT_SCL_DATUM_KIND_NULL; + } else { + switch (valNode->node.resType.type) { + case TSDB_DATA_TYPE_NULL: { + datum->kind = FLT_SCL_DATUM_KIND_NULL; + break; + } + case TSDB_DATA_TYPE_BOOL: { + datum->kind = FLT_SCL_DATUM_KIND_INT64; + datum->val = (valNode->datum.b) ? 0 : 1; + break; + } + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: { + datum->kind = FLT_SCL_DATUM_KIND_INT64; + datum->val = valNode->datum.i; + break; + } + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: { + datum->kind = FLT_SCL_DATUM_KIND_UINT64; + *(uint64_t *)&datum->val = valNode->datum.u; + break; + } + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { + datum->kind = FLT_SCL_DATUM_KIND_FLOAT64; + *(double *)&datum->val = valNode->datum.d; + break; + } + //TODO:varchar/nchar/json + default: { + qError("not supported"); + break; + } + } + + } + return TSDB_CODE_SUCCESS; +} + +int32_t fltSclBuildRangePoints(SFltSclOperator* oper, SArray* points) { + switch (oper->type) { + case OP_TYPE_GREATER_THAN: { + SFltSclDatum start; + fltSclBuildDatumFromValueNode(&start, oper->valNode); + SFltSclPoint startPt = {.start = true, .excl = true, .val = start}; + SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; + taosArrayPush(points, &start); + taosArrayPush(points, &end); break; - default: + } + default: { + qError("not supported op"); break; + } + } + return TSDB_CODE_SUCCESS; +} + +//TODO: process DNF composed of CNF +int32_t fltSclProcessCNF(SArray* sclOpList, SArray* colRangeList) { + size_t sz = taosArrayGetSize(sclOpList); + for (int32_t i = 0; i < sz; ++i) { + SFltSclOperator* sclOper = taosArrayGet(sclOpList, i); + SFltSclColumnRange *colRange = fltSclGetColumnRange(sclOper->colNode, colRangeList); + SArray* points = taosArrayInit(4, sizeof(SFltSclPoint)); + fltSclBuildRangePoints(sclOper, points); + SArray* merged = taosArrayInit(4, sizeof(SFltSclPoint)); + int32_t code = fltSclIntersect(colRange->points, points, merged); + taosArrayDestroy(colRange->points); + taosArrayDestroy(points); + colRange->points = merged; + } return TSDB_CODE_SUCCESS; } From 3e92ec7a8a4c6f701c31a95a2c4d80e816e6817f Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 19 May 2023 14:09:37 +0800 Subject: [PATCH 05/97] fix: save work --- source/libs/scalar/src/filter.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 01bd53008e..ffc165cba9 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4223,6 +4223,16 @@ int32_t fltSclBuildRangePoints(SFltSclOperator* oper, SArray* points) { taosArrayPush(points, &end); break; } + case OP_TYPE_GREATER_EQUAL: { + SFltSclDatum start; + fltSclBuildDatumFromValueNode(&start, oper->valNode); + SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; + SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; + taosArrayPush(points, &start); + taosArrayPush(points, &end); + } + case default: { qError("not supported op"); break; From 1f6d894dfd12170afd9208e67fb0136faf30b420 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 19 May 2023 17:06:03 +0800 Subject: [PATCH 06/97] fix: save work --- source/libs/scalar/src/filter.c | 121 ++++++++++++++++++++++++-------- 1 file changed, 91 insertions(+), 30 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index ffc165cba9..fd6e79ed42 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3958,10 +3958,10 @@ typedef enum { typedef struct { SFltSclDatumKind kind; union { - int64_t val; // for int64, uint64 and double and bool (1 true, 0 false) - uint8_t *pData; // for varchar, nchar + int64_t val; // for int64, uint64 and double and bool (1 true, 0 false) + uint8_t *pData; // for varchar, nchar }; - SDataType type; // TODO: original data type, may not be used? + SDataType type; // TODO: original data type, may not be used? } SFltSclDatum; typedef struct { @@ -3987,7 +3987,7 @@ int32_t fltSclCompareWithFloat64(SFltSclDatum *val1, SFltSclDatum *val2) { case FLT_SCL_DATUM_KIND_FLOAT64: { return compareDoubleVal(&val1->val, &val2->val); } - //TODO: varchar, nchar + // TODO: varchar, nchar default: qError("not support comparsion. %d, %d", val1->kind, val2->kind); return (val1->kind - val2->kind); @@ -4004,7 +4004,7 @@ int32_t fltSclCompareWithInt64(SFltSclDatum *val1, SFltSclDatum *val2) { case FLT_SCL_DATUM_KIND_FLOAT64: { return compareDoubleInt64(&val1->val, &val2->val); } - //TODO: varchar, nchar + // TODO: varchar, nchar default: qError("not support comparsion. %d, %d", val1->kind, val2->kind); return (val1->kind - val2->kind); @@ -4021,7 +4021,7 @@ int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { case FLT_SCL_DATUM_KIND_FLOAT64: { return compareDoubleUint64(&val1->val, &val2->val); } - //TODO: varchar, nchar + // TODO: varchar, nchar default: qError("not support comparsion. %d, %d", val1->kind, val2->kind); return (val1->kind - val2->kind); @@ -4047,7 +4047,7 @@ int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { case FLT_SCL_DATUM_KIND_FLOAT64: { return fltSclCompareWithFloat64(val1, val2); } - //TODO: varchar/nchar + // TODO: varchar/nchar default: fltError("not supported kind. just return 0"); return 0; @@ -4142,17 +4142,17 @@ typedef struct { EOperatorType type; } SFltSclOperator; -//TODO: column, constant +// TODO: column, constant typedef struct { - SColumnNode* colNode; - SArray* points; + SColumnNode *colNode; + SArray *points; } SFltSclColumnRange; -SFltSclColumnRange* fltSclGetColumnRange(SColumnNode* colNode, SArray* colRangeList) { +SFltSclColumnRange *fltSclGetColumnRange(SColumnNode *colNode, SArray *colRangeList) { for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) { - SFltSclColumnRange* colRange = taosArrayGet(colRangeList, i); - if (nodesEqualNode((SNode*)colRange->colNode, (SNode*)colNode)) { + SFltSclColumnRange *colRange = taosArrayGet(colRangeList, i); + if (nodesEqualNode((SNode *)colRange->colNode, (SNode *)colNode)) { return colRange; } } @@ -4161,7 +4161,7 @@ SFltSclColumnRange* fltSclGetColumnRange(SColumnNode* colNode, SArray* colRangeL return taosArrayGetLast(colRangeList); } -int32_t fltSclBuildDatumFromValueNode(SFltSclDatum* datum, SValueNode* valNode) { +int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) { datum->type = valNode->node.resType; if (valNode->isNull) { @@ -4200,39 +4200,101 @@ int32_t fltSclBuildDatumFromValueNode(SFltSclDatum* datum, SValueNode* valNode) *(double *)&datum->val = valNode->datum.d; break; } - //TODO:varchar/nchar/json + // TODO:varchar/nchar/json default: { qError("not supported"); break; } } - } return TSDB_CODE_SUCCESS; } -int32_t fltSclBuildRangePoints(SFltSclOperator* oper, SArray* points) { +int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { + if (IS_UNSIGNED_NUMERIC_TYPE(oper->colNode->node.resType.type) && + ((IS_SIGNED_NUMERIC_TYPE(oper->valNode->node.resType.type) && oper->valNode->datum.i < 0) || + (IS_FLOAT_TYPE(oper->valNode->node.resType.type) && oper->valNode->datum.d < 0))) { + if (oper->type == OP_TYPE_GREATER_THAN || oper->type == OP_TYPE_GREATER_EQUAL || oper->type == OP_TYPE_NOT_EQUAL) { + SFltSclDatum start; + fltSclBuildDatumFromValueNode(&start, oper->valNode); + start.val = 0; + SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; + SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + } + return TSDB_CODE_SUCCESS; + } switch (oper->type) { case OP_TYPE_GREATER_THAN: { SFltSclDatum start; fltSclBuildDatumFromValueNode(&start, oper->valNode); SFltSclPoint startPt = {.start = true, .excl = true, .val = start}; - SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; - taosArrayPush(points, &start); - taosArrayPush(points, &end); + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); break; } case OP_TYPE_GREATER_EQUAL: { SFltSclDatum start; fltSclBuildDatumFromValueNode(&start, oper->valNode); SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; - SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; - taosArrayPush(points, &start); - taosArrayPush(points, &end); + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + case OP_TYPE_LOWER_THAN: { + SFltSclDatum end; + fltSclBuildDatumFromValueNode(&end, oper->valNode); + SFltSclPoint endPt = {.start = false, .excl = true, .val = end}; + SFltSclDatum start = {.kind = FLT_SCL_DATUM_KIND_MIN, .type = oper->colNode->node.resType}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + case OP_TYPE_LOWER_EQUAL: { + SFltSclDatum end; + fltSclBuildDatumFromValueNode(&end, oper->valNode); + SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; + SFltSclDatum start = {.kind = FLT_SCL_DATUM_KIND_MIN, .type = oper->colNode->node.resType}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + case OP_TYPE_EQUAL: { + SFltSclDatum valDatum; + fltSclBuildDatumFromValueNode(&valDatum, oper->valNode); + SFltSclPoint startPt = {.start = true, .excl = false, .val = valDatum}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = valDatum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + case OP_TYPE_NOT_EQUAL: { + SFltSclDatum valDatum; + fltSclBuildDatumFromValueNode(&valDatum, oper->valNode); + { + SFltSclDatum start = {.kind = FLT_SCL_DATUM_KIND_MIN, .type = oper->colNode->node.resType}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; + SFltSclPoint endPt = {.start = false, .excl = true, .val = valDatum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + } + { + SFltSclPoint startPt = {.start = true, .excl = true, .val = valDatum}; + SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + } + break; } - case default: { qError("not supported op"); break; @@ -4241,20 +4303,19 @@ int32_t fltSclBuildRangePoints(SFltSclOperator* oper, SArray* points) { return TSDB_CODE_SUCCESS; } -//TODO: process DNF composed of CNF -int32_t fltSclProcessCNF(SArray* sclOpList, SArray* colRangeList) { +// TODO: process DNF composed of CNF +int32_t fltSclProcessCNF(SArray *sclOpList, SArray *colRangeList) { size_t sz = taosArrayGetSize(sclOpList); for (int32_t i = 0; i < sz; ++i) { - SFltSclOperator* sclOper = taosArrayGet(sclOpList, i); + SFltSclOperator *sclOper = taosArrayGet(sclOpList, i); SFltSclColumnRange *colRange = fltSclGetColumnRange(sclOper->colNode, colRangeList); - SArray* points = taosArrayInit(4, sizeof(SFltSclPoint)); + SArray *points = taosArrayInit(4, sizeof(SFltSclPoint)); fltSclBuildRangePoints(sclOper, points); - SArray* merged = taosArrayInit(4, sizeof(SFltSclPoint)); + SArray *merged = taosArrayInit(4, sizeof(SFltSclPoint)); int32_t code = fltSclIntersect(colRange->points, points, merged); taosArrayDestroy(colRange->points); taosArrayDestroy(points); colRange->points = merged; - } return TSDB_CODE_SUCCESS; } From f520de0dab9a2924091eba27f8e0be411f1f1830 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 22 May 2023 10:07:01 +0800 Subject: [PATCH 07/97] refactor: adjust the module dependency. --- source/dnode/vnode/src/tq/tqRestore.c | 5 +++-- source/libs/executor/CMakeLists.txt | 3 +-- source/libs/stream/src/streamExec.c | 3 +-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index ff9f95d5fa..6e3036a66b 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -61,9 +61,10 @@ static int32_t doSetOffsetForWalReader(SStreamTask *pTask, int32_t vgId) { // seek the stored version and extract data from WAL int64_t firstVer = walReaderGetValidFirstVer(pTask->exec.pWalReader); if (pTask->chkInfo.currentVer < firstVer) { + tqWarn("vgId:%d s-task:%s ver:%"PRId64" earlier than the first ver of wal range %" PRId64 ", forward to %" PRId64, vgId, + pTask->id.idStr, pTask->chkInfo.currentVer, firstVer, firstVer); + pTask->chkInfo.currentVer = firstVer; - tqWarn("vgId:%d s-task:%s ver earlier than the first ver of wal range %" PRId64 ", forward to %" PRId64, vgId, - pTask->id.idStr, firstVer, pTask->chkInfo.currentVer); // todo need retry if failed int32_t code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer); diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 8b3d04e32c..4adff71d85 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -1,10 +1,9 @@ aux_source_directory(src EXECUTOR_SRC) -#add_library(executor ${EXECUTOR_SRC}) add_library(executor STATIC ${EXECUTOR_SRC}) target_link_libraries(executor - PRIVATE os util common function parser planner qcom vnode scalar nodes index stream + PRIVATE os util common function parser planner qcom vnode scalar nodes index ) target_include_directories( diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 55474541ed..637d661343 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -163,7 +163,6 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { int32_t code = 0; ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); - void* exec = pTask->exec.pExecutor; qSetStreamOpOpen(exec); @@ -404,7 +403,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { { // set input - void* pExecutor = pTask->exec.pExecutor; + void* pExecutor = pTask->exec.pExecutor; const SStreamQueueItem* pItem = pInput; if (pItem->type == STREAM_INPUT__GET_RES) { From 6456d5dccfd11afe518fc3a282494eb50a0e8a38 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 22 May 2023 11:41:19 +0800 Subject: [PATCH 08/97] enhance: code finish --- source/libs/scalar/inc/filterInt.h | 7 + source/libs/scalar/src/filter.c | 700 ++++++++++++++++------------- 2 files changed, 407 insertions(+), 300 deletions(-) diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index 2023d38777..1ca8ac1d8c 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -227,8 +227,10 @@ typedef struct SFltTreeStat { SFilterInfo *info; } SFltTreeStat; + typedef struct SFltScalarCtx { SNode *node; + SArray* fltSclRange; } SFltScalarCtx; typedef struct SFltBuildGroupCtx { @@ -237,6 +239,11 @@ typedef struct SFltBuildGroupCtx { int32_t code; } SFltBuildGroupCtx; +typedef struct { + SColumnNode *colNode; + SArray *points; +} SFltSclColumnRange; + struct SFilterInfo { bool scalarMode; SFltScalarCtx sclCtx; diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index fd6e79ed42..33f37c722e 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1841,6 +1841,13 @@ void filterFreeInfo(SFilterInfo *info) { return; } + for (int32_t i = 0 ; i < taosArrayGetSize(info->sclCtx.fltSclRange); ++i) { + SFltSclColumnRange* colRange = taosArrayGet(info->sclCtx.fltSclRange, i); + nodesDestroyNode((SNode*)colRange->colNode); + taosArrayDestroy(colRange->points); + } + taosArrayDestroy(info->sclCtx.fltSclRange); + taosMemoryFreeClear(info->cunits); taosMemoryFreeClear(info->blkUnitRes); taosMemoryFreeClear(info->blkUnits); @@ -3424,8 +3431,335 @@ _return: return code; } + +// compare ranges, null < min < val < max. null=null, min=min, max=max +typedef enum { + FLT_SCL_DATUM_KIND_NULL, + FLT_SCL_DATUM_KIND_MIN, + FLT_SCL_DATUM_KIND_INT64, + FLT_SCL_DATUM_KIND_UINT64, + FLT_SCL_DATUM_KIND_FLOAT64, + FLT_SCL_DATUM_KIND_VARCHAR, + FLT_SCL_DATUM_KIND_NCHAR, + FLT_SCL_DATUM_KIND_MAX, +} SFltSclDatumKind; + +typedef struct { + SFltSclDatumKind kind; + union { + int64_t i; // for int64, uint64 and double and bool (1 true, 0 false) + uint64_t u; + double d; + uint8_t *pData; // for varchar, nchar + }; + SDataType type; // TODO: original data type, may not be used? +} SFltSclDatum; + +typedef struct { + SFltSclDatum val; + bool excl; + bool start; +} SFltSclPoint; + +int32_t fltSclCompareWithFloat64(SFltSclDatum *val1, SFltSclDatum *val2) { + // val2->kind == float64 + switch (val1->kind) { + case FLT_SCL_DATUM_KIND_UINT64: + return compareUint64Double(&val1->u, &val2->d); + case FLT_SCL_DATUM_KIND_INT64: + return compareInt64Double(&val1->i, &val2->d); + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareDoubleVal(&val1->d, &val2->d); + } + // TODO: varchar, nchar + default: + qError("not support comparsion. %d, %d", val1->kind, val2->kind); + return (val1->kind - val2->kind); + } +} + +int32_t fltSclCompareWithInt64(SFltSclDatum *val1, SFltSclDatum *val2) { + // val2->kind == int64 + switch (val1->kind) { + case FLT_SCL_DATUM_KIND_UINT64: + return compareUint64Int64(&val1->u, &val2->i); + case FLT_SCL_DATUM_KIND_INT64: + return compareInt64Val(&val1->i, &val2->i); + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareDoubleInt64(&val1->d, &val2->i); + } + // TODO: varchar, nchar + default: + qError("not support comparsion. %d, %d", val1->kind, val2->kind); + return (val1->kind - val2->kind); + } +} + +int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { + // val2 kind == uint64 + switch (val1->kind) { + case FLT_SCL_DATUM_KIND_UINT64: + return compareUint64Val(&val1->u, &val2->u); + case FLT_SCL_DATUM_KIND_INT64: + return compareInt64Uint64(&val1->i, &val2->u); + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareDoubleUint64(&val1->d, &val2->u); + } + // TODO: varchar, nchar + default: + qError("not support comparsion. %d, %d", val1->kind, val2->kind); + return (val1->kind - val2->kind); + } +} + +int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { + if (val2->kind == FLT_SCL_DATUM_KIND_NULL || val2->kind == FLT_SCL_DATUM_KIND_MIN || + val2->kind == FLT_SCL_DATUM_KIND_MAX) { + if (val1->kind == FLT_SCL_DATUM_KIND_NULL || val1->kind == FLT_SCL_DATUM_KIND_MIN || + val1->kind == FLT_SCL_DATUM_KIND_MAX) { + return (val1->kind < val2->kind) ? -1 : ((val1->kind > val2->kind) ? 1 : 0); + } + } + + switch (val2->kind) { + case FLT_SCL_DATUM_KIND_UINT64: { + return fltSclCompareWithUInt64(val1, val2); + } + case FLT_SCL_DATUM_KIND_INT64: { + return fltSclCompareWithInt64(val1, val2); + } + case FLT_SCL_DATUM_KIND_FLOAT64: { + return fltSclCompareWithFloat64(val1, val2); + } + // TODO: varchar/nchar + default: + fltError("not supported kind. just return 0"); + return 0; + break; + } + return 0; +} + +bool fltSclLessPoint(SFltSclPoint *pt1, SFltSclPoint *pt2) { + // first value compare + int32_t cmp = fltSclCompareDatum(&pt1->val, &pt2->val); + if (cmp != 0) { + return cmp < 0; + } + + if (pt1->start && pt2->start) { + return !pt1->excl && pt2->excl; + } else if (pt1->start) { + return pt1->excl && !pt2->excl; + } else if (pt2->start) { + return pt1->excl || pt2->excl; + } + return pt1->excl && !pt2->excl; +} + +int32_t fltSclMergeSort(SArray *pts1, SArray *pts2, SArray *result) { + size_t len1 = taosArrayGetSize(pts1); + size_t len2 = taosArrayGetSize(pts2); + size_t i = 0; + size_t j = 0; + while (i < len1 && j < len2) { + SFltSclPoint *pt1 = taosArrayGet(pts1, i); + SFltSclPoint *pt2 = taosArrayGet(pts2, j); + bool less = fltSclLessPoint(pt1, pt2); + if (less) { + taosArrayPush(result, pt1); + ++i; + } else { + taosArrayPush(result, pt2); + ++j; + } + } + if (i < len1) { + for (; i < len1; ++i) { + SFltSclPoint *pt1 = taosArrayGet(pts1, i); + taosArrayPush(result, pt1); + } + } + if (j < len2) { + for (; j < len2; ++j) { + SFltSclPoint *pt2 = taosArrayGet(pts2, j); + taosArrayPush(result, pt2); + } + } + return 0; +} + +// pts1 and pts2 must be ordered and de-duplicated and each range can not be a range of another range +int32_t fltSclMerge(SArray *pts1, SArray *pts2, bool isUnion, SArray *merged) { + size_t len1 = taosArrayGetSize(pts1); + size_t len2 = taosArrayGetSize(pts2); + // first merge sort pts1 and pts2 + SArray *all = taosArrayInit(len1 + len2, sizeof(SFltSclPoint)); + fltSclMergeSort(pts1, pts2, all); + int32_t countRequired = (isUnion) ? 1 : 2; + int32_t count = 0; + for (int32_t i = 0; i < taosArrayGetSize(all); ++i) { + SFltSclPoint *pt = taosArrayGet(pts1, i); + if (pt->start) { + ++count; + if (count == countRequired) { + taosArrayPush(merged, pt); + } + } else { + if (count == countRequired) { + taosArrayPush(merged, pt); + } + --count; + } + } + taosArrayDestroy(all); + return 0; +} + +int32_t fltSclIntersect(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, false, merged); } + +int32_t fltSclUnion(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, true, merged); } + +// TODO: column, constant +typedef struct { + SColumnNode *colNode; + SValueNode *valNode; + EOperatorType type; +} SFltSclOperator; + + +SFltSclColumnRange *fltSclGetOrCreateColumnRange(SColumnNode *colNode, SArray *colRangeList) { + for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) { + SFltSclColumnRange *colRange = taosArrayGet(colRangeList, i); + if (nodesEqualNode((SNode *)colRange->colNode, (SNode *)colNode)) { + return colRange; + } + } + SColumnNode* pColumnNode = (SColumnNode*)nodesCloneNode((SNode*)colNode); + SFltSclColumnRange newColRange = {.colNode = pColumnNode, .points = taosArrayInit(4, sizeof(SFltSclPoint))}; + taosArrayPush(colRangeList, &newColRange); + return taosArrayGetLast(colRangeList); +} + +int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) { + datum->type = valNode->node.resType; + + if (valNode->isNull) { + datum->kind = FLT_SCL_DATUM_KIND_NULL; + } else { + switch (valNode->node.resType.type) { + case TSDB_DATA_TYPE_NULL: { + datum->kind = FLT_SCL_DATUM_KIND_NULL; + break; + } + case TSDB_DATA_TYPE_BOOL: { + datum->kind = FLT_SCL_DATUM_KIND_INT64; + datum->i = (valNode->datum.b) ? 0 : 1; + break; + } + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: { + datum->kind = FLT_SCL_DATUM_KIND_INT64; + datum->i = valNode->datum.i; + break; + } + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: { + datum->kind = FLT_SCL_DATUM_KIND_UINT64; + datum->u = valNode->datum.u; + break; + } + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { + datum->kind = FLT_SCL_DATUM_KIND_FLOAT64; + datum->d = valNode->datum.d; + break; + } + // TODO:varchar/nchar/json + default: { + qError("not supported"); + break; + } + } + } + return TSDB_CODE_SUCCESS; +} + +int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum* datum, uint8_t type, int64_t val) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: { + datum->kind = FLT_SCL_DATUM_KIND_INT64; + datum->i = val; + break; + } + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: { + datum->kind = FLT_SCL_DATUM_KIND_UINT64; + datum->u = *(uint64_t*)&val; + break; + } + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { + datum->kind = FLT_SCL_DATUM_KIND_FLOAT64; + datum->d = *(double*)&val; + break; + } + // TODO:varchar/nchar/json + default: { + qError("not supported"); + break; + } + } + + + return TSDB_CODE_SUCCESS; +} + +int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange* colRange, SColumnDataAgg* pAgg, SArray* points) { + SFltSclDatum min; + fltSclBuildDatumFromBlockSmaValue(&min, colRange->colNode->node.resType.type, pAgg->min); + SFltSclPoint minPt = {.excl = false, .start = true, .val = min}; + SFltSclDatum max; + fltSclBuildDatumFromBlockSmaValue(&max, colRange->colNode->node.resType.type, pAgg->max); + SFltSclPoint maxPt = {.excl = false, .start = false, .val = max}; + taosArrayPush(points, &minPt); + taosArrayPush(points, &maxPt); + return TSDB_CODE_SUCCESS; +} + bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t numOfCols, int32_t numOfRows) { if (info->scalarMode) { + SArray* colRanges = info->sclCtx.fltSclRange; + for (int32_t i = 0; i < taosArrayGetSize(colRanges); ++i) { + SFltSclColumnRange* colRange = taosArrayGet(colRanges, i); + bool foundCol = false; + int32_t j = 0; + for (; j < numOfCols; ++j) { + if (pDataStatis[j] != NULL && pDataStatis[j]->colId == colRange->colNode->colId) { + foundCol = true; + } + } + if (foundCol) { + SColumnDataAgg* pAgg = pDataStatis[j]; + SArray* points = taosArrayInit(2, sizeof(SFltSclPoint)); + fltSclBuildRangeFromBlockSma(colRange, pAgg, points); + SArray* merged = taosArrayInit(8, sizeof(SFltSclPoint)); + fltSclIntersect(points, colRange->points, merged); + bool isIntersect = taosArrayGetSize(merged) != 0; + taosArrayDestroy(merged); + taosArrayDestroy(points); + if (!isIntersect) { + return false; + } + } + } return true; } @@ -3943,289 +4277,7 @@ _return: FLT_RET(code); } -// compare ranges, null < min < val < max. null=null, min=min, max=max -typedef enum { - FLT_SCL_DATUM_KIND_NULL, - FLT_SCL_DATUM_KIND_MIN, - FLT_SCL_DATUM_KIND_INT64, - FLT_SCL_DATUM_KIND_UINT64, - FLT_SCL_DATUM_KIND_FLOAT64, - FLT_SCL_DATUM_KIND_VARCHAR, - FLT_SCL_DATUM_KIND_NCHAR, - FLT_SCL_DATUM_KIND_MAX, -} SFltSclDatumKind; - -typedef struct { - SFltSclDatumKind kind; - union { - int64_t val; // for int64, uint64 and double and bool (1 true, 0 false) - uint8_t *pData; // for varchar, nchar - }; - SDataType type; // TODO: original data type, may not be used? -} SFltSclDatum; - -typedef struct { - SFltSclDatum val; - bool excl; - bool start; -} SFltSclPoint; - -typedef struct { - SFltSclDatum low; - SFltSclDatum high; - bool lowExcl; - bool highExcl; -} SFltSclRange; - -int32_t fltSclCompareWithFloat64(SFltSclDatum *val1, SFltSclDatum *val2) { - // val2->kind == int64 - switch (val1->kind) { - case FLT_SCL_DATUM_KIND_UINT64: - return compareUint64Double(&val1->val, &val2->val); - case FLT_SCL_DATUM_KIND_INT64: - return compareInt64Double(&val1->val, &val2->val); - case FLT_SCL_DATUM_KIND_FLOAT64: { - return compareDoubleVal(&val1->val, &val2->val); - } - // TODO: varchar, nchar - default: - qError("not support comparsion. %d, %d", val1->kind, val2->kind); - return (val1->kind - val2->kind); - } -} - -int32_t fltSclCompareWithInt64(SFltSclDatum *val1, SFltSclDatum *val2) { - // val2->kind == int64 - switch (val1->kind) { - case FLT_SCL_DATUM_KIND_UINT64: - return compareUint64Int64(&val1->val, &val2->val); - case FLT_SCL_DATUM_KIND_INT64: - return compareInt64Val(&val1->val, &val2->val); - case FLT_SCL_DATUM_KIND_FLOAT64: { - return compareDoubleInt64(&val1->val, &val2->val); - } - // TODO: varchar, nchar - default: - qError("not support comparsion. %d, %d", val1->kind, val2->kind); - return (val1->kind - val2->kind); - } -} - -int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { - // val2 kind == uint64 - switch (val1->kind) { - case FLT_SCL_DATUM_KIND_UINT64: - return compareUint64Val(&val1->val, &val2->val); - case FLT_SCL_DATUM_KIND_INT64: - return compareInt64Uint64(&val1->val, &val2->val); - case FLT_SCL_DATUM_KIND_FLOAT64: { - return compareDoubleUint64(&val1->val, &val2->val); - } - // TODO: varchar, nchar - default: - qError("not support comparsion. %d, %d", val1->kind, val2->kind); - return (val1->kind - val2->kind); - } -} - -int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { - if (val2->kind == FLT_SCL_DATUM_KIND_NULL || val2->kind == FLT_SCL_DATUM_KIND_MIN || - val2->kind == FLT_SCL_DATUM_KIND_MAX) { - if (val1->kind == FLT_SCL_DATUM_KIND_NULL || val1->kind == FLT_SCL_DATUM_KIND_MIN || - val1->kind == FLT_SCL_DATUM_KIND_MAX) { - return (val1->kind < val2->kind) ? -1 : ((val1->kind > val2->kind) ? 1 : 0); - } - } - - switch (val2->kind) { - case FLT_SCL_DATUM_KIND_UINT64: { - return fltSclCompareWithUInt64(val1, val2); - } - case FLT_SCL_DATUM_KIND_INT64: { - return fltSclCompareWithInt64(val1, val2); - } - case FLT_SCL_DATUM_KIND_FLOAT64: { - return fltSclCompareWithFloat64(val1, val2); - } - // TODO: varchar/nchar - default: - fltError("not supported kind. just return 0"); - return 0; - break; - } - return 0; -} - -bool fltSclLessPoint(SFltSclPoint *pt1, SFltSclPoint *pt2) { - // first value compare - int32_t cmp = fltSclCompareDatum(&pt1->val, &pt2->val); - if (cmp != 0) { - return cmp < 0; - } - - if (pt1->start && pt2->start) { - return !pt1->excl && pt2->excl; - } else if (pt1->start) { - return pt1->excl && !pt2->excl; - } else if (pt2->start) { - return pt1->excl || pt2->excl; - } - return pt1->excl && !pt2->excl; -} - -int32_t fltSclMergeSort(SArray *pts1, SArray *pts2, SArray *result) { - size_t len1 = taosArrayGetSize(pts1); - size_t len2 = taosArrayGetSize(pts2); - size_t i = 0; - size_t j = 0; - while (i < len1 && j < len2) { - SFltSclPoint *pt1 = taosArrayGet(pts1, i); - SFltSclPoint *pt2 = taosArrayGet(pts2, j); - bool less = fltSclLessPoint(pt1, pt2); - if (less) { - taosArrayPush(result, pt1); - ++i; - } else { - taosArrayPush(result, pt2); - ++j; - } - } - if (i < len1) { - for (; i < len1; ++i) { - SFltSclPoint *pt1 = taosArrayGet(pts1, i); - taosArrayPush(result, pt1); - } - } - if (j < len2) { - for (; j < len2; ++j) { - SFltSclPoint *pt2 = taosArrayGet(pts2, j); - taosArrayPush(result, pt2); - } - } - return 0; -} - -// pts1 and pts2 must be ordered and de-duplicated and each range can not be a range of another range -int32_t fltSclMerge(SArray *pts1, SArray *pts2, bool isUnion, SArray *merged) { - size_t len1 = taosArrayGetSize(pts1); - size_t len2 = taosArrayGetSize(pts2); - // first merge sort pts1 and pts2 - SArray *all = taosArrayInit(len1 + len2, sizeof(SFltSclPoint)); - fltSclMergeSort(pts1, pts2, all); - int32_t countRequired = (isUnion) ? 1 : 2; - int32_t count = 0; - for (int32_t i = 0; i < taosArrayGetSize(all); ++i) { - SFltSclPoint *pt = taosArrayGet(pts1, i); - if (pt->start) { - ++count; - if (count == countRequired) { - taosArrayPush(merged, pt); - } - } else { - if (count == countRequired) { - taosArrayPush(merged, pt); - } - --count; - } - } - taosArrayDestroy(all); - return 0; -} - -int32_t fltSclIntersect(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, false, merged); } - -int32_t fltSclUnion(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, true, merged); } - -typedef struct { - SColumnNode *colNode; - SValueNode *valNode; - EOperatorType type; -} SFltSclOperator; - -// TODO: column, constant - -typedef struct { - SColumnNode *colNode; - SArray *points; -} SFltSclColumnRange; - -SFltSclColumnRange *fltSclGetColumnRange(SColumnNode *colNode, SArray *colRangeList) { - for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) { - SFltSclColumnRange *colRange = taosArrayGet(colRangeList, i); - if (nodesEqualNode((SNode *)colRange->colNode, (SNode *)colNode)) { - return colRange; - } - } - SFltSclColumnRange newColRange = {.colNode = colNode, .points = taosArrayInit(4, sizeof(SFltSclPoint))}; - taosArrayPush(colRangeList, &newColRange); - return taosArrayGetLast(colRangeList); -} - -int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) { - datum->type = valNode->node.resType; - - if (valNode->isNull) { - datum->kind = FLT_SCL_DATUM_KIND_NULL; - } else { - switch (valNode->node.resType.type) { - case TSDB_DATA_TYPE_NULL: { - datum->kind = FLT_SCL_DATUM_KIND_NULL; - break; - } - case TSDB_DATA_TYPE_BOOL: { - datum->kind = FLT_SCL_DATUM_KIND_INT64; - datum->val = (valNode->datum.b) ? 0 : 1; - break; - } - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: { - datum->kind = FLT_SCL_DATUM_KIND_INT64; - datum->val = valNode->datum.i; - break; - } - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_UBIGINT: { - datum->kind = FLT_SCL_DATUM_KIND_UINT64; - *(uint64_t *)&datum->val = valNode->datum.u; - break; - } - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: { - datum->kind = FLT_SCL_DATUM_KIND_FLOAT64; - *(double *)&datum->val = valNode->datum.d; - break; - } - // TODO:varchar/nchar/json - default: { - qError("not supported"); - break; - } - } - } - return TSDB_CODE_SUCCESS; -} - int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { - if (IS_UNSIGNED_NUMERIC_TYPE(oper->colNode->node.resType.type) && - ((IS_SIGNED_NUMERIC_TYPE(oper->valNode->node.resType.type) && oper->valNode->datum.i < 0) || - (IS_FLOAT_TYPE(oper->valNode->node.resType.type) && oper->valNode->datum.d < 0))) { - if (oper->type == OP_TYPE_GREATER_THAN || oper->type == OP_TYPE_GREATER_EQUAL || oper->type == OP_TYPE_NOT_EQUAL) { - SFltSclDatum start; - fltSclBuildDatumFromValueNode(&start, oper->valNode); - start.val = 0; - SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; - SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; - SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; - taosArrayPush(points, &startPt); - taosArrayPush(points, &endPt); - } - return TSDB_CODE_SUCCESS; - } switch (oper->type) { case OP_TYPE_GREATER_THAN: { SFltSclDatum start; @@ -4304,36 +4356,84 @@ int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { } // TODO: process DNF composed of CNF -int32_t fltSclProcessCNF(SArray *sclOpList, SArray *colRangeList) { - size_t sz = taosArrayGetSize(sclOpList); +int32_t fltSclProcessCNF(SArray *sclOpListCNF, SArray *colRangeList) { + size_t sz = taosArrayGetSize(sclOpListCNF); for (int32_t i = 0; i < sz; ++i) { - SFltSclOperator *sclOper = taosArrayGet(sclOpList, i); - SFltSclColumnRange *colRange = fltSclGetColumnRange(sclOper->colNode, colRangeList); + SFltSclOperator *sclOper = taosArrayGet(sclOpListCNF, i); + SFltSclColumnRange *colRange = fltSclGetOrCreateColumnRange(sclOper->colNode, colRangeList); SArray *points = taosArrayInit(4, sizeof(SFltSclPoint)); fltSclBuildRangePoints(sclOper, points); - SArray *merged = taosArrayInit(4, sizeof(SFltSclPoint)); - int32_t code = fltSclIntersect(colRange->points, points, merged); - taosArrayDestroy(colRange->points); - taosArrayDestroy(points); - colRange->points = merged; + if (taosArrayGetSize(colRange->points) != 0) { + SArray *merged = taosArrayInit(4, sizeof(SFltSclPoint)); + int32_t code = fltSclIntersect(colRange->points, points, merged); + taosArrayDestroy(colRange->points); + taosArrayDestroy(points); + colRange->points = merged; + } else { + colRange->points = points; + } + } return TSDB_CODE_SUCCESS; } -typedef struct { - bool useRange; - SHashObj *colRanges; -} SFltSclOptCtx; +static int32_t fltSclCollectOperatorFromNode(SNode* pNode, SArray* sclOpList) { + if (nodeType(pNode) != QUERY_NODE_OPERATOR) { + return TSDB_CODE_SUCCESS; + } + SOperatorNode* pOper = (SOperatorNode*)pNode; + //TODO: left value node, right column node + //TODO: datatype + //TODO: operator + if (nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_VALUE && + (pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL || + pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL || + pOper->opType == OP_TYPE_NOT_EQUAL || pOper->opType == OP_TYPE_EQUAL)) { + SValueNode* valNode = (SValueNode*)pOper->pRight; + if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { + SNode *p = nodesCloneNode(pNode); + taosArrayPush(sclOpList, &p); + } + } + return TSDB_CODE_SUCCESS; +} -static EDealRes fltSclMayBeOptimed(SNode *pNode, void *pCtx) { - SFltSclOptCtx *ctx = (SFltSclOptCtx *)pCtx; - return DEAL_RES_CONTINUE; +static int32_t fltSclCollectOperatorsFromLogicCond(SNode* pNode, SArray* sclOpList) { + if (nodeType(pNode) != QUERY_NODE_LOGIC_CONDITION) { + return TSDB_CODE_SUCCESS; + } + SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pNode; + //TODO: support LOGIC_COND_TYPE_OR + if (pLogicCond->condType != LOGIC_COND_TYPE_AND) { + return TSDB_CODE_SUCCESS; + } + SNode* pExpr = NULL; + FOREACH(pExpr, pLogicCond->pParameterList) { fltSclCollectOperatorFromNode(pExpr, sclOpList); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t fltSclCollectOperators(SNode *pNode, SArray* sclOpList) { + if (nodeType(pNode) == QUERY_NODE_OPERATOR) { + fltSclCollectOperatorFromNode(pNode, sclOpList); + } else if (nodeType(pNode) == QUERY_NODE_LOGIC_CONDITION) { + fltSclCollectOperatorsFromLogicCond(pNode, sclOpList); + } + return TSDB_CODE_SUCCESS; } int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) { - SFltSclOptCtx ctx; - nodesWalkExprPostOrder(*pNode, fltSclMayBeOptimed, &ctx); + SArray* sclOpList = taosArrayInit(16, POINTER_BYTES); + fltSclCollectOperators(*pNode, sclOpList); + SArray* colRangeList = taosArrayInit(16, sizeof(SFltSclColumnRange)); + fltSclProcessCNF(sclOpList, colRangeList); + pInfo->sclCtx.fltSclRange = colRangeList; + for (int32_t i = 0; i < taosArrayGetSize(sclOpList); ++i) { + SNode* p = taosArrayGetP(sclOpList, i); + nodesDestroyNode(p); + } + taosArrayDestroy(sclOpList); return TSDB_CODE_SUCCESS; } From 43ac6fb1fd532c96f5deba1b21e46abd5b2e0762 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 23 May 2023 08:43:31 +0800 Subject: [PATCH 09/97] fix: all null from block sma --- source/libs/scalar/src/filter.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 33f37c722e..4475a07dcd 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3722,7 +3722,15 @@ int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum* datum, uint8_t type, int return TSDB_CODE_SUCCESS; } -int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange* colRange, SColumnDataAgg* pAgg, SArray* points) { +int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange* colRange, SColumnDataAgg* pAgg, int32_t numOfRows, SArray* points) { + if (pAgg->numOfNull == numOfRows) { + SFltSclDatum datum = {.kind = FLT_SCL_DATUM_KIND_NULL}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = datum}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = datum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + return TSDB_CODE_SUCCESS; + } SFltSclDatum min; fltSclBuildDatumFromBlockSmaValue(&min, colRange->colNode->node.resType.type, pAgg->min); SFltSclPoint minPt = {.excl = false, .start = true, .val = min}; @@ -3749,7 +3757,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t if (foundCol) { SColumnDataAgg* pAgg = pDataStatis[j]; SArray* points = taosArrayInit(2, sizeof(SFltSclPoint)); - fltSclBuildRangeFromBlockSma(colRange, pAgg, points); + fltSclBuildRangeFromBlockSma(colRange, pAgg, numOfRows, points); SArray* merged = taosArrayInit(8, sizeof(SFltSclPoint)); fltSclIntersect(points, colRange->points, merged); bool isIntersect = taosArrayGetSize(merged) != 0; From 24ae8d7ecb870852c9979083eddb5686b32bb278 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 23 May 2023 09:49:42 +0800 Subject: [PATCH 10/97] fix: fix ci memory leak error --- source/libs/scalar/src/filter.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 4475a07dcd..c21abc1380 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4378,6 +4378,7 @@ int32_t fltSclProcessCNF(SArray *sclOpListCNF, SArray *colRangeList) { taosArrayDestroy(points); colRange->points = merged; } else { + taosArrayDestroy(colRange->points); colRange->points = points; } From 77a1de444c24bd68d0db36c2d7820feb4b1a8c4b Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 23 May 2023 10:47:58 +0800 Subject: [PATCH 11/97] fix: where c7 means c7 is true, the right child is null --- source/libs/scalar/src/filter.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index c21abc1380..4a9ebb16c1 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4394,6 +4394,9 @@ static int32_t fltSclCollectOperatorFromNode(SNode* pNode, SArray* sclOpList) { //TODO: left value node, right column node //TODO: datatype //TODO: operator + if (pOper->pLeft == NULL || pOper->pRight == NULL) { + return TSDB_CODE_SUCCESS; + } if (nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_VALUE && (pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL || pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL || From c15b1e35203aa9e47809b69f40d542a0860d9712 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 23 May 2023 11:46:02 +0800 Subject: [PATCH 12/97] opt compile --- include/libs/index/index.h | 1 + include/libs/scalar/filter.h | 10 ++++++ source/dnode/vnode/inc/vnode.h | 47 ++++++++++--------------- source/dnode/vnode/src/meta/metaQuery.c | 27 +++++++++----- source/libs/executor/src/executil.c | 40 +++++++++++---------- source/libs/index/CMakeLists.txt | 1 - source/libs/index/src/indexFilter.c | 4 +-- 7 files changed, 71 insertions(+), 59 deletions(-) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 0d31ca2f68..207197ccda 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -212,6 +212,7 @@ typedef struct SIndexMetaArg { void* idx; void* ivtIdx; uint64_t suid; + int (*metaFilterFunc)(void* metaEx, void* param, SArray* result); } SIndexMetaArg; typedef enum { SFLT_NOT_INDEX, SFLT_COARSE_INDEX, SFLT_ACCURATE_INDEX } SIdxFltStatus; diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h index f20ba287de..7c5ca628cd 100644 --- a/include/libs/scalar/filter.h +++ b/include/libs/scalar/filter.h @@ -55,6 +55,16 @@ extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pColsAgg, int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond, SNode **pOtherCond); +typedef struct SMetaFltParam { + tb_uid_t suid; + int16_t cid; + int16_t type; + void *val; + bool reverse; + bool equal; + int (*filterFunc)(void *a, void *b, int16_t type); +} SMetaFltParam; + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 18fa893fa4..60e36fae0e 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -26,6 +26,7 @@ #include "tfs.h" #include "wal.h" +#include "filter.h" #include "tcommon.h" #include "tfs.h" #include "tgrant.h" @@ -125,35 +126,24 @@ int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, in int32_t payloadLen, double selectivityRatio); int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid); tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); -int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid); -int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); -int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, - int32_t payloadLen); +int32_t metaTbGroupCacheClear(SMeta *pMeta, uint64_t suid); +int32_t metaGetCachedTbGroup(SMeta *pMeta, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList); +int32_t metaPutTbGroupToCache(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, + int32_t payloadLen); -int64_t metaGetTbNum(SMeta *pMeta); -int64_t metaGetNtbNum(SMeta *pMeta); +int64_t metaGetTbNum(SMeta *pMeta); +int64_t metaGetNtbNum(SMeta *pMeta); typedef struct { int64_t uid; int64_t ctbNum; } SMetaStbStats; int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); -typedef struct SMetaFltParam { - tb_uid_t suid; - int16_t cid; - int16_t type; - void *val; - bool reverse; - bool equal; - int (*filterFunc)(void *a, void *b, int16_t type); - -} SMetaFltParam; - // TODO, refactor later -int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *results); -int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *parm, SArray *pUids); -int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids); -int32_t metaFilterTtl(SMeta *pMeta, SMetaFltParam *param, SArray *pUids); +int32_t metaFilterTableIds(void *pMeta, void *param, SArray *results); +int32_t metaFilterCreateTime(void *pMeta, void *parm, SArray *pUids); +int32_t metaFilterTableName(void *pMeta, void *param, SArray *pUids); +int32_t metaFilterTtl(void *pMeta, void *param, SArray *pUids); #if 1 // refact APIs below (TODO) typedef SVCreateTbReq STbCfg; @@ -183,7 +173,8 @@ typedef struct STsdbReader STsdbReader; #define CACHESCAN_RETRIEVE_LAST 0x8 int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, - SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly, SHashObj** pIgnoreTables); + SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly, + SHashObj **pIgnoreTables); int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num); void tsdbReaderSetId(STsdbReader *pReader, const char *idstr); void tsdbReaderClose(STsdbReader *pReader); @@ -198,9 +189,9 @@ void *tsdbGetIdx(SMeta *pMeta); void *tsdbGetIvtIdx(SMeta *pMeta); uint64_t tsdbGetReaderMaxVersion(STsdbReader *pReader); void tsdbReaderSetCloseFlag(STsdbReader *pReader); -int64_t tsdbGetLastTimestamp(SVnode* pVnode, void* pTableList, int32_t numOfTables, const char* pIdStr); +int64_t tsdbGetLastTimestamp(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr); -int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables); +int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr); int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, @@ -265,13 +256,13 @@ int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList); int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); -bool tqNextBlockInWal(STqReader* pReader, const char* idstr); -bool tqNextBlockImpl(STqReader *pReader, const char* idstr); +bool tqNextBlockInWal(STqReader *pReader, const char *idstr); +bool tqNextBlockImpl(STqReader *pReader, const char *idstr); -int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, const char* id); +int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, const char *id); int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); -int32_t tqRetrieveDataBlock(STqReader *pReader, const char* idstr); +int32_t tqRetrieveDataBlock(STqReader *pReader, const char *idstr); int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index d464f64de3..8e914d6dab 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1083,8 +1083,10 @@ typedef struct { int32_t vLen; } SIdxCursor; -int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { - int32_t ret = 0; +int32_t metaFilterCreateTime(void *meta, void *arg, SArray *pUids) { + SMeta *pMeta = meta; + SMetaFltParam *param = arg; + int32_t ret = 0; SIdxCursor *pCursor = NULL; pCursor = (SIdxCursor *)taosMemoryCalloc(1, sizeof(SIdxCursor)); @@ -1141,9 +1143,11 @@ END: return ret; } -int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { - int32_t ret = 0; - char *buf = NULL; +int32_t metaFilterTableName(void *meta, void *arg, SArray *pUids) { + SMeta *pMeta = meta; + SMetaFltParam *param = arg; + int32_t ret = 0; + char *buf = NULL; STagIdxKey *pKey = NULL; int32_t nKey = 0; @@ -1206,9 +1210,11 @@ END: return ret; } -int32_t metaFilterTtl(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { - int32_t ret = 0; - char *buf = NULL; +int32_t metaFilterTtl(void *meta, void *arg, SArray *pUids) { + SMeta *pMeta = meta; + SMetaFltParam *param = arg; + int32_t ret = 0; + char *buf = NULL; STtlIdxKey *pKey = NULL; int32_t nKey = 0; @@ -1235,7 +1241,10 @@ END: // impl later return 0; } -int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { +int32_t metaFilterTableIds(void *meta, void *arg, SArray *pUids) { + SMeta *pMeta = meta; + SMetaFltParam *param = arg; + SMetaEntry oStbEntry = {0}; int32_t ret = -1; char *buf = NULL; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index fa46715c22..82eeb4b1bd 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -435,7 +435,6 @@ static void genTagFilterDigest(const SNode* pTagCond, T_MD5_CTX* pContext) { taosMemoryFree(payload); } - static void genTbGroupDigest(const SNode* pGroup, uint8_t* filterDigest, T_MD5_CTX* pContext) { char* payload = NULL; int32_t len = 0; @@ -453,8 +452,8 @@ static void genTbGroupDigest(const SNode* pGroup, uint8_t* filterDigest, T_MD5_C taosMemoryFree(payload); } - -int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableListInfo* pTableListInfo, uint8_t *digest) { +int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableListInfo* pTableListInfo, + uint8_t* digest) { int32_t code = TSDB_CODE_SUCCESS; SArray* pBlockList = NULL; SSDataBlock* pResBlock = NULL; @@ -492,18 +491,19 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis if (tsTagFilterCache) { SNodeListNode* listNode = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); listNode->pNodeList = group; - genTbGroupDigest((SNode *)listNode, digest, &context); + genTbGroupDigest((SNode*)listNode, digest, &context); nodesFree(listNode); - + metaGetCachedTbGroup(metaHandle, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), &tableList); if (tableList) { taosArrayDestroy(pTableListInfo->pTableList); pTableListInfo->pTableList = tableList; - qDebug("retrieve tb group list from cache, numOfTables:%d", (int32_t)taosArrayGetSize(pTableListInfo->pTableList)); + qDebug("retrieve tb group list from cache, numOfTables:%d", + (int32_t)taosArrayGetSize(pTableListInfo->pTableList)); goto end; } } - + pUidTagList = taosArrayInit(8, sizeof(STUidTagInfo)); for (int32_t i = 0; i < rows; ++i) { STableKeyInfo* pkeyInfo = taosArrayGet(pTableListInfo->pTableList, i); @@ -632,9 +632,10 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis if (tsTagFilterCache) { tableList = taosArrayDup(pTableListInfo->pTableList, NULL); - metaPutTbGroupToCache(metaHandle, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), tableList, taosArrayGetSize(tableList) * sizeof(STableKeyInfo)); + metaPutTbGroupToCache(metaHandle, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), tableList, + taosArrayGetSize(tableList) * sizeof(STableKeyInfo)); } - + // int64_t st2 = taosGetTimestampUs(); // qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1); @@ -839,7 +840,6 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidLis return -1; } - static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* metaHandle) { SSDataBlock* pResBlock = createDataBlock(); @@ -1096,8 +1096,11 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, // failed to find the result in the cache, let try to calculate the results if (pTagIndexCond) { void* pIndex = tsdbGetIvtIdx(metaHandle); - SIndexMetaArg metaArg = { - .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = pIndex, .suid = pScanNode->uid}; + SIndexMetaArg metaArg = {.metaEx = metaHandle, + .idx = tsdbGetIdx(metaHandle), + .ivtIdx = pIndex, + .suid = pScanNode->uid, + .metaFilterFunc = metaFilterTableIds}; status = SFLT_NOT_INDEX; code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status); @@ -1514,7 +1517,7 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu return TSDB_CODE_OUT_OF_MEMORY; } - SHashObj *pSelectFuncs = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + SHashObj* pSelectFuncs = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); for (int32_t i = 0; i < numOfOutput; ++i) { const char* pName = pCtx[i].pExpr->pExpr->_function.functionName; if ((strcmp(pName, "_select_value") == 0) || (strcmp(pName, "_group_key") == 0)) { @@ -1841,9 +1844,7 @@ uint64_t tableListGetSize(const STableListInfo* pTableList) { return taosArrayGetSize(pTableList->pTableList); } -uint64_t tableListGetSuid(const STableListInfo* pTableList) { - return pTableList->idInfo.suid; -} +uint64_t tableListGetSuid(const STableListInfo* pTableList) { return pTableList->idInfo.suid; } STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index) { if (taosArrayGetSize(pTableList->pTableList) == 0) { @@ -2033,8 +2034,8 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) { return TDB_CODE_SUCCESS; } -int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SScanPhysiNode* pScanNode, SNodeList* group, - bool groupSort, uint8_t *digest) { +int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SScanPhysiNode* pScanNode, + SNodeList* group, bool groupSort, uint8_t* digest) { int32_t code = TSDB_CODE_SUCCESS; bool groupByTbname = groupbyTbname(group); @@ -2086,7 +2087,8 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags } uint8_t digest[17] = {0}; - int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo, digest, idStr); + int32_t code = + getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo, digest, idStr); if (code != TSDB_CODE_SUCCESS) { qError("failed to getTableList, code: %s", tstrerror(code)); return code; diff --git a/source/libs/index/CMakeLists.txt b/source/libs/index/CMakeLists.txt index 0c2ce37c40..6f3f48610c 100644 --- a/source/libs/index/CMakeLists.txt +++ b/source/libs/index/CMakeLists.txt @@ -12,7 +12,6 @@ target_link_libraries( PUBLIC os PUBLIC util PUBLIC common - PUBLIC vnode PUBLIC nodes PUBLIC scalar PUBLIC function diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 02ed0d2d05..fc064b762f 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "filter.h" #include "index.h" #include "indexComm.h" #include "indexInt.h" @@ -20,7 +21,6 @@ #include "querynodes.h" #include "scalar.h" #include "tdatablock.h" -#include "vnode.h" // clang-format off #define SIF_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) @@ -659,7 +659,7 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP } else { if (sifSetFltParam(left, right, &typedata, ¶m) != 0) return -1; } - ret = metaFilterTableIds(arg->metaEx, ¶m, output->result); + ret = arg->metaFilterFunc(arg->metaEx, ¶m, output->result); } return ret; } From c9140e547e652e7e241abeee7b5fc175bcbeeb4c Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 23 May 2023 14:08:20 +0800 Subject: [PATCH 13/97] fix: fix bugs of wrongly use sfltscloperator --- source/libs/scalar/src/filter.c | 76 ++++++++++++++++----------------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 4a9ebb16c1..3b61e3a97a 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1841,9 +1841,9 @@ void filterFreeInfo(SFilterInfo *info) { return; } - for (int32_t i = 0 ; i < taosArrayGetSize(info->sclCtx.fltSclRange); ++i) { - SFltSclColumnRange* colRange = taosArrayGet(info->sclCtx.fltSclRange, i); - nodesDestroyNode((SNode*)colRange->colNode); + for (int32_t i = 0; i < taosArrayGetSize(info->sclCtx.fltSclRange); ++i) { + SFltSclColumnRange *colRange = taosArrayGet(info->sclCtx.fltSclRange, i); + nodesDestroyNode((SNode *)colRange->colNode); taosArrayDestroy(colRange->points); } taosArrayDestroy(info->sclCtx.fltSclRange); @@ -3431,7 +3431,6 @@ _return: return code; } - // compare ranges, null < min < val < max. null=null, min=min, max=max typedef enum { FLT_SCL_DATUM_KIND_NULL, @@ -3447,7 +3446,7 @@ typedef enum { typedef struct { SFltSclDatumKind kind; union { - int64_t i; // for int64, uint64 and double and bool (1 true, 0 false) + int64_t i; // for int64, uint64 and double and bool (1 true, 0 false) uint64_t u; double d; uint8_t *pData; // for varchar, nchar @@ -3627,7 +3626,6 @@ typedef struct { EOperatorType type; } SFltSclOperator; - SFltSclColumnRange *fltSclGetOrCreateColumnRange(SColumnNode *colNode, SArray *colRangeList) { for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) { SFltSclColumnRange *colRange = taosArrayGet(colRangeList, i); @@ -3635,7 +3633,7 @@ SFltSclColumnRange *fltSclGetOrCreateColumnRange(SColumnNode *colNode, SArray *c return colRange; } } - SColumnNode* pColumnNode = (SColumnNode*)nodesCloneNode((SNode*)colNode); + SColumnNode *pColumnNode = (SColumnNode *)nodesCloneNode((SNode *)colNode); SFltSclColumnRange newColRange = {.colNode = pColumnNode, .points = taosArrayInit(4, sizeof(SFltSclPoint))}; taosArrayPush(colRangeList, &newColRange); return taosArrayGetLast(colRangeList); @@ -3690,7 +3688,7 @@ int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) return TSDB_CODE_SUCCESS; } -int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum* datum, uint8_t type, int64_t val) { +int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum *datum, uint8_t type, int64_t val) { switch (type) { case TSDB_DATA_TYPE_BOOL: { datum->kind = FLT_SCL_DATUM_KIND_INT64; @@ -3702,13 +3700,13 @@ int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum* datum, uint8_t type, int case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UBIGINT: { datum->kind = FLT_SCL_DATUM_KIND_UINT64; - datum->u = *(uint64_t*)&val; + datum->u = *(uint64_t *)&val; break; } case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: { datum->kind = FLT_SCL_DATUM_KIND_FLOAT64; - datum->d = *(double*)&val; + datum->d = *(double *)&val; break; } // TODO:varchar/nchar/json @@ -3718,11 +3716,11 @@ int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum* datum, uint8_t type, int } } - return TSDB_CODE_SUCCESS; } -int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange* colRange, SColumnDataAgg* pAgg, int32_t numOfRows, SArray* points) { +int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange *colRange, SColumnDataAgg *pAgg, int32_t numOfRows, + SArray *points) { if (pAgg->numOfNull == numOfRows) { SFltSclDatum datum = {.kind = FLT_SCL_DATUM_KIND_NULL}; SFltSclPoint startPt = {.start = true, .excl = false, .val = datum}; @@ -3744,21 +3742,21 @@ int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange* colRange, SColumnDataAg bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t numOfCols, int32_t numOfRows) { if (info->scalarMode) { - SArray* colRanges = info->sclCtx.fltSclRange; + SArray *colRanges = info->sclCtx.fltSclRange; for (int32_t i = 0; i < taosArrayGetSize(colRanges); ++i) { - SFltSclColumnRange* colRange = taosArrayGet(colRanges, i); - bool foundCol = false; - int32_t j = 0; + SFltSclColumnRange *colRange = taosArrayGet(colRanges, i); + bool foundCol = false; + int32_t j = 0; for (; j < numOfCols; ++j) { if (pDataStatis[j] != NULL && pDataStatis[j]->colId == colRange->colNode->colId) { foundCol = true; } } if (foundCol) { - SColumnDataAgg* pAgg = pDataStatis[j]; - SArray* points = taosArrayInit(2, sizeof(SFltSclPoint)); + SColumnDataAgg *pAgg = pDataStatis[j]; + SArray *points = taosArrayInit(2, sizeof(SFltSclPoint)); fltSclBuildRangeFromBlockSma(colRange, pAgg, numOfRows, points); - SArray* merged = taosArrayInit(8, sizeof(SFltSclPoint)); + SArray *merged = taosArrayInit(8, sizeof(SFltSclPoint)); fltSclIntersect(points, colRange->points, merged); bool isIntersect = taosArrayGetSize(merged) != 0; taosArrayDestroy(merged); @@ -4381,19 +4379,18 @@ int32_t fltSclProcessCNF(SArray *sclOpListCNF, SArray *colRangeList) { taosArrayDestroy(colRange->points); colRange->points = points; } - } return TSDB_CODE_SUCCESS; } -static int32_t fltSclCollectOperatorFromNode(SNode* pNode, SArray* sclOpList) { +static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) { if (nodeType(pNode) != QUERY_NODE_OPERATOR) { return TSDB_CODE_SUCCESS; } - SOperatorNode* pOper = (SOperatorNode*)pNode; - //TODO: left value node, right column node - //TODO: datatype - //TODO: operator + SOperatorNode *pOper = (SOperatorNode *)pNode; + // TODO: left value node, right column node + // TODO: datatype + // TODO: operator if (pOper->pLeft == NULL || pOper->pRight == NULL) { return TSDB_CODE_SUCCESS; } @@ -4401,31 +4398,31 @@ static int32_t fltSclCollectOperatorFromNode(SNode* pNode, SArray* sclOpList) { (pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL || pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL || pOper->opType == OP_TYPE_NOT_EQUAL || pOper->opType == OP_TYPE_EQUAL)) { - SValueNode* valNode = (SValueNode*)pOper->pRight; + SValueNode *valNode = (SValueNode *)pOper->pRight; if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { - SNode *p = nodesCloneNode(pNode); - taosArrayPush(sclOpList, &p); + SFltSclOperator sclOp = { + .colNode = nodesCloneNode(pOper->pLeft), .valNode = nodesCloneNode(pOper->pRight), .type = pOper->opType}; + taosArrayPush(sclOpList, &sclOp); } } return TSDB_CODE_SUCCESS; } -static int32_t fltSclCollectOperatorsFromLogicCond(SNode* pNode, SArray* sclOpList) { +static int32_t fltSclCollectOperatorsFromLogicCond(SNode *pNode, SArray *sclOpList) { if (nodeType(pNode) != QUERY_NODE_LOGIC_CONDITION) { return TSDB_CODE_SUCCESS; } - SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pNode; - //TODO: support LOGIC_COND_TYPE_OR + SLogicConditionNode *pLogicCond = (SLogicConditionNode *)pNode; + // TODO: support LOGIC_COND_TYPE_OR if (pLogicCond->condType != LOGIC_COND_TYPE_AND) { return TSDB_CODE_SUCCESS; } - SNode* pExpr = NULL; - FOREACH(pExpr, pLogicCond->pParameterList) { fltSclCollectOperatorFromNode(pExpr, sclOpList); - } + SNode *pExpr = NULL; + FOREACH(pExpr, pLogicCond->pParameterList) { fltSclCollectOperatorFromNode(pExpr, sclOpList); } return TSDB_CODE_SUCCESS; } -static int32_t fltSclCollectOperators(SNode *pNode, SArray* sclOpList) { +static int32_t fltSclCollectOperators(SNode *pNode, SArray *sclOpList) { if (nodeType(pNode) == QUERY_NODE_OPERATOR) { fltSclCollectOperatorFromNode(pNode, sclOpList); } else if (nodeType(pNode) == QUERY_NODE_LOGIC_CONDITION) { @@ -4435,15 +4432,16 @@ static int32_t fltSclCollectOperators(SNode *pNode, SArray* sclOpList) { } int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) { - SArray* sclOpList = taosArrayInit(16, POINTER_BYTES); + SArray *sclOpList = taosArrayInit(16, POINTER_BYTES); fltSclCollectOperators(*pNode, sclOpList); - SArray* colRangeList = taosArrayInit(16, sizeof(SFltSclColumnRange)); + SArray *colRangeList = taosArrayInit(16, sizeof(SFltSclColumnRange)); fltSclProcessCNF(sclOpList, colRangeList); pInfo->sclCtx.fltSclRange = colRangeList; for (int32_t i = 0; i < taosArrayGetSize(sclOpList); ++i) { - SNode* p = taosArrayGetP(sclOpList, i); - nodesDestroyNode(p); + SFltSclOperator *sclOp = taosArrayGet(sclOpList, i); + nodesDestroyNode(sclOp->colNode); + nodesDestroyNode(sclOp->valNode); } taosArrayDestroy(sclOpList); return TSDB_CODE_SUCCESS; From 99620d434da9fdf995a923380416e5138ce5a32a Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 23 May 2023 14:11:52 +0800 Subject: [PATCH 14/97] fix: fix bugs --- source/libs/scalar/src/filter.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 3b61e3a97a..c1de62a407 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4401,7 +4401,7 @@ static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) { SValueNode *valNode = (SValueNode *)pOper->pRight; if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { SFltSclOperator sclOp = { - .colNode = nodesCloneNode(pOper->pLeft), .valNode = nodesCloneNode(pOper->pRight), .type = pOper->opType}; + .colNode = (SColumnNode*)nodesCloneNode(pOper->pLeft), .valNode = (SValueNode*)nodesCloneNode(pOper->pRight), .type = pOper->opType}; taosArrayPush(sclOpList, &sclOp); } } @@ -4440,8 +4440,8 @@ int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) for (int32_t i = 0; i < taosArrayGetSize(sclOpList); ++i) { SFltSclOperator *sclOp = taosArrayGet(sclOpList, i); - nodesDestroyNode(sclOp->colNode); - nodesDestroyNode(sclOp->valNode); + nodesDestroyNode((SNode*)sclOp->colNode); + nodesDestroyNode((SNode*)sclOp->valNode); } taosArrayDestroy(sclOpList); return TSDB_CODE_SUCCESS; From 857cf3d269952a1f4a3010f01f67c2a8d77b9af4 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 23 May 2023 14:29:33 +0800 Subject: [PATCH 15/97] fix: fix bugs --- source/libs/scalar/src/filter.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index c1de62a407..2e0750359f 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3750,6 +3750,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t for (; j < numOfCols; ++j) { if (pDataStatis[j] != NULL && pDataStatis[j]->colId == colRange->colNode->colId) { foundCol = true; + break; } } if (foundCol) { From 22616d235ae120362973ab67cdbe3da0fabf2a0b Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 23 May 2023 16:36:27 +0800 Subject: [PATCH 16/97] fix: fix minior bugs --- source/libs/scalar/src/filter.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 2e0750359f..e7f88908e5 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3598,7 +3598,7 @@ int32_t fltSclMerge(SArray *pts1, SArray *pts2, bool isUnion, SArray *merged) { int32_t countRequired = (isUnion) ? 1 : 2; int32_t count = 0; for (int32_t i = 0; i < taosArrayGetSize(all); ++i) { - SFltSclPoint *pt = taosArrayGet(pts1, i); + SFltSclPoint *pt = taosArrayGet(all, i); if (pt->start) { ++count; if (count == countRequired) { @@ -4402,7 +4402,9 @@ static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) { SValueNode *valNode = (SValueNode *)pOper->pRight; if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { SFltSclOperator sclOp = { - .colNode = (SColumnNode*)nodesCloneNode(pOper->pLeft), .valNode = (SValueNode*)nodesCloneNode(pOper->pRight), .type = pOper->opType}; + .colNode = (SColumnNode*)nodesCloneNode(pOper->pLeft), + .valNode = (SValueNode*)nodesCloneNode(pOper->pRight), + .type = pOper->opType}; taosArrayPush(sclOpList, &sclOp); } } @@ -4433,7 +4435,7 @@ static int32_t fltSclCollectOperators(SNode *pNode, SArray *sclOpList) { } int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) { - SArray *sclOpList = taosArrayInit(16, POINTER_BYTES); + SArray *sclOpList = taosArrayInit(16, sizeof(SFltSclOperator)); fltSclCollectOperators(*pNode, sclOpList); SArray *colRangeList = taosArrayInit(16, sizeof(SFltSclColumnRange)); fltSclProcessCNF(sclOpList, colRangeList); From e67b532d8a4f8ebaf47b34381e1279f1379a41f3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 May 2023 18:29:23 +0800 Subject: [PATCH 17/97] refactor: refactor the module dependency. --- include/common/tcommon.h | 7 + include/libs/executor/executor.h | 7 +- include/libs/executor/storageapi.h | 576 ++++++++++++++++++ include/libs/index/index.h | 31 + include/libs/stream/streamState.h | 82 ++- include/libs/stream/tstreamFileState.h | 9 +- include/libs/stream/tstreamUpdate.h | 26 +- source/dnode/qnode/CMakeLists.txt | 2 + source/dnode/qnode/src/qnode.c | 1 - source/dnode/snode/CMakeLists.txt | 1 + source/dnode/vnode/inc/vnode.h | 186 +++--- source/dnode/vnode/src/inc/tsdb.h | 3 - source/dnode/vnode/src/meta/metaQuery.c | 12 +- source/dnode/vnode/src/meta/metaSnapshot.c | 30 +- source/dnode/vnode/src/tq/tqRead.c | 25 +- source/libs/executor/CMakeLists.txt | 2 +- source/libs/executor/inc/dataSinkInt.h | 4 +- source/libs/executor/inc/executil.h | 12 +- source/libs/executor/inc/executorInt.h | 51 +- source/libs/executor/inc/operator.h | 2 +- source/libs/executor/inc/querytask.h | 28 +- source/libs/executor/src/aggregateoperator.c | 3 +- source/libs/executor/src/cachescanoperator.c | 22 +- source/libs/executor/src/dataInserter.c | 4 +- source/libs/executor/src/executil.c | 124 ++-- source/libs/executor/src/executor.c | 67 +- source/libs/executor/src/executorInt.c | 141 +---- source/libs/executor/src/filloperator.c | 85 +-- source/libs/executor/src/groupoperator.c | 27 +- source/libs/executor/src/operator.c | 14 +- source/libs/executor/src/projectoperator.c | 1 + source/libs/executor/src/querytask.c | 18 +- source/libs/executor/src/scanoperator.c | 354 +++++------ source/libs/executor/src/sysscanoperator.c | 191 +++--- source/libs/executor/src/timesliceoperator.c | 1 + source/libs/executor/src/timewindowoperator.c | 154 +++-- source/libs/function/CMakeLists.txt | 1 - source/libs/function/src/builtinsimpl.c | 12 +- source/libs/index/CMakeLists.txt | 2 +- source/libs/index/src/indexFilter.c | 5 +- source/libs/scalar/CMakeLists.txt | 1 + source/libs/stream/inc/streamBackendRocksdb.h | 1 - source/libs/stream/src/streamBackendRocksdb.c | 18 +- source/libs/stream/src/tstreamFileState.c | 1 + 44 files changed, 1477 insertions(+), 867 deletions(-) create mode 100644 include/libs/executor/storageapi.h diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 520a8e9c2c..2f93f8c3e3 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -37,6 +37,13 @@ extern "C" { ) // clang-format on +typedef bool (*state_key_cmpr_fn)(void* pKey1, void* pKey2); + +typedef struct STableKeyInfo { + uint64_t uid; + uint64_t groupId; +} STableKeyInfo; + typedef struct SWinKey { uint64_t groupId; TSKEY ts; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 1fb00e743f..7f86b8f5e2 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -23,6 +23,7 @@ extern "C" { #include "query.h" #include "tcommon.h" #include "tmsgcb.h" +#include "storageapi.h" typedef void* qTaskInfo_t; typedef void* DataSinkHandle; @@ -51,10 +52,10 @@ typedef struct { bool initTableReader; bool initTqReader; int32_t numOfVgroups; + void* sContext; // SSnapContext* - void* sContext; // SSnapContext* - - void* pStateBackend; + void* pStateBackend; + struct SStorageAPI api; } SReadHandle; // in queue mode, data streams are seperated by msg diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h new file mode 100644 index 0000000000..9b53f11ad9 --- /dev/null +++ b/include/libs/executor/storageapi.h @@ -0,0 +1,576 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_STORAGEAPI_H +#define TDENGINE_STORAGEAPI_H + +#include "tsimplehash.h" +#include "tscalablebf.h" +//#include "tdb.h" +#include "taosdef.h" +#include "tmsg.h" +#include "tcommon.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define TIMEWINDOW_RANGE_CONTAINED 1 +#define TIMEWINDOW_RANGE_EXTERNAL 2 + +#define CACHESCAN_RETRIEVE_TYPE_ALL 0x1 +#define CACHESCAN_RETRIEVE_TYPE_SINGLE 0x2 +#define CACHESCAN_RETRIEVE_LAST_ROW 0x4 +#define CACHESCAN_RETRIEVE_LAST 0x8 + +typedef struct SMeta SMeta; + +typedef struct SMetaEntry { + int64_t version; + int8_t type; + int8_t flags; // TODO: need refactor? + tb_uid_t uid; + char * name; + union { + struct { + SSchemaWrapper schemaRow; + SSchemaWrapper schemaTag; + SRSmaParam rsmaParam; + } stbEntry; + struct { + int64_t ctime; + int32_t ttlDays; + int32_t commentLen; + char * comment; + tb_uid_t suid; + uint8_t *pTags; + } ctbEntry; + struct { + int64_t ctime; + int32_t ttlDays; + int32_t commentLen; + char * comment; + int32_t ncid; // next column id + SSchemaWrapper schemaRow; + } ntbEntry; + struct { + STSma *tsma; + } smaEntry; + }; + + uint8_t *pBuf; +} SMetaEntry; + +// int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables); +// int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, +// SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr); +// int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, +// SArray *pTableUids); +// void *tsdbCacherowsReaderClose(void *pReader); +// int32_t tsdbGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); + +// int32_t tsdbReaderOpen(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, +// SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly, SHashObj** pIgnoreTables); +// int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num); +// void tsdbReaderSetId(STsdbReader *pReader, const char *idstr); +// void tsdbReaderClose(STsdbReader *pReader); +// int32_t tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext); +// int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave); +// void tsdbReleaseDataBlock(STsdbReader *pReader); +// SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); +// int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); +// int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); +// int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle); +// void *tsdbGetIdx(void *pMeta); +// void *tsdbGetIvtIdx(void *pMeta); +// uint64_t tsdbGetReaderMaxVersion(STsdbReader *pReader); +// void tsdbReaderSetCloseFlag(STsdbReader *pReader); +// int64_t tsdbGetLastTimestamp(void* pVnode, void* pTableList, int32_t numOfTables, const char* pIdStr); + +// int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list); +// int32_t vnodeGetCtbIdListByFilter(void *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg); +// int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list); +// void *vnodeGetIdx(void *pVnode); +// void *vnodeGetIvtIdx(void *pVnode); + +// void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); +// void metaReaderReleaseLock(SMetaReader *pReader); +// void metaReaderClear(SMetaReader *pReader); +// int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); +// int32_t metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); +// int metaGetTableEntryByName(SMetaReader *pReader, const char *name); +// int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList); +// int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList); +// const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); +// int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); +// +// int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName); +// int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid); +// int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); +// bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid); +// int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList, +// bool *acquired); +// int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, +// int32_t payloadLen, double selectivityRatio); +// int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid); +// tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); +// int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); +// int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, +// int32_t payloadLen); + +// tq +typedef struct SMetaTableInfo { + int64_t suid; + int64_t uid; + SSchemaWrapper *schema; + char tbName[TSDB_TABLE_NAME_LEN]; +} SMetaTableInfo; + +typedef struct SSnapContext { + SMeta * pMeta; + int64_t snapVersion; + struct TBC *pCur; + int64_t suid; + int8_t subType; + SHashObj * idVersion; + SHashObj * suidInfo; + SArray * idList; + int32_t index; + bool withMeta; + bool queryMeta; // true-get meta, false-get data +} SSnapContext; + +typedef struct { + int64_t uid; + int64_t ctbNum; +} SMetaStbStats; + +// void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList); +// int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); +// int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList); +// int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); +// bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid); +// bool tqCurrentBlockConsumed(const STqReader* pReader); +// +// int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); +// bool tqNextBlockInWal(STqReader* pReader, const char* idstr); +// bool tqNextBlockImpl(STqReader *pReader, const char* idstr); + +// int32_t getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); +// SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx); +// int32_t setForSnapShot(SSnapContext *ctx, int64_t uid); +// int32_t destroySnapContext(SSnapContext *ctx); + +// SMTbCursor *metaOpenTbCursor(SMeta *pMeta); +// void metaCloseTbCursor(SMTbCursor *pTbCur); +// int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); +// int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); + +#define META_READER_NOLOCK 0x1 + +/*-------------------------------------------------new api format---------------------------------------------------*/ + +// typedef int32_t (*__store_reader_(STsdbReader *pReader, const void *pTableList, int32_t num); +// typedef void (*tsdbReaderSetId(STsdbReader *pReader, const char *idstr); +// typedef void (*tsdbReaderClose(STsdbReader *pReader); +// typedef int32_t (*tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext); +// typedef int32_t (*tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave); +// typedef void (*tsdbReleaseDataBlock(STsdbReader *pReader); +// typedef SSDataBlock * (*tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); +// typedef int32_t (*tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); +// typedef int32_t (*tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); +// typedef int64_t (*tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle); +// typedef void * (*tsdbGetIdx(void *pMeta); +// typedef void * (*tsdbGetIvtIdx(void *pMeta); +// typedef uint64_t (*tsdbGetReaderMaxVersion(STsdbReader *pReader); +// typedef void (*tsdbReaderSetCloseFlag(STsdbReader *pReader); +// typedef int64_t (*tsdbGetLastTimestamp(void* pVnode, void* pTableList, int32_t numOfTables, const char* pIdStr); + +typedef int32_t (*__store_reader_open_fn_t)(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, + int32_t numOfTables, SSDataBlock *pResBlock, void **ppReader, + const char *idstr, bool countOnly, SHashObj **pIgnoreTables); + +typedef struct SStoreDataReaderFn { + __store_reader_open_fn_t storeReaderOpen; + void (*storeReaderClose)(); + void (*setReaderId)(void *pReader, const char *pId); + void (*storeReaderSetTableList)(); + int32_t (*storeReaderNextDataBlock)(); + int32_t (*storeReaderRetrieveBlockSMA)(); + + SSDataBlock *(*storeReaderRetrieveDataBlock)(); + void (*storeReaderReleaseDataBlock)(); + + void (*storeReaderResetStatus)(); + void (*storeReaderGetDataBlockDistInfo)(); + void (*storeReaderGetNumOfInMemRows)(); + void (*storeReaderNotifyClosing)(); +} SStoreDataReaderFn; + +/** + * int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables); +int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, + SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr); +int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, + SArray *pTableUids); +void *tsdbCacherowsReaderClose(void *pReader); + */ +typedef struct SStoreCachedDataReaderFn { + int32_t (*openReader)(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, + SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr); + void *(*closeReader)(void *pReader); + int32_t (*retrieveRows)(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, + SArray *pTableUidList); + void (*reuseReader)(void *pReader, void *pTableIdList, int32_t numOfTables); +} SStoreCachedDataReaderFn; + +/*------------------------------------------------------------------------------------------------------------------*/ +/* + * +void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList); +int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); +int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList); +int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); +bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid); +bool tqCurrentBlockConsumed(const STqReader* pReader); + +int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); +bool tqNextBlockInWal(STqReader* pReader, const char* idstr); +bool tqNextBlockImpl(STqReader *pReader, const char* idstr); + + int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char* idstr); +STqReader *tqReaderOpen(void *pVnode); +void tqCloseReader(STqReader *); + +int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); +bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); +SWalReader* tqGetWalReader(STqReader* pReader); +int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); +*/ +// todo rename +typedef struct SStoreTqReaderFn { + void *(*tqReaderOpen)(); + void (*tqReaderClose)(); + + int32_t (*tqReaderSeek)(); + int32_t (*tqRetrieveBlock)(); + bool (*tqReaderNextBlockInWal)(); + bool (*tqNextBlockImpl)(); // todo remove it + + void (*tqReaderSetColIdList)(); + int32_t (*tqReaderSetTargetTableList)(); + + int32_t (*tqReaderAddTables)(); + int32_t (*tqReaderRemoveTables)(); + + bool (*tqReaderIsQueriedTable)(); + bool (*tqReaderCurrentBlockConsumed)(); + + struct SWalReader *(*tqReaderGetWalReader)(); // todo remove it + void (*tqReaderRetrieveTaosXBlock)(); // todo remove it + + int32_t (*tqReaderSetSubmitMsg)(); // todo remove it + void (*tqReaderNextBlockFilterOut)(); +} SStoreTqReaderFn; + +typedef struct SStoreSnapshotFn { + /* + int32_t getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); + SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx); + int32_t setForSnapShot(SSnapContext *ctx, int64_t uid); + int32_t destroySnapContext(SSnapContext *ctx); + */ + int32_t (*storeCreateSnapshot)(); + void (*storeDestroySnapshot)(); + SMetaTableInfo (*storeSSGetTableInfo)(); + int32_t (*storeSSGetMetaInfo)(); +} SStoreSnapshotFn; + +/** +void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); +void metaReaderReleaseLock(SMetaReader *pReader); +void metaReaderClear(SMetaReader *pReader); +int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); +int32_t metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); +int metaGetTableEntryByName(SMetaReader *pReader, const char *name); +int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList); +const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); +int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); + +int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName); +int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid); +int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); +bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid); +int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList, + bool *acquired); +int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, + int32_t payloadLen, double selectivityRatio); +int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid); +tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); +int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); +int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, + int32_t payloadLen); + */ +typedef struct SMetaReader { + int32_t flags; + void * pMeta; + SDecoder coder; + SMetaEntry me; + void * pBuf; + int32_t szBuf; + struct SStorageAPI *storageAPI; +} SMetaReader; + +typedef struct SStoreMetaReaderFn { + void (*initReader)(void *pReader, void *pMeta, int32_t flags); + void *(*clearReader)(); + + void (*readerReleaseLock)(); + + int32_t (*getTableEntryByUid)(); + int32_t (*getTableEntryByName)(); + int32_t (*readerGetEntryGetUidCache)(SMetaReader *pReader, tb_uid_t uid); +} SStoreMetaReaderFn; + +typedef struct SStoreMetaFn { + /* +SMTbCursor *metaOpenTbCursor(SMeta *pMeta); +void metaCloseTbCursor(SMTbCursor *pTbCur); +int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); +int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); + */ + void *(*openMetaCursor)(); + void (*closeMetaCursor)(); + int32_t (*cursorNext)(); + void (*cursorPrev)(); + + int32_t (*getTableTags)(void *pVnode, uint64_t suid, SArray *uidList); + int32_t (*getTableTagsByUid)(); + const char *(*extractTagVal)(const void *tag, int16_t type, STagVal *tagVal); // todo remove it + + int32_t (*getTableUidByName)(void *pVnode, char *tbName, uint64_t *uid); + int32_t (*getTableTypeByName)(void *pVnode, char *tbName, ETableType *tbType); + int32_t (*getTableNameByUid)(void *pVnode, uint64_t uid, char *tbName); + bool (*isTableExisted)(void *pVnode, uint64_t uid); + + /** + * int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, + int32_t payloadLen, double selectivityRatio); +int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid); +tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); +int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); +int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, + int32_t payloadLen); + */ + void (*getCachedTableList)(); + void (*putTableListIntoCache)(); + + /** + * + */ + void *(*storeGetIndexInfo)(); + void *(*storeGetInvertIndex)(); + void (*storeGetChildTableList)(); // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] + int32_t (*storeGetTableList)(); // vnodeGetStbIdList & vnodeGetAllTableList + void *storeGetVersionRange; + void *storeGetLastTimestamp; + + int32_t (*getTableSchema)(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); // tsdbGetTableSchema + + // db name, vgId, numOfTables, numOfSTables + void (*storeGetNumOfChildTables)(); // int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); + void (*storeGetBasicInfo)(); // vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) & metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta); + + int64_t (*getNumOfRowsInMem)(); + /** +int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list); +int32_t vnodeGetCtbIdListByFilter(void *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg); +int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list); + */ +} SStoreMetaFn; + +typedef struct STdbState { + void* rocksdb; + void** pHandle; + void* writeOpts; + void* readOpts; + void** cfOpts; + void* dbOpt; + struct SStreamTask* pOwner; + void* param; + void* env; + SListNode* pComparNode; + void* pBackendHandle; + char idstr[64]; + void* compactFactory; + + void* db; + void* pStateDb; + void* pFuncStateDb; + void* pFillStateDb; // todo refactor + void* pSessionStateDb; + void* pParNameDb; + void* pParTagDb; + void* txn; +} STdbState; + +// incremental state storage +typedef struct { + STdbState* pTdbState; + struct SStreamFileState* pFileState; + int32_t number; + SSHashObj* parNameMap; + int64_t checkPointId; + int32_t taskId; + int64_t streamId; +} SStreamState; + +typedef struct SUpdateInfo { + SArray *pTsBuckets; + uint64_t numBuckets; + SArray *pTsSBFs; + uint64_t numSBFs; + int64_t interval; + int64_t watermark; + TSKEY minTS; + SScalableBf *pCloseWinSBF; + SHashObj *pMap; + uint64_t maxDataVersion; +} SUpdateInfo; + +typedef struct { + void* iter; + void* snapshot; + void* readOpt; + void* db; +// rocksdb_iterator_t* iter; +// rocksdb_snapshot_t* snapshot; +// rocksdb_readoptions_t* readOpt; +// rocksdb_t* db; + + void* pCur; + int64_t number; +} SStreamStateCur; + +typedef TSKEY (*GetTsFun)(void*); + +typedef struct SStateStore { + int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname); + int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal); + + int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); + int32_t (*streamStateReleaseBuf)(SStreamState* pState, const SWinKey* key, void* pVal); + void (*streamStateFreeVal)(void* val); + + int32_t (*streamStatePut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); + int32_t (*streamStateGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); + bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key); + int32_t (*streamStateGetByPos)(SStreamState* pState, void* pos, void** pVal); + int32_t (*streamStateDel)(SStreamState* pState, const SWinKey* key); + int32_t (*streamStateClear)(SStreamState* pState); + void (*streamStateSetNumber)(SStreamState* pState, int32_t number); + int32_t (*streamStateSaveInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen); + int32_t (*streamStateGetInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen); + + int32_t (*streamStateFillPut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); + int32_t (*streamStateFillGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); + int32_t (*streamStateFillDel)(SStreamState* pState, const SWinKey* key); + + int32_t (*streamStateCurNext)(SStreamState* pState, void* pCur); + int32_t (*streamStateCurPrev)(SStreamState* pState, void* pCur); + + void* (*streamStateGetAndCheckCur)(SStreamState* pState, SWinKey* key); + void* (*streamStateSeekKeyNext)(SStreamState* pState, const SWinKey* key); + void* (*streamStateFillSeekKeyNext)(SStreamState* pState, const SWinKey* key); + void* (*streamStateFillSeekKeyPrev)(SStreamState* pState, const SWinKey* key); + void (*streamStateFreeCur)(void* pCur); + + int32_t (*streamStateGetGroupKVByCur)(void* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); + int32_t (*streamStateGetKVByCur)(void* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); + + int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen); + int32_t (*streamStateSessionPut)(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen); + int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); + int32_t (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key); + int32_t (*streamStateSessionClear)(SStreamState* pState); + int32_t (*streamStateSessionGetKVByCur)(void* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); + int32_t (*streamStateStateAddIfNotExist)(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); + int32_t (*streamStateSessionGetKeyByRange)(void* pState, const SSessionKey* range, SSessionKey* curKey); + + void* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark); + TSKEY (*updateInfoFillBlockData)(void *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol); + bool (*updateInfoIsUpdated)(void *pInfo, uint64_t tableId, TSKEY ts); + bool (*updateInfoIsTableInserted)(void *pInfo, int64_t tbUid); + void (*updateInfoDestroy)(void *pInfo); + + SUpdateInfo* (*updateInfoInitP)(SInterval *pInterval, int64_t watermark); + void (*updateInfoAddCloseWindowSBF)(void *pInfo); + void (*updateInfoDestoryColseWinSBF)(void *pInfo); + int32_t (*updateInfoSerialize)(void *buf, int32_t bufLen, const void *pInfo); + int32_t (*updateInfoDeserialize)(void *buf, int32_t bufLen, void *pInfo); + + void* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key); + void* (*streamStateSessionSeekKeyCurrentPrev)(SStreamState* pState, const SSessionKey* key); + void* (*streamStateSessionSeekKeyCurrentNext)(SStreamState* pState, const SSessionKey* key); + + void* (*streamFileStateInit)(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, GetTsFun fp, + void* pFile, TSKEY delMark); + + void (*streamFileStateDestroy)(void* pFileState); + void (*streamFileStateClear)(void* pFileState); + bool (*needClearDiskBuff)(void* pFileState); + + SStreamState* (*streamStateOpen)(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages); + void (*streamStateClose)(SStreamState* pState, bool remove); + int32_t (*streamStateBegin)(SStreamState* pState); + int32_t (*streamStateCommit)(SStreamState* pState); + void (*streamStateDestroy)(SStreamState* pState, bool remove); + int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark); +} SStateStore; + +typedef struct SStorageAPI { + SStoreMetaFn metaFn; // todo: refactor + SStoreDataReaderFn storeReader; + SStoreMetaReaderFn metaReaderFn; + SStoreCachedDataReaderFn cacheFn; + SStoreSnapshotFn snapshotFn; + SStoreTqReaderFn tqReaderFn; + SStateStore stateStore; +} SStorageAPI; + +typedef struct SMTbCursor { + struct TBC *pDbc; + void * pKey; + void * pVal; + int32_t kLen; + int32_t vLen; + SMetaReader mr; +} SMTbCursor; + +typedef struct SRowBuffPos { + void* pRowBuff; + void* pKey; + bool beFlushed; + bool beUsed; +} SRowBuffPos; + + + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_STORAGEAPI_H diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 0d31ca2f68..93e4f18ae9 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -21,6 +21,7 @@ #include "taoserror.h" #include "tarray.h" #include "tglobal.h" +#include "tsdstorage.h" #ifdef __cplusplus extern "C" { @@ -231,6 +232,36 @@ void indexInit(int32_t threads); */ void indexCleanup(); +/** + * the underlying storage module must implement this API to employ the index functions. + * @param pMeta + * @param param + * @param results + * @return + */ + +typedef struct SMetaFltParam { + uint64_t suid; + int16_t cid; + int16_t type; + void *val; + bool reverse; + bool equal; + int (*filterFunc)(void *a, void *b, int16_t type); +} SMetaFltParam; + +typedef struct SStoreAPI { + int32_t (*metaFilterTableIds)(); + int32_t (*metaFilterCreateTime)(); + int32_t (*metaFilterTableName)(); + int32_t (*metaFilterTtl)(); +} SStoreAPI; + +//int32_t metaFilterTableIds(void *pMeta, SMetaFltParam *param, SArray *results); +//int32_t metaFilterCreateTime(void *pMeta, SMetaFltParam *parm, SArray *pUids); +//int32_t metaFilterTableName(void *pMeta, SMetaFltParam *param, SArray *pUids); +//int32_t metaFilterTtl(void *pMeta, SMetaFltParam *param, SArray *pUids); + #ifdef __cplusplus } #endif diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 1cc61ec072..f9fffef7c2 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -27,47 +27,37 @@ extern "C" { #endif +#include "storageapi.h" + // void* streamBackendInit(const char* path); // void streamBackendCleanup(void* arg); // SListNode* streamBackendAddCompare(void* backend, void* arg); // void streamBackendDelCompare(void* backend, void* arg); -typedef bool (*state_key_cmpr_fn)(void* pKey1, void* pKey2); -typedef struct STdbState { - rocksdb_t* rocksdb; - rocksdb_column_family_handle_t** pHandle; - rocksdb_writeoptions_t* writeOpts; - rocksdb_readoptions_t* readOpts; - rocksdb_options_t** cfOpts; - rocksdb_options_t* dbOpt; - struct SStreamTask* pOwner; - void* param; - void* env; - SListNode* pComparNode; - void* pBackendHandle; - char idstr[64]; - void* compactFactory; - - TDB* db; - TTB* pStateDb; - TTB* pFuncStateDb; - TTB* pFillStateDb; // todo refactor - TTB* pSessionStateDb; - TTB* pParNameDb; - TTB* pParTagDb; - TXN* txn; -} STdbState; - -// incremental state storage -typedef struct { - STdbState* pTdbState; - SStreamFileState* pFileState; - int32_t number; - SSHashObj* parNameMap; - int64_t checkPointId; - int32_t taskId; - int64_t streamId; -} SStreamState; +//typedef struct STdbState { +// rocksdb_t* rocksdb; +// rocksdb_column_family_handle_t** pHandle; +// rocksdb_writeoptions_t* writeOpts; +// rocksdb_readoptions_t* readOpts; +// rocksdb_options_t** cfOpts; +// rocksdb_options_t* dbOpt; +// struct SStreamTask* pOwner; +// void* param; +// void* env; +// SListNode* pComparNode; +// void* pBackendHandle; +// char idstr[64]; +// void* compactFactory; +// +// TDB* db; +// TTB* pStateDb; +// TTB* pFuncStateDb; +// TTB* pFillStateDb; // todo refactor +// TTB* pSessionStateDb; +// TTB* pParNameDb; +// TTB* pParTagDb; +// TXN* txn; +//} STdbState; SStreamState* streamStateOpen(char* path, struct SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages); void streamStateClose(SStreamState* pState, bool remove); @@ -76,15 +66,15 @@ int32_t streamStateCommit(SStreamState* pState); void streamStateDestroy(SStreamState* pState, bool remove); int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark); -typedef struct { - rocksdb_iterator_t* iter; - rocksdb_snapshot_t* snapshot; - rocksdb_readoptions_t* readOpt; - rocksdb_t* db; - - TBC* pCur; - int64_t number; -} SStreamStateCur; +//typedef struct { +// rocksdb_iterator_t* iter; +// rocksdb_snapshot_t* snapshot; +// rocksdb_readoptions_t* readOpt; +// rocksdb_t* db; +// +// TBC* pCur; +// int64_t number; +//} SStreamStateCur; int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen); @@ -119,7 +109,7 @@ int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key); int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal); -void streamFreeVal(void* val); +void streamStateFreeVal(void* val); SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key); SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key); diff --git a/include/libs/stream/tstreamFileState.h b/include/libs/stream/tstreamFileState.h index 7124e2d251..0dbacf6c9f 100644 --- a/include/libs/stream/tstreamFileState.h +++ b/include/libs/stream/tstreamFileState.h @@ -21,23 +21,16 @@ #include "tarray.h" #include "tdef.h" #include "tlist.h" +#include "storageapi.h" #ifdef __cplusplus extern "C" { #endif typedef struct SStreamFileState SStreamFileState; -typedef struct SRowBuffPos { - void* pRowBuff; - void* pKey; - bool beFlushed; - bool beUsed; -} SRowBuffPos; typedef SList SStreamSnapshot; -typedef TSKEY (*GetTsFun)(void*); - SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, GetTsFun fp, void* pFile, TSKEY delMark); void streamFileStateDestroy(SStreamFileState* pFileState); diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h index 4678aa0bd9..bd5a3be8de 100644 --- a/include/libs/stream/tstreamUpdate.h +++ b/include/libs/stream/tstreamUpdate.h @@ -19,7 +19,7 @@ #include "tarray.h" #include "tcommon.h" #include "tmsg.h" -#include "tscalablebf.h" +#include "storageapi.h" #ifdef __cplusplus extern "C" { @@ -30,18 +30,18 @@ typedef struct SUpdateKey { TSKEY ts; } SUpdateKey; -typedef struct SUpdateInfo { - SArray *pTsBuckets; - uint64_t numBuckets; - SArray *pTsSBFs; - uint64_t numSBFs; - int64_t interval; - int64_t watermark; - TSKEY minTS; - SScalableBf *pCloseWinSBF; - SHashObj *pMap; - uint64_t maxDataVersion; -} SUpdateInfo; +//typedef struct SUpdateInfo { +// SArray *pTsBuckets; +// uint64_t numBuckets; +// SArray *pTsSBFs; +// uint64_t numSBFs; +// int64_t interval; +// int64_t watermark; +// TSKEY minTS; +// SScalableBf *pCloseWinSBF; +// SHashObj *pMap; +// uint64_t maxDataVersion; +//} SUpdateInfo; SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark); SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark); diff --git a/source/dnode/qnode/CMakeLists.txt b/source/dnode/qnode/CMakeLists.txt index 5426cd55d3..13429d0cc1 100644 --- a/source/dnode/qnode/CMakeLists.txt +++ b/source/dnode/qnode/CMakeLists.txt @@ -14,4 +14,6 @@ target_link_libraries( PRIVATE qworker PRIVATE qcom PRIVATE executor + PRIVATE tdb + PRIVATE wal ) \ No newline at end of file diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index 5efc714e95..3482355512 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -14,7 +14,6 @@ */ #include "executor.h" -#include "libs/function/function.h" #include "qndInt.h" #include "query.h" #include "qworker.h" diff --git a/source/dnode/snode/CMakeLists.txt b/source/dnode/snode/CMakeLists.txt index 6f1e7f9593..1d8c80c422 100644 --- a/source/dnode/snode/CMakeLists.txt +++ b/source/dnode/snode/CMakeLists.txt @@ -14,4 +14,5 @@ target_link_libraries( PRIVATE util PRIVATE qcom PRIVATE stream + PRIVATE wal ) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 18fa893fa4..8afeb05716 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -33,6 +33,7 @@ #include "trow.h" #include "tdb.h" +#include "storageapi.h" #ifdef __cplusplus extern "C" { @@ -132,28 +133,28 @@ int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, in int64_t metaGetTbNum(SMeta *pMeta); int64_t metaGetNtbNum(SMeta *pMeta); -typedef struct { - int64_t uid; - int64_t ctbNum; -} SMetaStbStats; +//typedef struct { +// int64_t uid; +// int64_t ctbNum; +//} SMetaStbStats; int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); -typedef struct SMetaFltParam { - tb_uid_t suid; - int16_t cid; - int16_t type; - void *val; - bool reverse; - bool equal; - int (*filterFunc)(void *a, void *b, int16_t type); - -} SMetaFltParam; +//typedef struct SMetaFltParam { +// tb_uid_t suid; +// int16_t cid; +// int16_t type; +// void *val; +// bool reverse; +// bool equal; +// int (*filterFunc)(void *a, void *b, int16_t type); +// +//} SMetaFltParam; // TODO, refactor later -int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *results); -int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *parm, SArray *pUids); -int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids); -int32_t metaFilterTtl(SMeta *pMeta, SMetaFltParam *param, SArray *pUids); +//int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *results); +//int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *parm, SArray *pUids); +//int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids); +//int32_t metaFilterTtl(SMeta *pMeta, SMetaFltParam *param, SArray *pUids); #if 1 // refact APIs below (TODO) typedef SVCreateTbReq STbCfg; @@ -213,32 +214,32 @@ size_t tsdbCacheGetCapacity(SVnode *pVnode); size_t tsdbCacheGetUsage(SVnode *pVnode); int32_t tsdbCacheGetElems(SVnode *pVnode); -// tq -typedef struct SMetaTableInfo { - int64_t suid; - int64_t uid; - SSchemaWrapper *schema; - char tbName[TSDB_TABLE_NAME_LEN]; -} SMetaTableInfo; +//// tq +//typedef struct SMetaTableInfo { +// int64_t suid; +// int64_t uid; +// SSchemaWrapper *schema; +// char tbName[TSDB_TABLE_NAME_LEN]; +//} SMetaTableInfo; typedef struct SIdInfo { int64_t version; int32_t index; } SIdInfo; -typedef struct SSnapContext { - SMeta *pMeta; - int64_t snapVersion; - TBC *pCur; - int64_t suid; - int8_t subType; - SHashObj *idVersion; - SHashObj *suidInfo; - SArray *idList; - int32_t index; - bool withMeta; - bool queryMeta; // true-get meta, false-get data -} SSnapContext; +//typedef struct SSnapContext { +// SMeta *pMeta; +// int64_t snapVersion; +// TBC *pCur; +// int64_t suid; +// int8_t subType; +// SHashObj *idVersion; +// SHashObj *suidInfo; +// SArray *idList; +// int32_t index; +// bool withMeta; +// bool queryMeta; // true-get meta, false-get data +//} SSnapContext; typedef struct STqReader { SPackedData msg; @@ -271,7 +272,7 @@ bool tqNextBlockImpl(STqReader *pReader, const char* idstr); int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, const char* id); int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); -int32_t tqRetrieveDataBlock(STqReader *pReader, const char* idstr); +int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock** pRes, const char* idstr); int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg); @@ -350,67 +351,62 @@ struct SVnodeCfg { int32_t tsdbPageSize; }; -typedef struct { - uint64_t uid; - uint64_t groupId; -} STableKeyInfo; - #define TABLE_ROLLUP_ON ((int8_t)0x1) #define TABLE_IS_ROLLUP(FLG) (((FLG) & (TABLE_ROLLUP_ON)) != 0) #define TABLE_SET_ROLLUP(FLG) ((FLG) |= TABLE_ROLLUP_ON) -struct SMetaEntry { - int64_t version; - int8_t type; - int8_t flags; // TODO: need refactor? - tb_uid_t uid; - char *name; - union { - struct { - SSchemaWrapper schemaRow; - SSchemaWrapper schemaTag; - SRSmaParam rsmaParam; - } stbEntry; - struct { - int64_t ctime; - int32_t ttlDays; - int32_t commentLen; - char *comment; - tb_uid_t suid; - uint8_t *pTags; - } ctbEntry; - struct { - int64_t ctime; - int32_t ttlDays; - int32_t commentLen; - char *comment; - int32_t ncid; // next column id - SSchemaWrapper schemaRow; - } ntbEntry; - struct { - STSma *tsma; - } smaEntry; - }; +//struct SMetaEntry { +// int64_t version; +// int8_t type; +// int8_t flags; // TODO: need refactor? +// tb_uid_t uid; +// char *name; +// union { +// struct { +// SSchemaWrapper schemaRow; +// SSchemaWrapper schemaTag; +// SRSmaParam rsmaParam; +// } stbEntry; +// struct { +// int64_t ctime; +// int32_t ttlDays; +// int32_t commentLen; +// char *comment; +// tb_uid_t suid; +// uint8_t *pTags; +// } ctbEntry; +// struct { +// int64_t ctime; +// int32_t ttlDays; +// int32_t commentLen; +// char *comment; +// int32_t ncid; // next column id +// SSchemaWrapper schemaRow; +// } ntbEntry; +// struct { +// STSma *tsma; +// } smaEntry; +// }; +// +// uint8_t *pBuf; +//}; - uint8_t *pBuf; -}; +//struct SMetaReader { +// int32_t flags; +// SMeta *pMeta; +// SDecoder coder; +// SMetaEntry me; +// void *pBuf; +// int32_t szBuf; +//}; -struct SMetaReader { - int32_t flags; - SMeta *pMeta; - SDecoder coder; - SMetaEntry me; - void *pBuf; - int32_t szBuf; -}; - -struct SMTbCursor { - TBC *pDbc; - void *pKey; - void *pVal; - int32_t kLen; - int32_t vLen; - SMetaReader mr; -}; +//struct SMTbCursor { +// TBC *pDbc; +// void *pKey; +// void *pVal; +// int32_t kLen; +// int32_t vLen; +// SMetaReader mr; +//}; #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index b2bc9abf33..4bd16cf14f 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -844,9 +844,6 @@ int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); -void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); -size_t tsdbCacheGetCapacity(SVnode *pVnode); - // int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema); // ========== inline functions ========== diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index d464f64de3..4cc4176fda 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -249,7 +249,7 @@ int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) { SMetaReader *pReader = &mr; // query name.idx - if (tdbTbGet(pReader->pMeta->pNameIdx, tbName, strlen(tbName) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { + if (tdbTbGet(((SMeta*)pReader->pMeta)->pNameIdx, tbName, strlen(tbName) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; metaReaderClear(&mr); return -1; @@ -293,9 +293,9 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { metaReaderInit(&pTbCur->mr, pMeta, 0); - tdbTbcOpen(pMeta->pUidIdx, &pTbCur->pDbc, NULL); + tdbTbcOpen(pMeta->pUidIdx, (TBC **)&pTbCur->pDbc, NULL); - tdbTbcMoveToFirst(pTbCur->pDbc); + tdbTbcMoveToFirst((TBC *)pTbCur->pDbc); return pTbCur; } @@ -306,7 +306,7 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) { tdbFree(pTbCur->pVal); metaReaderClear(&pTbCur->mr); if (pTbCur->pDbc) { - tdbTbcClose(pTbCur->pDbc); + tdbTbcClose((TBC *)pTbCur->pDbc); } taosMemoryFree(pTbCur); } @@ -318,7 +318,7 @@ int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType) { STbCfg tbCfg; for (;;) { - ret = tdbTbcNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); + ret = tdbTbcNext((TBC *)pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); if (ret < 0) { return -1; } @@ -342,7 +342,7 @@ int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType) { STbCfg tbCfg; for (;;) { - ret = tdbTbcPrev(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); + ret = tdbTbcPrev((TBC *)pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); if (ret < 0) { return -1; } diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 707dd66e30..d3b6471282 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -222,29 +222,29 @@ static void destroySTableInfoForChildTable(void* data) { } static void MoveToSnapShotVersion(SSnapContext* ctx) { - tdbTbcClose(ctx->pCur); - tdbTbcOpen(ctx->pMeta->pTbDb, &ctx->pCur, NULL); + tdbTbcClose((TBC*)ctx->pCur); + tdbTbcOpen(ctx->pMeta->pTbDb, (TBC**)&ctx->pCur, NULL); STbDbKey key = {.version = ctx->snapVersion, .uid = INT64_MAX}; int c = 0; - tdbTbcMoveTo(ctx->pCur, &key, sizeof(key), &c); + tdbTbcMoveTo((TBC*)ctx->pCur, &key, sizeof(key), &c); if (c < 0) { - tdbTbcMoveToPrev(ctx->pCur); + tdbTbcMoveToPrev((TBC*)ctx->pCur); } } static int32_t MoveToPosition(SSnapContext* ctx, int64_t ver, int64_t uid) { - tdbTbcClose(ctx->pCur); - tdbTbcOpen(ctx->pMeta->pTbDb, &ctx->pCur, NULL); + tdbTbcClose((TBC*)ctx->pCur); + tdbTbcOpen(ctx->pMeta->pTbDb, (TBC**)&ctx->pCur, NULL); STbDbKey key = {.version = ver, .uid = uid}; int c = 0; - tdbTbcMoveTo(ctx->pCur, &key, sizeof(key), &c); + tdbTbcMoveTo((TBC*)ctx->pCur, &key, sizeof(key), &c); return c; } static void MoveToFirst(SSnapContext* ctx) { - tdbTbcClose(ctx->pCur); - tdbTbcOpen(ctx->pMeta->pTbDb, &ctx->pCur, NULL); - tdbTbcMoveToFirst(ctx->pCur); + tdbTbcClose((TBC*)ctx->pCur); + tdbTbcOpen(ctx->pMeta->pTbDb, (TBC**)&ctx->pCur, NULL); + tdbTbcMoveToFirst((TBC*)ctx->pCur); } static void saveSuperTableInfoForChildTable(SMetaEntry* me, SHashObj* suidInfo) { @@ -291,7 +291,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t metaDebug("tmqsnap init snapVersion:%" PRIi64, ctx->snapVersion); MoveToFirst(ctx); while (1) { - int32_t ret = tdbTbcNext(ctx->pCur, &pKey, &kLen, &pVal, &vLen); + int32_t ret = tdbTbcNext((TBC*)ctx->pCur, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; STbDbKey* tmp = (STbDbKey*)pKey; if (tmp->version > ctx->snapVersion) break; @@ -329,7 +329,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t MoveToSnapShotVersion(ctx); while (1) { - int32_t ret = tdbTbcPrev(ctx->pCur, &pKey, &kLen, &pVal, &vLen); + int32_t ret = tdbTbcPrev((TBC*)ctx->pCur, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; STbDbKey* tmp = (STbDbKey*)pKey; @@ -378,7 +378,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t } int32_t destroySnapContext(SSnapContext* ctx) { - tdbTbcClose(ctx->pCur); + tdbTbcClose((TBC*)ctx->pCur); taosArrayDestroy(ctx->idList); taosHashCleanup(ctx->idVersion); taosHashCleanup(ctx->suidInfo); @@ -496,7 +496,7 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in metaDebug("tmqsnap get meta not exist uid:%" PRIi64 " version:%" PRIi64, *uid, idInfo->version); } - tdbTbcGet(ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen); + tdbTbcGet((TBC*)ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen); SDecoder dc = {0}; SMetaEntry me = {0}; tDecoderInit(&dc, pVal, vLen); @@ -622,7 +622,7 @@ SMetaTableInfo getUidfromSnapShot(SSnapContext* ctx) { metaDebug("tmqsnap getUidfromSnapShot not exist uid:%" PRIi64 " version:%" PRIi64, *uidTmp, idInfo->version); continue; } - tdbTbcGet(ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen); + tdbTbcGet((TBC*)ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen); SDecoder dc = {0}; SMetaEntry me = {0}; tDecoderInit(&dc, pVal, vLen); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index b756f99f32..84ead71fb5 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -394,8 +394,9 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id) { SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); if (pReader->tbIdHash == NULL) { - int32_t code = tqRetrieveDataBlock(pReader, NULL); - if (code == TSDB_CODE_SUCCESS && pReader->pResBlock->info.rows > 0) { + SSDataBlock* pRes = NULL; + int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL); + if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) { return true; } } @@ -404,8 +405,9 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id) { if (ret != NULL) { tqDebug("tq reader return submit block, uid:%" PRId64 ", ver:%" PRId64, pSubmitTbData->uid, pReader->msg.ver); - int32_t code = tqRetrieveDataBlock(pReader, NULL); - if (code == TSDB_CODE_SUCCESS && pReader->pResBlock->info.rows > 0) { + SSDataBlock* pRes = NULL; + int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL); + if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) { return true; } } else { @@ -440,6 +442,11 @@ int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, i return 0; } +SWalReader* tqGetWalReader(STqReader* pReader) { + return pReader->pWalReader; +} + + bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { if (pReader->msg.msgStr == NULL) { return false; @@ -592,7 +599,7 @@ static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SCol return code; } -int32_t tqRetrieveDataBlock(STqReader* pReader, const char* id) { +int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* id) { tqDebug("tq reader retrieve data block %p, index:%d", pReader->msg.msgStr, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk++); @@ -1024,6 +1031,14 @@ int tqReaderAddTbUidList(STqReader* pReader, const SArray* pTableUidList) { return 0; } +bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid) { + return taosHashGet(pReader->tbIdHash, &uid, sizeof(uint64_t)); +} + +bool tqCurrentBlockConsumed(const STqReader* pReader) { + return pReader->msg.msgStr == NULL; +} + int tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) { for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i); diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 4adff71d85..d2c39aba74 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -3,7 +3,7 @@ aux_source_directory(src EXECUTOR_SRC) add_library(executor STATIC ${EXECUTOR_SRC}) target_link_libraries(executor - PRIVATE os util common function parser planner qcom vnode scalar nodes index + PRIVATE os util common function parser planner qcom scalar nodes index wal tdb ) target_include_directories( diff --git a/source/libs/executor/inc/dataSinkInt.h b/source/libs/executor/inc/dataSinkInt.h index 57a771b275..3255831476 100644 --- a/source/libs/executor/inc/dataSinkInt.h +++ b/source/libs/executor/inc/dataSinkInt.h @@ -22,13 +22,15 @@ extern "C" { #include "dataSinkMgt.h" #include "plannodes.h" +#include "storageapi.h" #include "tcommon.h" struct SDataSink; struct SDataSinkHandle; typedef struct SDataSinkManager { - SDataSinkMgtCfg cfg; + SDataSinkMgtCfg cfg; + SStorageAPI storeFn; } SDataSinkManager; typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue); diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 896858ac7f..6dc359c502 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -12,17 +12,17 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef TDENGINE_QUERYUTIL_H -#define TDENGINE_QUERYUTIL_H +#ifndef TDENGINE_EXECUTIL_H +#define TDENGINE_EXECUTIL_H #include "executor.h" #include "function.h" #include "nodes.h" #include "plannodes.h" +#include "storageapi.h" #include "tcommon.h" #include "tpagedbuf.h" #include "tsimplehash.h" -#include "vnode.h" #define T_LONG_JMP(_obj, _c) \ do { \ @@ -154,7 +154,7 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); SSDataBlock* createDataBlockFromDescNode(SDataBlockDescNode* pNode); EDealRes doTranslateTagExpr(SNode** pNode, void* pContext); -int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId); +int32_t getGroupIdFromTagsVal(void* pVnode, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId, SStorageAPI* pAPI); size_t getTableTagsBufLen(const SNodeList* pGroups); SArray* createSortInfo(SNodeList* pNodeList); @@ -178,8 +178,8 @@ void cleanupQueryTableDataCond(SQueryTableDataCond* pCond); int32_t convertFillType(int32_t mode); int32_t resultrowComparAsc(const void* p1, const void* p2); -int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified); +int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified, SStorageAPI *pAPI); void printDataBlock(SSDataBlock* pBlock, const char* flag); -#endif // TDENGINE_QUERYUTIL_H +#endif // TDENGINE_EXECUTIL_H diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 2daeb70260..57b3bc3292 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -38,13 +38,15 @@ extern "C" { #include "tlockfree.h" #include "tmsg.h" #include "tpagedbuf.h" -#include "tstream.h" -#include "tstreamUpdate.h" - -#include "vnode.h" +//#include "tstream.h" +//#include "tstreamUpdate.h" +#include "tlrucache.h" typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); +typedef struct STsdbReader STsdbReader; +typedef struct STqReader STqReader; + #define IS_VALID_SESSION_WIN(winInfo) ((winInfo).sessionWin.win.skey > 0) #define SET_SESSION_WIN_INVALID(winInfo) ((winInfo).sessionWin.win.skey = INT64_MIN) #define IS_INVALID_SESSION_WIN_KEY(winKey) ((winKey).win.skey <= 0) @@ -206,6 +208,7 @@ typedef struct STableScanBase { SLimitInfo limitInfo; // there are more than one table list exists in one task, if only one vnode exists. STableListInfo* pTableListInfo; + SStoreDataReaderFn readerAPI; } STableScanBase; typedef struct STableScanInfo { @@ -221,6 +224,7 @@ typedef struct STableScanInfo { int8_t assignBlockUid; bool hasGroupByTag; bool countOnly; + SStoreDataReaderFn readerAPI; } STableScanInfo; typedef struct STableMergeScanInfo { @@ -280,6 +284,7 @@ typedef struct SStreamAggSupporter { int32_t stateKeySize; int16_t stateKeyType; SDiskbasedBuf* pResultBuf; + SStateStore stateStore; } SStreamAggSupporter; typedef struct SWindowSupporter { @@ -335,7 +340,7 @@ typedef struct SStreamScanInfo { STqReader* tqReader; uint64_t groupId; - SUpdateInfo* pUpdateInfo; + struct SUpdateInfo* pUpdateInfo; EStreamScanMode scanMode; struct SOperatorInfo* pStreamScanOp; @@ -366,15 +371,18 @@ typedef struct SStreamScanInfo { SSDataBlock* pCreateTbRes; int8_t igCheckUpdate; int8_t igExpired; - SStreamState* pState; + void* pState; //void + SStoreTqReaderFn readerFn; + SStateStore stateStore; } SStreamScanInfo; typedef struct { - SVnode* vnode; - SSDataBlock pRes; // result SSDataBlock - STsdbReader* dataReader; - SSnapContext* sContext; - STableListInfo* pTableListInfo; + struct SVnode* vnode; // todo remove this + SSDataBlock pRes; // result SSDataBlock + STsdbReader* dataReader; + struct SSnapContext* sContext; + SStorageAPI* pAPI; + STableListInfo* pTableListInfo; } SStreamRawScanInfo; typedef struct STableCountScanSupp { @@ -441,12 +449,13 @@ typedef struct SStreamIntervalOperatorInfo { bool isFinal; SArray* pChildren; int32_t numOfChild; - SStreamState* pState; + SStreamState* pState; // void SWinKey delKey; uint64_t numOfDatapack; SArray* pUpdated; SSHashObj* pUpdatedMap; int64_t dataVersion; + SStateStore statestore; } SStreamIntervalOperatorInfo; typedef struct SDataGroupInfo { @@ -543,6 +552,7 @@ typedef struct SStreamFillSupporter { int32_t rowSize; SSHashObj* pResMap; bool hasDelete; + SStorageAPI* pAPI; } SStreamFillSupporter; typedef struct SStreamFillOperatorInfo { @@ -636,7 +646,7 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); bool functionNeedToExecute(SqlFunctionCtx* pCtx); bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); -bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup); +bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup, SStateStore* pStore); void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp, void* pTbName); uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); @@ -645,20 +655,17 @@ int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPos SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); bool groupbyTbname(SNodeList* pGroupList); -int32_t buildDataBlockFromGroupRes(struct SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup, +int32_t buildDataBlockFromGroupRes(struct SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo); -int32_t saveSessionDiscBuf(SStreamState* pState, SSessionKey* key, void* buf, int32_t size); -int32_t buildSessionResultDataBlock(struct SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, +int32_t saveSessionDiscBuf(void* pState, SSessionKey* key, void* buf, int32_t size, SStateStore* pAPI); +int32_t buildSessionResultDataBlock(struct SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo); -int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, - SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup); -int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult); -int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize); +int32_t releaseOutputBuf(void* pState, SWinKey* pKey, SResultRow* pResult, SStateStore* pAPI); void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order); int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, int32_t order, int64_t* pData); -void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, - SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock); +void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, + SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI); SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag); SExprInfo* createExpr(SNodeList* pNodeList, int32_t* numOfExprs); diff --git a/source/libs/executor/inc/operator.h b/source/libs/executor/inc/operator.h index 632b817a07..1d2685b8c6 100644 --- a/source/libs/executor/inc/operator.h +++ b/source/libs/executor/inc/operator.h @@ -156,7 +156,7 @@ void destroyOperator(SOperatorInfo* pOperator); SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id); int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder); -int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr); +int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr, SStorageAPI* pAPI); int32_t getOperatorExplainExecInfo(struct SOperatorInfo* operatorInfo, SArray* pExecInfoList); #ifdef __cplusplus diff --git a/source/libs/executor/inc/querytask.h b/source/libs/executor/inc/querytask.h index 37c93fef5c..1c2ed6c076 100644 --- a/source/libs/executor/inc/querytask.h +++ b/source/libs/executor/inc/querytask.h @@ -56,20 +56,19 @@ typedef struct STaskStopInfo { } STaskStopInfo; typedef struct { - STqOffsetVal currentOffset; // for tmq - SMqMetaRsp metaRsp; // for tmq fetching meta - int64_t snapshotVer; -// SPackedData submit; // todo remove it - SSchemaWrapper* schema; - char tbName[TSDB_TABLE_NAME_LEN]; // this is the current scan table: todo refactor - int8_t recoverStep; - int8_t recoverScanFinished; - SQueryTableDataCond tableCond; - int64_t fillHistoryVer1; - int64_t fillHistoryVer2; - SStreamState* pState; - int64_t dataVersion; - int64_t checkPointId; + STqOffsetVal currentOffset; // for tmq + SMqMetaRsp metaRsp; // for tmq fetching meta + int64_t snapshotVer; + SSchemaWrapper* schema; + char tbName[TSDB_TABLE_NAME_LEN]; // this is the current scan table: todo refactor + int8_t recoverStep; + int8_t recoverScanFinished; + SQueryTableDataCond tableCond; + int64_t fillHistoryVer1; + int64_t fillHistoryVer2; + SStreamState* pState; + int64_t dataVersion; + int64_t checkPointId; } SStreamTaskInfo; struct SExecTaskInfo { @@ -92,6 +91,7 @@ struct SExecTaskInfo { SArray* pResultBlockList; // result block list STaskStopInfo stopInfo; SRWLatch lock; // secure the access of STableListInfo + SStorageAPI storageAPI; }; void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst); diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index 5cd95d3311..918038635b 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -21,7 +21,7 @@ #include "tname.h" #include "executorInt.h" -#include "index.h" +#include "tsdstorage.h" #include "operator.h" #include "query.h" #include "querytask.h" @@ -30,6 +30,7 @@ #include "tglobal.h" #include "thash.h" #include "ttypes.h" +#include "index.h" typedef struct { bool hasAgg; diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 96dac643a5..1b6c06b6c4 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "function.h" #include "os.h" +#include "function.h" #include "tname.h" #include "tdatablock.h" @@ -27,6 +27,8 @@ #include "thash.h" #include "ttypes.h" +#include "storageapi.h" + typedef struct SCacheRowsScanInfo { SSDataBlock* pRes; SReadHandle readHandle; @@ -102,9 +104,9 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0); uint64_t suid = tableListGetSuid(pTableListInfo); - code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables, - taosArrayGetSize(pInfo->matchInfo.pList), pCidList, pInfo->pSlotIds, suid, - &pInfo->pLastrowReader, pTaskInfo->id.str); + code = pInfo->readHandle.api.cacheFn.openReader(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables, + taosArrayGetSize(pInfo->matchInfo.pList), pCidList, pInfo->pSlotIds, + suid, &pInfo->pLastrowReader, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -172,7 +174,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { blockDataCleanup(pInfo->pBufferredRes); taosArrayClear(pInfo->pUidList); - int32_t code = tsdbRetrieveCacheRows(pInfo->pLastrowReader, pInfo->pBufferredRes, pInfo->pSlotIds, + int32_t code = pInfo->readHandle.api.cacheFn.retrieveRows(pInfo->pLastrowReader, pInfo->pBufferredRes, pInfo->pSlotIds, pInfo->pDstSlotIds, pInfo->pUidList); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); @@ -239,7 +241,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } if (NULL == pInfo->pLastrowReader) { - code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num, + code = pInfo->readHandle.api.cacheFn.openReader(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num, taosArrayGetSize(pInfo->matchInfo.pList), pInfo->pCidList, pInfo->pSlotIds, suid, &pInfo->pLastrowReader, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { @@ -248,12 +250,12 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { continue; } } else { - tsdbReuseCacherowsReader(pInfo->pLastrowReader, pList, num); + pInfo->readHandle.api.cacheFn.reuseReader(pInfo->pLastrowReader, pList, num); } taosArrayClear(pInfo->pUidList); - code = tsdbRetrieveCacheRows(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds, pInfo->pDstSlotIds, + code = pInfo->readHandle.api.cacheFn.retrieveRows(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds, pInfo->pDstSlotIds, pInfo->pUidList); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); @@ -287,7 +289,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } } - pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); + pInfo->pLastrowReader = pInfo->readHandle.api.cacheFn.closeReader(pInfo->pLastrowReader); setOperatorCompleted(pOperator); return NULL; } @@ -305,7 +307,7 @@ void destroyCacheScanOperator(void* param) { tableListDestroy(pInfo->pTableList); if (pInfo->pLastrowReader != NULL) { - pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); + pInfo->pLastrowReader = pInfo->readHandle.api.cacheFn.closeReader(pInfo->pLastrowReader); } cleanupExprSupp(&pInfo->pseudoExprSup); diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index d31ac0bc51..aadbf1f1a9 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -17,6 +17,7 @@ #include "dataSinkMgt.h" #include "executorInt.h" #include "planner.h" +#include "storageapi.h" #include "tcompression.h" #include "tdatablock.h" #include "tglobal.h" @@ -428,8 +429,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat inserter->explain = pInserterNode->explain; int64_t suid = 0; - int32_t code = - tsdbGetTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid); + int32_t code = pManager->storeFn.metaFn.getTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid); if (code) { destroyDataSinker((SDataSinkHandle*)inserter); taosMemoryFree(inserter); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index fa46715c22..c742794498 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -27,6 +27,7 @@ #include "executorInt.h" #include "querytask.h" #include "tcompression.h" +#include "storageapi.h" typedef struct tagFilterAssist { SHashObj* colHash; @@ -41,13 +42,13 @@ typedef enum { } FilterCondType; static FilterCondType checkTagCond(SNode* cond); -static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond); -static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* list, SNode* pTagCond); +static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond, SStorageAPI* pAPI); +static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* list, SNode* pTagCond, SStorageAPI* pStoreAPI); -static int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, - SNode* pTagIndexCond, STableListInfo* pListInfo, uint8_t* digest, const char* idstr); +static int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, + SNode* pTagIndexCond, STableListInfo* pListInfo, uint8_t* digest, const char* idstr, SStorageAPI* pStorageAPI); static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, - void* metaHandle); + void* pVnode, SStorageAPI* pStorageAPI); static int64_t getLimit(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->limit; } static int64_t getOffset(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->offset; } @@ -262,7 +263,7 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { STagVal tagVal = {0}; tagVal.cid = pSColumnNode->colId; - const char* p = metaGetTableTagVal(mr->me.ctbEntry.pTags, pSColumnNode->node.resType.type, &tagVal); + const char* p = mr->storageAPI->metaFn.extractTagVal(mr->me.ctbEntry.pTags, pSColumnNode->node.resType.type, &tagVal); if (p == NULL) { res->node.resType.type = TSDB_DATA_TYPE_NULL; } else if (pSColumnNode->node.resType.type == TSDB_DATA_TYPE_JSON) { @@ -301,14 +302,14 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified) { +int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified, SStorageAPI *pAPI) { int32_t code = TSDB_CODE_SUCCESS; SMetaReader mr = {0}; - metaReaderInit(&mr, metaHandle, 0); - code = metaGetTableEntryByUidCache(&mr, info->uid); + pAPI->metaReaderFn.initReader(&mr, metaHandle, 0); + code = pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, info->uid); if (TSDB_CODE_SUCCESS != code) { - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); *pQualified = false; return TSDB_CODE_SUCCESS; @@ -317,7 +318,7 @@ int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, SNode* pTagCondTmp = nodesCloneNode(pTagCond); nodesRewriteExprPostOrder(&pTagCondTmp, doTranslateTagExpr, &mr); - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); SNode* pNew = NULL; code = scalarCalculateConstants(pTagCondTmp, &pNew); @@ -453,8 +454,8 @@ static void genTbGroupDigest(const SNode* pGroup, uint8_t* filterDigest, T_MD5_C taosMemoryFree(payload); } - -int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableListInfo* pTableListInfo, uint8_t *digest) { +int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInfo* pTableListInfo, uint8_t* digest, + SStorageAPI* pAPI) { int32_t code = TSDB_CODE_SUCCESS; SArray* pBlockList = NULL; SSDataBlock* pResBlock = NULL; @@ -465,7 +466,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis int32_t rows = taosArrayGetSize(pTableListInfo->pTableList); if (rows == 0) { - return TDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } tagFilterAssist ctx = {0}; @@ -494,8 +495,8 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis listNode->pNodeList = group; genTbGroupDigest((SNode *)listNode, digest, &context); nodesFree(listNode); - - metaGetCachedTbGroup(metaHandle, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), &tableList); + + pAPI->metaFn.getCachedTableList(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), &tableList); if (tableList) { taosArrayDestroy(pTableListInfo->pTableList); pTableListInfo->pTableList = tableList; @@ -511,14 +512,13 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis taosArrayPush(pUidTagList, &info); } - // int64_t stt = taosGetTimestampUs(); - code = metaGetTableTags(metaHandle, pTableListInfo->idInfo.suid, pUidTagList); + code = pAPI->metaFn.getTableTags(pVnode, pTableListInfo->idInfo.suid, pUidTagList); if (code != TSDB_CODE_SUCCESS) { goto end; } int32_t numOfTables = taosArrayGetSize(pUidTagList); - pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, metaHandle); + pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, pVnode, pAPI); if (pResBlock == NULL) { code = terrno; goto end; @@ -632,7 +632,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis if (tsTagFilterCache) { tableList = taosArrayDup(pTableListInfo->pTableList, NULL); - metaPutTbGroupToCache(metaHandle, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), tableList, taosArrayGetSize(tableList) * sizeof(STableKeyInfo)); + pAPI->metaFn.putTableListIntoCache(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), tableList, taosArrayGetSize(tableList) * sizeof(STableKeyInfo)); } // int64_t st2 = taosGetTimestampUs(); @@ -734,12 +734,12 @@ static FilterCondType checkTagCond(SNode* cond) { return FILTER_OTHER; } -static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* cond) { +static int32_t optimizeTbnameInCond(void* pVnode, int64_t suid, SArray* list, SNode* cond, SStorageAPI* pAPI) { int32_t ret = -1; int32_t ntype = nodeType(cond); if (ntype == QUERY_NODE_OPERATOR) { - ret = optimizeTbnameInCondImpl(metaHandle, list, cond); + ret = optimizeTbnameInCondImpl(pVnode, list, cond, pAPI); } if (ntype != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) { @@ -758,7 +758,7 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list SListCell* cell = pList->pHead; for (int i = 0; i < len; i++) { if (cell == NULL) break; - if (optimizeTbnameInCondImpl(metaHandle, list, cell->pNode) == 0) { + if (optimizeTbnameInCondImpl(pVnode, list, cell->pNode, pAPI) == 0) { hasTbnameCond = true; break; } @@ -769,14 +769,14 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list taosArrayRemoveDuplicate(list, filterTableInfoCompare, NULL); if (hasTbnameCond) { - ret = metaGetTableTagsByUids(metaHandle, suid, list); + ret = pAPI->metaFn.getTableTagsByUid(pVnode, suid, list); } return ret; } // only return uid that does not contained in pExistedUidList -static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidList, SNode* pTagCond) { +static int32_t optimizeTbnameInCondImpl(void* pVnode, SArray* pExistedUidList, SNode* pTagCond, SStorageAPI* pStoreAPI) { if (nodeType(pTagCond) != QUERY_NODE_OPERATOR) { return -1; } @@ -813,9 +813,9 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidLis char* name = taosArrayGetP(pTbList, i); uint64_t uid = 0; - if (metaGetTableUidByName(metaHandle, name, &uid) == 0) { + if (pStoreAPI->metaFn.getTableUidByName(pVnode, name, &uid) == 0) { ETableType tbType = TSDB_TABLE_MAX; - if (metaGetTableTypeByName(metaHandle, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) { + if (pStoreAPI->metaFn.getTableTypeByName(pVnode, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) { if (NULL == uHash || taosHashGet(uHash, &uid, sizeof(uid)) == NULL) { STUidTagInfo s = {.uid = uid, .name = name, .pTagVal = NULL}; taosArrayPush(pExistedUidList, &s); @@ -841,7 +841,7 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidLis static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, - void* metaHandle) { + void* pVnode, SStorageAPI* pStorageAPI) { SSDataBlock* pResBlock = createDataBlock(); if (pResBlock == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -876,7 +876,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa if (p1->name != NULL) { STR_TO_VARSTR(str, p1->name); } else { // name is not retrieved during filter - metaGetTableNameByUid(metaHandle, p1->uid, str); + pStorageAPI->metaFn.getTableNameByUid(pVnode, p1->uid, str); } colDataSetVal(pColInfo, i, str, false); @@ -889,7 +889,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa if (p1->pTagVal == NULL) { colDataSetNULL(pColInfo, i); } else { - const char* p = metaGetTableTagVal(p1->pTagVal, pColInfo->info.type, &tagVal); + const char* p = pStorageAPI->metaFn.extractTagVal(p1->pTagVal, pColInfo->info.type, &tagVal); if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)) { colDataSetNULL(pColInfo, i); @@ -949,13 +949,13 @@ static void copyExistedUids(SArray* pUidTagList, const SArray* pUidList) { } } -static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SNode* pTagCond, void* metaHandle, - SIdxFltStatus status) { +static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SNode* pTagCond, void* pVnode, + SIdxFltStatus status, SStorageAPI* pAPI) { if (pTagCond == NULL) { return TSDB_CODE_SUCCESS; } - terrno = TDB_CODE_SUCCESS; + terrno = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS; SArray* pBlockList = NULL; @@ -985,7 +985,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN FilterCondType condType = checkTagCond(pTagCond); - int32_t filter = optimizeTbnameInCond(metaHandle, pListInfo->idInfo.suid, pUidTagList, pTagCond); + int32_t filter = optimizeTbnameInCond(pVnode, pListInfo->idInfo.suid, pUidTagList, pTagCond, pAPI); if (filter == 0) { // tbname in filter is activated, do nothing and return taosArrayClear(pUidList); @@ -998,9 +998,9 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN terrno = 0; } else { if ((condType == FILTER_NO_LOGIC || condType == FILTER_AND) && status != SFLT_NOT_INDEX) { - code = metaGetTableTagsByUids(metaHandle, pListInfo->idInfo.suid, pUidTagList); + code = pAPI->metaFn.getTableTagsByUid(pVnode, pListInfo->idInfo.suid, pUidTagList); } else { - code = metaGetTableTags(metaHandle, pListInfo->idInfo.suid, pUidTagList); + code = pAPI->metaFn.getTableTags(pVnode, pListInfo->idInfo.suid, pUidTagList); } if (code != TSDB_CODE_SUCCESS) { qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), pListInfo->idInfo.suid); @@ -1014,7 +1014,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN goto end; } - pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, metaHandle); + pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, pVnode, pAPI); if (pResBlock == NULL) { code = terrno; goto end; @@ -1052,8 +1052,8 @@ end: return code; } -int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, - STableListInfo* pListInfo, uint8_t* digest, const char* idstr) { +int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, + STableListInfo* pListInfo, uint8_t* digest, const char* idstr, SStorageAPI* pStorageAPI) { int32_t code = TSDB_CODE_SUCCESS; size_t numOfTables = 0; @@ -1065,10 +1065,10 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SIdxFltStatus status = SFLT_NOT_INDEX; if (pScanNode->tableType != TSDB_SUPER_TABLE) { pListInfo->idInfo.uid = pScanNode->uid; - if (metaIsTableExist(metaHandle, pScanNode->uid)) { + if (pStorageAPI->metaFn.isTableExisted(pVnode, pScanNode->uid)) { taosArrayPush(pUidList, &pScanNode->uid); } - code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle, status); + code = doFilterByTagCond(pListInfo, pUidList, pTagCond, pVnode, status, pStorageAPI); if (code != TSDB_CODE_SUCCESS) { goto _end; } @@ -1080,7 +1080,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, genTagFilterDigest(pTagCond, &context); bool acquired = false; - metaGetCachedTableUidList(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pUidList, + pStorageAPI->metaFn.getCachedTableList(pVnode, pScanNode->suid, context.digest, tListLen(context.digest), pUidList, &acquired); if (acquired) { digest[0] = 1; @@ -1091,26 +1091,28 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, } if (!pTagCond) { // no tag filter condition exists, let's fetch all tables of this super table - vnodeGetCtbIdList(pVnode, pScanNode->suid, pUidList); +// vnodeGetCtbIdList(); + pStorageAPI->metaFn.storeGetChildTableList(pVnode, pScanNode->suid, pUidList); } else { // failed to find the result in the cache, let try to calculate the results if (pTagIndexCond) { - void* pIndex = tsdbGetIvtIdx(metaHandle); + void* pIndex = pStorageAPI->metaFn.storeGetInvertIndex(pVnode); + SIndexMetaArg metaArg = { - .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = pIndex, .suid = pScanNode->uid}; + .metaEx = pVnode, .idx = pStorageAPI->metaFn.storeGetIndexInfo(pVnode), .ivtIdx = pIndex, .suid = pScanNode->uid}; status = SFLT_NOT_INDEX; code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status); if (code != 0 || status == SFLT_NOT_INDEX) { // temporarily disable it for performance sake qWarn("failed to get tableIds from index, suid:%" PRIu64, pScanNode->uid); - code = TDB_CODE_SUCCESS; + code = TSDB_CODE_SUCCESS; } else { qInfo("succ to get filter result, table num: %d", (int)taosArrayGetSize(pUidList)); } } } - code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle, status); + code = doFilterByTagCond(pListInfo, pUidList, pTagCond, pVnode, status, pStorageAPI); if (code != TSDB_CODE_SUCCESS) { goto _end; } @@ -1127,7 +1129,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, memcpy(pPayload + sizeof(int32_t), taosArrayGet(pUidList, 0), numOfTables * sizeof(uint64_t)); } - metaUidFilterCachePut(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pPayload, size, 1); +// metaUidFilterCachePut(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pPayload, size, 1); digest[0] = 1; memcpy(digest + 1, context.digest, tListLen(context.digest)); } @@ -1164,11 +1166,13 @@ size_t getTableTagsBufLen(const SNodeList* pGroups) { return keyLen; } -int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId) { +int32_t getGroupIdFromTagsVal(void* pVnode, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId, + SStorageAPI* pAPI) { SMetaReader mr = {0}; - metaReaderInit(&mr, pMeta, 0); - if (metaGetTableEntryByUidCache(&mr, uid) != 0) { // table not exist - metaReaderClear(&mr); + + pAPI->metaReaderFn.initReader(&mr, pVnode, 0); + if (pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, uid) != 0) { // table not exist + pAPI->metaReaderFn.clearReader(&mr); return TSDB_CODE_PAR_TABLE_NOT_EXIST; } @@ -1187,7 +1191,7 @@ int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, REPLACE_NODE(pNew); } else { nodesDestroyList(groupNew); - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); return code; } @@ -1204,7 +1208,7 @@ int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, if (tTagIsJson(data)) { terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR; nodesDestroyList(groupNew); - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); return terrno; } int32_t len = getJsonValueLen(data); @@ -1224,7 +1228,7 @@ int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, *pGroupId = calcGroupId(keyBuf, len); nodesDestroyList(groupNew); - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); return TSDB_CODE_SUCCESS; } @@ -2030,11 +2034,11 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) { memcpy(pTableListInfo->groupOffset, taosArrayGet(pList, 0), sizeof(int32_t) * pTableListInfo->numOfOuputGroups); taosArrayDestroy(pList); - return TDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SScanPhysiNode* pScanNode, SNodeList* group, - bool groupSort, uint8_t *digest) { + bool groupSort, uint8_t *digest, SStorageAPI* pAPI) { int32_t code = TSDB_CODE_SUCCESS; bool groupByTbname = groupbyTbname(group); @@ -2054,7 +2058,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pTableListInfo->numOfOuputGroups = 1; } } else { - code = getColInfoResultForGroupby(pHandle->meta, group, pTableListInfo, digest); + code = getColInfoResultForGroupby(pHandle->meta, group, pTableListInfo, digest, pAPI); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2086,7 +2090,7 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags } uint8_t digest[17] = {0}; - int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo, digest, idStr); + int32_t code = getTableList(pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo, digest, idStr, &pTaskInfo->storageAPI); if (code != TSDB_CODE_SUCCESS) { qError("failed to getTableList, code: %s", tstrerror(code)); return code; @@ -2104,7 +2108,7 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags return TSDB_CODE_SUCCESS; } - code = buildGroupIdMapForAllTables(pTableListInfo, pHandle, pScanNode, pGroupTags, groupSort, digest); + code = buildGroupIdMapForAllTables(pTableListInfo, pHandle, pScanNode, pGroupTags, groupSort, digest, &pTaskInfo->storageAPI); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index b6d4615997..e89db5b59c 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -14,6 +14,8 @@ */ #include "executor.h" +#include +#include #include "executorInt.h" #include "operator.h" #include "planner.h" @@ -21,7 +23,8 @@ #include "tdatablock.h" #include "tref.h" #include "tudf.h" -#include "vnode.h" + +#include "storageapi.h" static TdThreadOnce initPoolOnce = PTHREAD_ONCE_INIT; int32_t exchangeObjRefPool = -1; @@ -156,18 +159,18 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu } } -void doSetTaskId(SOperatorInfo* pOperator) { +void doSetTaskId(SOperatorInfo* pOperator, SStorageAPI *pAPI) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { SStreamScanInfo* pStreamScanInfo = pOperator->info; if (pStreamScanInfo->pTableScanOp != NULL) { STableScanInfo* pScanInfo = pStreamScanInfo->pTableScanOp->info; if (pScanInfo->base.dataReader != NULL) { - tsdbReaderSetId(pScanInfo->base.dataReader, pTaskInfo->id.str); + pAPI->storeReader.setReaderId(pScanInfo->base.dataReader, pTaskInfo->id.str); } } } else { - doSetTaskId(pOperator->pDownstream[0]); + doSetTaskId(pOperator->pDownstream[0], pAPI); } } @@ -177,7 +180,7 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { buildTaskId(taskId, queryId, pTaskInfo->id.str); // set the idstr for tsdbReader - doSetTaskId(pTaskInfo->pRoot); + doSetTaskId(pTaskInfo->pRoot, &pTaskInfo->storageAPI); } int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { @@ -320,7 +323,8 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v return pTaskInfo; } -static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr) { +static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr, + SStorageAPI* pAPI) { SArray* qa = taosArrayInit(4, sizeof(tb_uid_t)); int32_t numOfUids = taosArrayGetSize(tableIdList); if (numOfUids == 0) { @@ -336,11 +340,11 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S // let's discard the tables those are not created according to the queried super table. SMetaReader mr = {0}; - metaReaderInit(&mr, pScanInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&mr, pScanInfo->readHandle.meta, 0); for (int32_t i = 0; i < numOfUids; ++i) { uint64_t* id = (uint64_t*)taosArrayGet(tableIdList, i); - int32_t code = metaGetTableEntryByUid(&mr, *id); + int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr, *id); if (code != TSDB_CODE_SUCCESS) { qError("failed to get table meta, uid:%" PRIu64 " code:%s, %s", *id, tstrerror(terrno), idstr); continue; @@ -368,7 +372,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S if (pScanInfo->pTagCond != NULL) { bool qualified = false; STableKeyInfo info = {.groupId = 0, .uid = mr.me.uid}; - code = isQualifiedTable(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified); + code = isQualifiedTable(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified, pAPI); if (code != TSDB_CODE_SUCCESS) { qError("failed to filter new table, uid:0x%" PRIx64 ", %s", info.uid, idstr); continue; @@ -383,7 +387,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S taosArrayPush(qa, id); } - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); return qa; } @@ -401,10 +405,10 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI SStreamScanInfo* pScanInfo = pInfo->info; if (isAdd) { // add new table id - SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, id); + SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, id, &pTaskInfo->storageAPI); int32_t numOfQualifiedTables = taosArrayGetSize(qa); qDebug("%d qualified child tables added into stream scanner, %s", numOfQualifiedTables, id); - code = tqReaderAddTbUidList(pScanInfo->tqReader, qa); + code = pTaskInfo->storageAPI.tqReaderFn.tqReaderAddTables(pScanInfo->tqReader, qa); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(qa); return code; @@ -434,7 +438,7 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI keyInfo.groupId = keyInfo.uid; } else { code = getGroupIdFromTagsVal(pScanInfo->readHandle.meta, keyInfo.uid, pScanInfo->pGroupTags, keyBuf, - &keyInfo.groupId); + &keyInfo.groupId, &pTaskInfo->storageAPI); if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(keyBuf); taosArrayDestroy(qa); @@ -456,7 +460,7 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI } else { // remove the table id in current list qDebug("%d remove child tables from the stream scanner, %s", (int32_t)taosArrayGetSize(tableIdList), id); taosWLockLatch(&pTaskInfo->lock); - code = tqReaderRemoveTbUidList(pScanInfo->tqReader, tableIdList); + code = pTaskInfo->storageAPI.tqReaderFn.tqReaderRemoveTables(pScanInfo->tqReader, tableIdList); taosWUnLockLatch(&pTaskInfo->lock); } @@ -1060,6 +1064,8 @@ void qStreamSetOpen(qTaskInfo_t tinfo) { int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SOperatorInfo* pOperator = pTaskInfo->pRoot; const char* id = GET_TASKID(pTaskInfo); @@ -1081,14 +1087,15 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT if (pOffset->type == TMQ_OFFSET__LOG) { // todo refactor: move away - tsdbReaderClose(pScanBaseInfo->dataReader); + pTaskInfo->storageAPI.storeReader.storeReaderClose(pScanBaseInfo->dataReader); pScanBaseInfo->dataReader = NULL; - walReaderVerifyOffset(pInfo->tqReader->pWalReader, pOffset); - if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) { - qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id); - return -1; - } + ASSERT(0); +// walReaderVerifyOffset(pInfo->tqReader->pWalReader, pOffset); +// if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) { +// qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id); +// return -1; +// } } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { // iterate all tables from tableInfoList, and retrieve rows from each table one-by-one // those data are from the snapshot in tsdb, besides the data in the wal file. @@ -1141,8 +1148,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT pScanInfo->scanTimes = 0; if (pScanBaseInfo->dataReader == NULL) { - int32_t code = tsdbReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, - pScanInfo->pResBlock, &pScanBaseInfo->dataReader, id, false, NULL); + int32_t code = pTaskInfo->storageAPI.storeReader.storeReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, + pScanInfo->pResBlock, (void**) &pScanBaseInfo->dataReader, id, false, NULL); if (code != TSDB_CODE_SUCCESS) { qError("prepare read tsdb snapshot failed, uid:%" PRId64 ", code:%s %s", pOffset->uid, tstrerror(code), id); terrno = code; @@ -1152,8 +1159,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts:%" PRId64 " table index:%d, total:%d, %s", uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); } else { - tsdbSetTableList(pScanBaseInfo->dataReader, &keyInfo, 1); - tsdbReaderReset(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); + pTaskInfo->storageAPI.storeReader.storeReaderSetTableList(pScanBaseInfo->dataReader, &keyInfo, 1); + pTaskInfo->storageAPI.storeReader.storeReaderResetStatus(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 " table index:%d numOfTable:%d, %s", uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); } @@ -1175,14 +1182,14 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT SOperatorInfo* p = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, id); STableListInfo* pTableListInfo = ((SStreamRawScanInfo*)(p->info))->pTableListInfo; - if (setForSnapShot(sContext, pOffset->uid) != 0) { + if (pAPI->snapshotFn.storeCreateSnapshot(sContext, pOffset->uid) != 0) { qError("setDataForSnapShot error. uid:%" PRId64 " , %s", pOffset->uid, id); terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } - SMetaTableInfo mtInfo = getUidfromSnapShot(sContext); - tsdbReaderClose(pInfo->dataReader); + SMetaTableInfo mtInfo = pTaskInfo->storageAPI.snapshotFn.storeSSGetTableInfo(sContext); + pTaskInfo->storageAPI.storeReader.storeReaderClose(pInfo->dataReader); pInfo->dataReader = NULL; cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); @@ -1200,7 +1207,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0); int32_t size = tableListGetSize(pTableListInfo); - tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL, + pTaskInfo->storageAPI.storeReader.storeReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, (void**) &pInfo->dataReader, NULL, false, NULL); cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); @@ -1212,7 +1219,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) { SStreamRawScanInfo* pInfo = pOperator->info; SSnapContext* sContext = pInfo->sContext; - if (setForSnapShot(sContext, pOffset->uid) != 0) { + if (pTaskInfo->storageAPI.snapshotFn.storeCreateSnapshot(sContext, pOffset->uid) != 0) { qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version); terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; @@ -1221,7 +1228,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT id); } else if (pOffset->type == TMQ_OFFSET__LOG) { SStreamRawScanInfo* pInfo = pOperator->info; - tsdbReaderClose(pInfo->dataReader); + pTaskInfo->storageAPI.storeReader.storeReaderClose(pInfo->dataReader); pInfo->dataReader = NULL; qDebug("tmqsnap qStreamPrepareScan snapshot log, %s", id); } diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index d229f7d0ee..c11c5be1ae 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -33,7 +33,7 @@ #include "tcompare.h" #include "thash.h" #include "ttypes.h" -#include "vnode.h" +#include "storageapi.h" #define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) @@ -844,6 +844,8 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SSDataBlock* pBlock = pbInfo->pRes; // set output datablock version @@ -860,12 +862,13 @@ void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGr doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo, pOperator->resultInfo.threshold, false); void* tbname = NULL; - if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { + if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { pBlock->info.parTbName[0] = 0; } else { memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); } - streamFreeVal(tbname); + + pAPI->stateStore.streamStateFreeVal(tbname); } void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, @@ -1066,122 +1069,22 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo* return TSDB_CODE_SUCCESS; } -int32_t resultRowEncode(void* k, int32_t* size, char* buf) { - // SResultRow* key = k; - // int len = 0; - // int struLen = *size; - // len += taosEncodeFixedI32((void**)&buf, key->pageId); - - // uint32_t offset = key->offset; - // len += taosEncodeFixedU32((void**)&buf, offset); - - // len += taosEncodeFixedI8((void**)&buf, key->startInterp); - // len += taosEncodeFixedI8((void**)&buf, key->endInterp); - // len += taosEncodeFixedI8((void**)&buf, key->closed); - // len += taosEncodeFixedU32((void**)&buf, key->numOfRows); - - // len += taosEncodeFixedI64((void**)&buf, key->win.skey); - // len += taosEncodeFixedI64((void**)&buf, key->win.ekey); - - // int32_t numOfEntryInfo = (struLen - sizeof(SResultRow)) / sizeof(struct SResultRowEntryInfo); - // len += taosEncodeFixedI32((void**)&buf, numOfEntryInfo); - // for (int i = 0; i < numOfEntryInfo; i++) { - // SResultRowEntryInfo* p = &key->pEntryInfo[i]; - - // uint8_t value = p->initialized ? 1 : 0; - // len += taosEncodeFixedU8((void**)&buf, value); - - // value = p->complete ? 1 : 0; - // len += taosEncodeFixedU8((void**)&buf, value); - - // value = p->isNullRes; - // len += taosEncodeFixedU8((void**)&buf, value); - - // len += taosEncodeFixedU16((void**)&buf, p->numOfRes); - // } - // { - // char* strBuf = taosMemoryCalloc(1, *size * 100); - // resultRowToString(key, *size, strBuf); - // qWarn("encode result row:%s", strBuf); - // } - - // return len; - return 0; -} - -int32_t resultRowDecode(void** k, size_t size, char* buf) { - // char* p1 = buf; - // int32_t numOfEntryInfo = 0; - // uint32_t entryOffset = sizeof(int32_t) + sizeof(uint32_t) + sizeof(int8_t) + sizeof(int8_t) + sizeof(int8_t) + - // sizeof(uint32_t) + sizeof(int64_t) + sizeof(int64_t); - // taosDecodeFixedI32(p1 + entryOffset, &numOfEntryInfo); - - // char* p = buf; - // size = sizeof(SResultRow) + numOfEntryInfo * sizeof(SResultRowEntryInfo); - // SResultRow* key = taosMemoryCalloc(1, size); - - // p = taosDecodeFixedI32(p, (int32_t*)&key->pageId); - // uint32_t offset = 0; - // p = taosDecodeFixedU32(p, &offset); - // key->offset = offset; - - // p = taosDecodeFixedI8(p, (int8_t*)(&key->startInterp)); - // p = taosDecodeFixedI8(p, (int8_t*)(&key->endInterp)); - // p = taosDecodeFixedI8(p, (int8_t*)&key->closed); - // p = taosDecodeFixedU32(p, &key->numOfRows); - - // p = taosDecodeFixedI64(p, &key->win.skey); - // p = taosDecodeFixedI64(p, &key->win.ekey); - // p = taosDecodeFixedI32(p, &numOfEntryInfo); - // for (int i = 0; i < numOfEntryInfo; i++) { - // SResultRowEntryInfo* pInfo = &key->pEntryInfo[i]; - // uint8_t value = 0; - // p = taosDecodeFixedU8(p, &value); - // pInfo->initialized = (value == 1) ? true : false; - - // p = taosDecodeFixedU8(p, &value); - // pInfo->complete = (value == 1) ? true : false; - - // p = taosDecodeFixedU8(p, &value); - // pInfo->isNullRes = value; - - // p = taosDecodeFixedU16(p, &pInfo->numOfRes); - // } - // *k = key; - - // { - // char* strBuf = taosMemoryCalloc(1, size * 100); - // resultRowToString(key, size, strBuf); - // qWarn("decode result row:%s", strBuf); - // } - // return size; - return 0; -} - -int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize) { - // char* buf = taosMemoryCalloc(1, resSize * 10); - // int len = resultRowEncode((void*)pResult, &resSize, buf); - // char* buf = taosMemoryCalloc(1, resSize); - // memcpy(buf, pResult, resSize); - streamStatePut(pState, pKey, (char*)pResult, resSize); - // taosMemoryFree(buf); +int32_t releaseOutputBuf(void* pState, SWinKey* pKey, SResultRow* pResult, SStateStore* pAPI) { + pAPI->streamStateReleaseBuf(pState, pKey, pResult); return TSDB_CODE_SUCCESS; } -int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult) { - streamStateReleaseBuf(pState, pKey, pResult); +int32_t saveSessionDiscBuf(void* pState, SSessionKey* key, void* buf, int32_t size, SStateStore* pAPI) { + pAPI->streamStateSessionPut(pState, key, (const void*)buf, size); + releaseOutputBuf(pState, NULL, (SResultRow*)buf, pAPI); return TSDB_CODE_SUCCESS; } -int32_t saveSessionDiscBuf(SStreamState* pState, SSessionKey* key, void* buf, int32_t size) { - streamStateSessionPut(pState, key, (const void*)buf, size); - releaseOutputBuf(pState, NULL, (SResultRow*)buf); - return TSDB_CODE_SUCCESS; -} - -int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, +int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SExprInfo* pExprInfo = pSup->pExprInfo; int32_t numOfExprs = pSup->numOfExprs; int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; @@ -1193,7 +1096,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta SSessionKey* pKey = taosArrayGet(pGroupResInfo->pRows, i); int32_t size = 0; void* pVal = NULL; - int32_t code = streamStateSessionGet(pState, pKey, &pVal, &size); + int32_t code = pAPI->stateStore.streamStateSessionGet(pState, pKey, &pVal, &size); ASSERT(code == 0); if (code == -1) { // coverity scan @@ -1205,7 +1108,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta // no results, continue to check the next one if (pRow->numOfRows == 0) { pGroupResInfo->index += 1; - releaseOutputBuf(pState, NULL, pRow); + releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore); continue; } @@ -1213,23 +1116,23 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta pBlock->info.id.groupId = pKey->groupId; void* tbname = NULL; - if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { + if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { pBlock->info.parTbName[0] = 0; } else { memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); } - streamFreeVal(tbname); + pAPI->stateStore.streamStateFreeVal(tbname); } else { // current value belongs to different group, it can't be packed into one datablock if (pBlock->info.id.groupId != pKey->groupId) { - releaseOutputBuf(pState, NULL, pRow); + releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore); break; } } if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { ASSERT(pBlock->info.rows > 0); - releaseOutputBuf(pState, NULL, pRow); + releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore); break; } @@ -1260,7 +1163,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta pBlock->info.dataLoad = 1; pBlock->info.rows += pRow->numOfRows; - releaseOutputBuf(pState, NULL, pRow); + releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore); } blockDataUpdateTsWindow(pBlock, 0); return TSDB_CODE_SUCCESS; @@ -1292,7 +1195,7 @@ void qStreamCloseTsdbReader(void* task) { qDebug("wait for the reader stopping"); } - tsdbReaderClose(pTSInfo->base.dataReader); + pTaskInfo->storageAPI.storeReader.storeReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; // restore the status, todo refactor. diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 5101e36992..0b89dbc80b 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -443,7 +443,8 @@ void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols); tSimpleHashCleanup(pFillSup->pResMap); pFillSup->pResMap = NULL; - releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal); + ASSERT(0); + releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal, &pFillSup->pAPI->stateStore); //????? pFillSup->cur.pRowVal = NULL; cleanupExprSupp(&pFillSup->notFillExprSup); @@ -490,74 +491,79 @@ static void resetFillWindow(SResultRowData* pRowData) { pRowData->pRowVal = NULL; } -void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, SStreamState* pState) { +void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, void* pState, SStorageAPI* pAPI) { resetFillWindow(&pFillSup->prev); - releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal); + ASSERT(0); + releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal, &pAPI->stateStore); //??? resetFillWindow(&pFillSup->cur); resetFillWindow(&pFillSup->next); resetFillWindow(&pFillSup->nextNext); } void getCurWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { - SStreamState* pState = pOperator->pTaskInfo->streamInfo.pState; - resetPrevAndNextWindow(pFillSup, pState); + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + + void* pState = pOperator->pTaskInfo->streamInfo.pState; + resetPrevAndNextWindow(pFillSup, pState, pAPI); SWinKey key = {.ts = ts, .groupId = groupId}; int32_t curVLen = 0; - int32_t code = streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen); + + int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen); ASSERT(code == TSDB_CODE_SUCCESS); pFillSup->cur.key = key.ts; } void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { - SStreamState* pState = pOperator->pTaskInfo->streamInfo.pState; - resetPrevAndNextWindow(pFillSup, pState); + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + void* pState = pOperator->pTaskInfo->streamInfo.pState; + resetPrevAndNextWindow(pFillSup, pState, pAPI); SWinKey key = {.ts = ts, .groupId = groupId}; void* curVal = NULL; int32_t curVLen = 0; - int32_t code = streamStateFillGet(pState, &key, (void**)&curVal, &curVLen); + int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&curVal, &curVLen); ASSERT(code == TSDB_CODE_SUCCESS); pFillSup->cur.key = key.ts; pFillSup->cur.pRowVal = curVal; - SStreamStateCur* pCur = streamStateFillSeekKeyPrev(pState, &key); + void* pCur = pAPI->stateStore.streamStateFillSeekKeyPrev(pState, &key); SWinKey preKey = {.ts = INT64_MIN, .groupId = groupId}; void* preVal = NULL; int32_t preVLen = 0; - code = streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen); + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen); if (code == TSDB_CODE_SUCCESS) { pFillSup->prev.key = preKey.ts; pFillSup->prev.pRowVal = preVal; - code = streamStateCurNext(pState, pCur); + code = pAPI->stateStore.streamStateCurNext(pState, pCur); ASSERT(code == TSDB_CODE_SUCCESS); - code = streamStateCurNext(pState, pCur); + code = pAPI->stateStore.streamStateCurNext(pState, pCur); if (code != TSDB_CODE_SUCCESS) { - streamStateFreeCur(pCur); + pAPI->stateStore.streamStateFreeCur(pCur); pCur = NULL; } } else { - streamStateFreeCur(pCur); - pCur = streamStateFillSeekKeyNext(pState, &key); + pAPI->stateStore.streamStateFreeCur(pCur); + pCur = pAPI->stateStore.streamStateFillSeekKeyNext(pState, &key); } SWinKey nextKey = {.ts = INT64_MIN, .groupId = groupId}; void* nextVal = NULL; int32_t nextVLen = 0; - code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen); + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen); if (code == TSDB_CODE_SUCCESS) { pFillSup->next.key = nextKey.ts; pFillSup->next.pRowVal = nextVal; if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) { - code = streamStateCurNext(pState, pCur); + code = pAPI->stateStore.streamStateCurNext(pState, pCur); if (code == TSDB_CODE_SUCCESS) { SWinKey nextNextKey = {.groupId = groupId}; void* nextNextVal = NULL; int32_t nextNextVLen = 0; - code = streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen); + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen); if (code == TSDB_CODE_SUCCESS) { pFillSup->nextNext.key = nextNextKey.ts; pFillSup->nextNext.pRowVal = nextNextVal; @@ -565,7 +571,7 @@ void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, } } } - streamStateFreeCur(pCur); + pAPI->stateStore.streamStateFreeCur(pCur); } static bool hasPrevWindow(SStreamFillSupporter* pFillSup) { return pFillSup->prev.key != INT64_MIN; } @@ -922,8 +928,10 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* } static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SResultRowData* pRow, int32_t len) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SWinKey key = {.groupId = groupId, .ts = pRow->key}; - int32_t code = streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len); + int32_t code = pAPI->stateStore.streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len); qDebug("===stream===fill operator save key ts:%" PRId64 " group id:%" PRIu64 " code:%d", key.ts, key.groupId, code); ASSERT(code == TSDB_CODE_SUCCESS); } @@ -1021,7 +1029,8 @@ static void doStreamFillImpl(SOperatorInfo* pOperator) { } static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) { - SStreamState* pState = pOp->pTaskInfo->streamInfo.pState; + SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI; + void* pState = pOp->pTaskInfo->streamInfo.pState; SSDataBlock* pBlock = delRes; SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); @@ -1041,14 +1050,14 @@ static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_ SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); void* tbname = NULL; - streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname); + pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname); if (tbname == NULL) { colDataSetNULL(pTableCol, pBlock->info.rows); } else { char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); - streamFreeVal(tbname); + pAPI->stateStore.streamStateFreeVal(tbname); } pBlock->info.rows++; @@ -1070,12 +1079,13 @@ static void buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY end } static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; SStreamFillOperatorInfo* pInfo = pOperator->info; getWindowFromDiscBuf(pOperator, startTs, groupId, pInfo->pFillSup); setDeleteFillValueInfo(startTs, endTs, pInfo->pFillSup, pInfo->pFillInfo); SWinKey key = {.ts = startTs, .groupId = groupId}; if (!pInfo->pFillInfo->needFill) { - streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); + pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes); } else { STimeRange tw = { @@ -1093,6 +1103,8 @@ static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKE } static void doDeleteFillFinalize(SOperatorInfo* pOperator) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SStreamFillOperatorInfo* pInfo = pOperator->info; SStreamFillInfo* pFillInfo = pInfo->pFillInfo; int32_t size = taosArrayGetSize(pFillInfo->delRanges); @@ -1109,17 +1121,16 @@ static void doDeleteFillFinalize(SOperatorInfo* pOperator) { pInfo->pRes->info.id.groupId = range->groupId; } SWinKey key = {.ts = range->skey, .groupId = range->groupId}; - streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); + pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); } } static void doDeleteFillResult(SOperatorInfo* pOperator) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SStreamFillOperatorInfo* pInfo = pOperator->info; - SStreamFillSupporter* pFillSup = pInfo->pFillSup; SStreamFillInfo* pFillInfo = pInfo->pFillInfo; SSDataBlock* pBlock = pInfo->pSrcDelBlock; - SSDataBlock* pRes = pInfo->pRes; - SSDataBlock* pDelRes = pInfo->pDelRes; SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* tsStarts = (TSKEY*)pStartCol->pData; @@ -1130,7 +1141,7 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { TSKEY endTs = ts; uint64_t groupId = groupIds[pInfo->srcDelRowIndex]; SWinKey key = {.ts = ts, .groupId = groupId}; - SStreamStateCur* pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key); + void* pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key); if (!pCur) { pInfo->srcDelRowIndex++; continue; @@ -1152,15 +1163,15 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { SWinKey delKey = {.groupId = delGroupId, .ts = delTs}; if (delTs == nextKey.ts) { - code = streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur); + code = pAPI->stateStore.streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur); if (code == TSDB_CODE_SUCCESS) { - code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextLen); + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextLen); } // ts will be deleted later if (delTs != ts) { - streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey); - streamStateFreeCur(pCur); - pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey); + pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey); + pAPI->stateStore.streamStateFreeCur(pCur); + pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey); } endTs = TMAX(delTs, nextKey.ts - 1); if (code != TSDB_CODE_SUCCESS) { @@ -1169,9 +1180,11 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { } pInfo->srcDelRowIndex++; } - streamStateFreeCur(pCur); + + pAPI->stateStore.streamStateFreeCur(pCur); doDeleteFillResultImpl(pOperator, ts, endTs, groupId); } + pFillInfo->current = pFillInfo->end + 1; } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index f2eee82579..77d251bab9 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -950,6 +950,8 @@ static bool hasRemainPartion(SStreamPartitionOperatorInfo* pInfo) { return pInfo static bool hasRemainTbName(SStreamPartitionOperatorInfo* pInfo) { return pInfo->pTbNameIte != NULL; } static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SStreamPartitionOperatorInfo* pInfo = pOperator->info; SSDataBlock* pDest = pInfo->binfo.pRes; ASSERT(hasRemainPartion(pInfo)); @@ -972,9 +974,9 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { pDest->info.parTbName[0] = 0; if (pInfo->tbnameCalSup.numOfExprs > 0) { void* tbname = NULL; - if (streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname) == 0) { + if (pAPI->stateStore.streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname) == 0) { memcpy(pDest->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); - streamFreeVal(tbname); + pAPI->stateStore.streamStateFreeVal(tbname); } } taosArrayDestroy(pParInfo->rowIds); @@ -990,10 +992,10 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { return pDest; } -void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, - SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock) { +void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, + SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI) { void* pValue = NULL; - if (streamStateGetParName(pState, groupId, &pValue) != 0) { + if (pAPI->streamStateGetParName(pState, groupId, &pValue) != 0) { SSDataBlock* pTmpBlock = blockCopyOneRow(pSrcBlock, rowId); memset(pTmpBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN); pTmpBlock->info.id.groupId = groupId; @@ -1010,7 +1012,7 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* void* pData = colDataGetData(pTbCol, pDestBlock->info.rows - 1); len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); memcpy(tbName, varDataVal(pData), len); - streamStatePutParName(pState, groupId, tbName); + pAPI->streamStatePutParName(pState, groupId, tbName); } memcpy(pTmpBlock->info.parTbName, tbName, len); pDestBlock->info.rows--; @@ -1034,10 +1036,12 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* } else { memcpy(pSrcBlock->info.parTbName, pValue, TSDB_TABLE_NAME_LEN); } - streamStateReleaseBuf(pState, NULL, pValue); + pAPI->streamStateReleaseBuf(pState, NULL, pValue); } static SSDataBlock* buildStreamCreateTableResult(SOperatorInfo* pOperator) { + SExecTaskInfo* pTask = pOperator->pTaskInfo; + SStreamPartitionOperatorInfo* pInfo = pOperator->info; if ((pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) || taosHashGetSize(pInfo->pPartitions) == 0) { @@ -1050,8 +1054,8 @@ static SSDataBlock* buildStreamCreateTableResult(SOperatorInfo* pOperator) { if (pInfo->pTbNameIte != NULL) { SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->pTbNameIte; int32_t rowId = *(int32_t*)taosArrayGet(pParInfo->rowIds, 0); - appendCreateTableRow(pOperator->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, - pParInfo->groupId, pSrc, rowId, pInfo->pCreateTbRes); + appendCreateTableRow(pTask->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, + pParInfo->groupId, pSrc, rowId, pInfo->pCreateTbRes, &pTask->storageAPI.stateStore); pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, pInfo->pTbNameIte); } return pInfo->pCreateTbRes->info.rows > 0 ? pInfo->pCreateTbRes : NULL; @@ -1164,14 +1168,17 @@ static void destroyStreamPartitionOperatorInfo(void* param) { } void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr) { + SStorageAPI* pAPI = &downstream->pTaskInfo->storageAPI; + if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { return; } + SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->partitionSup = *pParSup; pScanInfo->pPartScalarSup = pExpr; if (!pScanInfo->igCheckUpdate && !pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0); + pScanInfo->pUpdateInfo = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0); } } diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 729178dc60..1b85aedc1e 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -25,7 +25,8 @@ #include "operator.h" #include "query.h" #include "querytask.h" -#include "vnode.h" + +#include "storageapi.h" SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, @@ -233,11 +234,12 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan } static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, const char* pIdStr) { + SStorageAPI* pAPI = pParam->pParam; if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { STableScanInfo* pInfo = pOperator->info; if (pInfo->base.dataReader != NULL) { - tsdbReaderSetCloseFlag(pInfo->base.dataReader); + pAPI->storeReader.storeReaderNotifyClosing(pInfo->base.dataReader); } return OPTR_FN_RET_ABORT; } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { @@ -246,7 +248,7 @@ static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, if (pInfo->pTableScanOp != NULL) { STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; if (pTableScanInfo != NULL && pTableScanInfo->base.dataReader != NULL) { - tsdbReaderSetCloseFlag(pTableScanInfo->base.dataReader); + pAPI->storeReader.storeReaderNotifyClosing(pTableScanInfo->base.dataReader); } } @@ -256,8 +258,8 @@ static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, return OPTR_FN_RET_CONTINUE; } -int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr) { - STraverParam p = {0}; +int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr, SStorageAPI* pAPI) { + STraverParam p = {.pParam = pAPI}; traverseOperatorTree(pOperator, doStopDataReader, &p, pIdStr); return p.code; } @@ -379,7 +381,7 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR if (pBlockNode->tableType == TSDB_SUPER_TABLE) { SArray* pList = taosArrayInit(4, sizeof(STableKeyInfo)); - int32_t code = vnodeGetAllTableList(pHandle->vnode, pBlockNode->uid, pList); + int32_t code = pTaskInfo->storageAPI.metaFn.storeGetTableList(pHandle->vnode, pBlockNode->uid, pList); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = terrno; return NULL; diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 02f504bef0..ae72627575 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -18,6 +18,7 @@ #include "functionMgt.h" #include "operator.h" #include "querytask.h" +#include "tdatablock.h" typedef struct SProjectOperatorInfo { SOptrBasicInfo binfo; diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c index 7716b5976b..a48de1d158 100644 --- a/source/libs/executor/src/querytask.c +++ b/source/libs/executor/src/querytask.c @@ -29,9 +29,9 @@ #include "operator.h" #include "query.h" #include "querytask.h" +#include "storageapi.h" #include "thash.h" #include "ttypes.h" -#include "vnode.h" #define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) @@ -63,7 +63,7 @@ bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code); } void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { pTaskInfo->code = rspCode; - stopTableScanOperator(pTaskInfo->pRoot, pTaskInfo->id.str); + stopTableScanOperator(pTaskInfo->pRoot, pTaskInfo->id.str, &pTaskInfo->storageAPI); } void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { @@ -120,13 +120,15 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo return terrno; } - metaReaderInit(&mr, pHandle->meta, 0); - int32_t code = metaGetTableEntryByUidCache(&mr, pScanNode->uid); + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + + pAPI->metaReaderFn.initReader(&mr, pHandle->meta, 0); + int32_t code = pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, pScanNode->uid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get the table meta, uid:0x%" PRIx64 ", suid:0x%" PRIx64 ", %s", pScanNode->uid, pScanNode->suid, GET_TASKID(pTaskInfo)); - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); return terrno; } @@ -142,9 +144,9 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo tDecoderClear(&mr.coder); tb_uid_t suid = mr.me.ctbEntry.suid; - code = metaGetTableEntryByUidCache(&mr, suid); + code = pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, suid); if (code != TSDB_CODE_SUCCESS) { - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); return terrno; } @@ -154,7 +156,7 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo pSchemaInfo->sw = tCloneSSchemaWrapper(&mr.me.ntbEntry.schemaRow); } - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); pSchemaInfo->qsw = extractQueriedColumnSchema(pScanNode); return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0537702a75..ac8ca675ce 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -33,6 +33,9 @@ #include "operator.h" #include "querytask.h" +#include "storageapi.h" +#include "wal.h" + int32_t scanDebug = 0; #define MULTI_READER_MAX_TABLE_NUM 5000 @@ -261,8 +264,10 @@ static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsA } static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + bool allColumnsHaveAgg = true; - int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); + int32_t code = pAPI->storeReader.storeReaderRetrieveBlockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -323,6 +328,8 @@ bool applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder; pCost->totalBlocks += 1; @@ -343,7 +350,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->filterOutBlocks += 1; pCost->totalRows += pBlock->info.rows; - tsdbReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", uid:%" PRIu64, @@ -351,7 +358,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pBlockInfo->id.uid); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); pCost->skipBlocks += 1; - tsdbReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_SMA_LOAD) { pCost->loadBlockStatis += 1; @@ -361,7 +368,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); - tsdbReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else { qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo)); @@ -383,7 +390,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->filterOutBlocks += 1; (*status) = FUNC_DATA_REQUIRED_FILTEROUT; - tsdbReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } } @@ -398,10 +405,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; - tsdbReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); - STableScanInfo* pTableScanInfo = pOperator->info; - if (taosHashGetSize(pTableScanInfo->pIgnoreTables) == taosArrayGetSize(pTableScanInfo->base.pTableListInfo->pTableList)) { + STableScanInfo* p1 = pOperator->info; + if (taosHashGetSize(p1->pIgnoreTables) == taosArrayGetSize(p1->base.pTableListInfo->pTableList)) { *status = FUNC_DATA_REQUIRED_ALL_FILTEROUT; } else { *status = FUNC_DATA_REQUIRED_FILTEROUT; @@ -412,7 +419,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->totalCheckedRows += pBlock->info.rows; pCost->loadBlocks += 1; - SSDataBlock* p = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL); + SSDataBlock* p = pAPI->storeReader.storeReaderRetrieveDataBlock(pTableScanInfo->dataReader, NULL); if (p == NULL) { return terrno; } @@ -524,8 +531,8 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int // 1. check if it is existed in meta cache if (pCache == NULL) { - metaReaderInit(&mr, pHandle->meta, 0); - code = metaGetTableEntryByUidCache(&mr, pBlock->info.id.uid); + pHandle->api.metaReaderFn.initReader(&mr, pHandle->meta, 0); + code = pHandle->api.metaReaderFn.readerGetEntryGetUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { // when encounter the TSDB_CODE_PAR_TABLE_NOT_EXIST error, we proceed. if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { @@ -538,11 +545,11 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno), idStr); } - metaReaderClear(&mr); + pHandle->api.metaReaderFn.clearReader(&mr); return terrno; } - metaReaderReleaseLock(&mr); + pHandle->api.metaReaderFn.readerReleaseLock(&mr); val.pName = mr.me.name; val.pTags = (STag*)mr.me.ctbEntry.pTags; @@ -553,8 +560,8 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int h = taosLRUCacheLookup(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid)); if (h == NULL) { - metaReaderInit(&mr, pHandle->meta, 0); - code = metaGetTableEntryByUidCache(&mr, pBlock->info.id.uid); + pHandle->api.metaReaderFn.initReader(&mr, pHandle->meta, 0); + code = pHandle->api.metaReaderFn.readerGetEntryGetUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", @@ -565,11 +572,11 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno), idStr); } - metaReaderClear(&mr); + pHandle->api.metaReaderFn.clearReader(&mr); return terrno; } - metaReaderReleaseLock(&mr); + pHandle->api.metaReaderFn.readerReleaseLock(&mr); STableCachedVal* pVal = createTableCacheVal(&mr); @@ -609,7 +616,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int } else { // these are tags STagVal tagVal = {0}; tagVal.cid = pExpr1->base.pParam[0].pCol->colId; - const char* p = metaGetTableTagVal(val.pTags, pColInfoData->info.type, &tagVal); + const char* p = pHandle->api.metaFn.extractTagVal(val.pTags, pColInfoData->info.type, &tagVal); char* data = NULL; if (pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL) { @@ -628,7 +635,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int } if (code) { if (freeReader) { - metaReaderClear(&mr); + pHandle->api.metaReaderFn.clearReader(&mr); } return code; } @@ -643,7 +650,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int // restore the rows pBlock->info.rows = backupRows; if (freeReader) { - metaReaderClear(&mr); + pHandle->api.metaReaderFn.clearReader(&mr); } return TSDB_CODE_SUCCESS; @@ -677,6 +684,8 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SSDataBlock* pBlock = pTableScanInfo->pResBlock; bool hasNext = false; int32_t code = TSDB_CODE_SUCCESS; @@ -684,9 +693,9 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { int64_t st = taosGetTimestampUs(); while (true) { - code = tsdbNextDataBlock(pTableScanInfo->base.dataReader, &hasNext); + code = pAPI->storeReader.storeReaderNextDataBlock(pTableScanInfo->base.dataReader, &hasNext); if (code) { - tsdbReleaseDataBlock(pTableScanInfo->base.dataReader); + pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); T_LONG_JMP(pTaskInfo->env, code); } @@ -695,12 +704,12 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { } if (isTaskKilled(pTaskInfo)) { - tsdbReleaseDataBlock(pTableScanInfo->base.dataReader); + pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } if (pOperator->status == OP_EXEC_DONE) { - tsdbReleaseDataBlock(pTableScanInfo->base.dataReader); + pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); break; } @@ -733,14 +742,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { pTableScanInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; pOperator->cost.totalCost = pTableScanInfo->base.readRecorder.elapsedTime; - - // todo refactor - /*pTableScanInfo->lastStatus.uid = pBlock->info.id.uid;*/ - /*pTableScanInfo->lastStatus.ts = pBlock->info.window.ekey;*/ - // pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA; - // pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid; - // pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey; - return pBlock; } return NULL; @@ -749,7 +750,8 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + // The read handle is not initialized yet, since no qualified tables exists if (pTableScanInfo->base.dataReader == NULL || pOperator->status == OP_EXEC_DONE) { return NULL; @@ -772,7 +774,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { qDebug("start to repeat ascending order scan data blocks due to query func required, %s", GET_TASKID(pTaskInfo)); // do prepare for the next round table scan operation - tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pAPI->storeReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); } } @@ -780,7 +782,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { if (pTableScanInfo->scanTimes < total) { if (pTableScanInfo->base.cond.order == TSDB_ORDER_ASC) { prepareForDescendingScan(&pTableScanInfo->base, pOperator->exprSupp.pCtx, 0); - tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pAPI->storeReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); } @@ -798,7 +800,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { pTableScanInfo->base.scanFlag = MAIN_SCAN; qDebug("%s start to repeat descending order scan data blocks", GET_TASKID(pTaskInfo)); - tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pAPI->storeReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); } } } @@ -809,6 +811,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; // scan table one by one sequentially if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) { @@ -836,11 +839,11 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable); taosRUnLockLatch(&pTaskInfo->lock); - tsdbSetTableList(pInfo->base.dataReader, &tInfo, 1); + pAPI->storeReader.storeReaderSetTableList(pInfo->base.dataReader, &tInfo, 1); qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", tInfo.uid, numOfTables, pInfo->currentTable, numOfTables, GET_TASKID(pTaskInfo)); - tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond); + pAPI->storeReader.storeReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; } } else { // scan table group by group sequentially @@ -855,8 +858,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); ASSERT(pInfo->base.dataReader == NULL); - int32_t code = tsdbReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, - (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly, &pInfo->pIgnoreTables); + int32_t code = pAPI->storeReader.storeReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, + (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly, &pInfo->pIgnoreTables); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -884,8 +887,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableKeyInfo* pList = NULL; tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); - tsdbSetTableList(pInfo->base.dataReader, pList, num); - tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond); + pAPI->storeReader.storeReaderSetTableList(pInfo->base.dataReader, pList, num); + pAPI->storeReader.storeReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; result = doGroupedTableScan(pOperator); @@ -907,10 +910,10 @@ static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptr return 0; } -static void destroyTableScanBase(STableScanBase* pBase) { +static void destroyTableScanBase(STableScanBase* pBase, SStoreDataReaderFn* pAPI) { cleanupQueryTableDataCond(&pBase->cond); - tsdbReaderClose(pBase->dataReader); + pAPI->storeReaderClose(pBase->dataReader); pBase->dataReader = NULL; if (pBase->matchInfo.pList != NULL) { @@ -926,7 +929,7 @@ static void destroyTableScanOperatorInfo(void* param) { STableScanInfo* pTableScanInfo = (STableScanInfo*)param; blockDataDestroy(pTableScanInfo->pResBlock); taosHashCleanup(pTableScanInfo->pIgnoreTables); - destroyTableScanBase(&pTableScanInfo->base); + destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->readerAPI); taosMemoryFreeClear(param); } @@ -1071,8 +1074,7 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint6 pTableScanInfo->base.cond.endVersion = version; pTableScanInfo->scanTimes = 0; pTableScanInfo->currentGroupId = -1; - tsdbReaderClose(pTableScanInfo->base.dataReader); - qDebug("1"); + pTableScanInfo->readerAPI.storeReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; } @@ -1088,11 +1090,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU cond.twindows = (STimeWindow){.skey = startTs, .ekey = endTs}; SExecTaskInfo* pTaskInfo = pTableScanOp->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; SSDataBlock* pBlock = pTableScanInfo->pResBlock; STsdbReader* pReader = NULL; - int32_t code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, - (STsdbReader**)&pReader, GET_TASKID(pTaskInfo), false, NULL); + int32_t code = pAPI->storeReader.storeReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, + (void**)&pReader, GET_TASKID(pTaskInfo), false, NULL); if (code != TSDB_CODE_SUCCESS) { terrno = code; T_LONG_JMP(pTaskInfo->env, code); @@ -1100,7 +1103,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } bool hasNext = false; - code = tsdbNextDataBlock(pReader, &hasNext); + code = pAPI->storeReader.storeReaderNextDataBlock(pReader, &hasNext); if (code != TSDB_CODE_SUCCESS) { terrno = code; T_LONG_JMP(pTaskInfo->env, code); @@ -1108,12 +1111,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } if (hasNext) { - /*SSDataBlock* p = */ tsdbRetrieveDataBlock(pReader, NULL); + /*SSDataBlock* p = */ pAPI->storeReader.storeReaderRetrieveDataBlock(pReader, NULL); doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); pBlock->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); } - tsdbReaderClose(pReader); + pAPI->storeReader.storeReaderClose(pReader); qDebug("retrieve prev rows:%" PRId64 ", skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64 ", suid:%" PRIu64, pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid); @@ -1187,7 +1190,8 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ break; } - resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion); + ASSERT(0); +// resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion); pInfo->pTableScanOp->status = OP_OPENED; return true; } @@ -1245,8 +1249,7 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 *pRowIndex = 0; pInfo->updateWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - tsdbReaderClose(pTableScanInfo->base.dataReader); - qDebug("2"); + pTableScanInfo->readerAPI.storeReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; return NULL; } @@ -1291,8 +1294,8 @@ static int32_t getPreSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, pKey->win.ekey = endTs; pKey->groupId = groupId; - SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, pKey); - int32_t code = streamStateSessionGetKVByCur(pCur, pKey, NULL, 0); + void* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, pKey); + int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, pKey, NULL, 0); if (code != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_KEY_INVALID(pKey); } @@ -1463,11 +1466,11 @@ static int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pS } if (pInfo->tbnameCalSup.pExprInfo) { void* parTbname = NULL; - streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname); + pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname); memcpy(varDataVal(tbname), parTbname, TSDB_TABLE_NAME_LEN); varDataSetLen(tbname, strlen(varDataVal(tbname))); - streamFreeVal(parTbname); + pInfo->stateStore.streamStateFreeVal(parTbname); } appendOneRowToStreamSpecialBlock(pDestBlock, srcStartTsCol + i, srcEndTsCol + i, srcUidData + i, &groupId, tbname[0] == 0 ? NULL : tbname); @@ -1498,7 +1501,7 @@ static void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { pBlock->info.parTbName[0] = 0; } else { appendCreateTableRow(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, - pBlock->info.id.groupId, pBlock, 0, pInfo->pCreateTbRes); + pBlock->info.id.groupId, pBlock, 0, pInfo->pCreateTbRes, &pInfo->stateStore); } } @@ -1529,7 +1532,7 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); TSKEY* tsCol = (TSKEY*)pColDataInfo->pData; - bool tableInserted = updateInfoIsTableInserted(pInfo->pUpdateInfo, pBlock->info.id.uid); + bool tableInserted = pInfo->stateStore.updateInfoIsTableInserted(pInfo->pUpdateInfo, pBlock->info.id.uid); for (int32_t rowId = 0; rowId < pBlock->info.rows; rowId++) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; @@ -1545,10 +1548,9 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock isClosed = isCloseWindow(&win, &pInfo->twAggSup); } // must check update info first. - bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.id.uid, tsCol[rowId]); + bool update = pInfo->stateStore.updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.id.uid, tsCol[rowId]); bool closedWin = isClosed && isSignleIntervalWindow(pInfo) && - isDeletedStreamWindow(&win, pBlock->info.id.groupId, - pInfo->pState, &pInfo->twAggSup); + isDeletedStreamWindow(&win, pBlock->info.id.groupId, pInfo->pState, &pInfo->twAggSup, &pInfo->stateStore); if ((update || closedWin) && out) { qDebug("stream update check not pass, update %d, closedWin %d", update, closedWin); uint64_t gpId = 0; @@ -1636,58 +1638,30 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock } static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SStreamScanInfo* pInfo = pOperator->info; const char* id = GET_TASKID(pTaskInfo); qDebug("start to exec queue scan, %s", id); -#if 0 - if (pTaskInfo->streamInfo.submit.msgStr != NULL) { - if (pInfo->tqReader->msg.msgStr == NULL) { - SPackedData submit = pTaskInfo->streamInfo.submit; - if (tqReaderSetSubmitMsg(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) { - qError("submit msg messed up when initing stream submit block %p", submit.msgStr); - return NULL; - } - } - - blockDataCleanup(pInfo->pRes); - SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; - - while (tqNextBlockImpl(pInfo->tqReader, NULL)) { - int32_t code = tqRetrieveDataBlock(pInfo->tqReader, NULL); - if (code != TSDB_CODE_SUCCESS || pInfo->tqReader->pResBlock->info.rows == 0) { - continue; - } - - setBlockIntoRes(pInfo, pInfo->tqReader->pResBlock, true); - - if (pBlockInfo->rows > 0) { - return pInfo->pRes; - } - } - - pInfo->tqReader->msg = (SPackedData){0}; - pTaskInfo->streamInfo.submit = (SPackedData){0}; - return NULL; - } -#endif - if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); if (pResult && pResult->info.rows > 0) { - qDebug("queue scan tsdb return %" PRId64 " rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, - pResult->info.rows, pResult->info.window.skey, pResult->info.window.ekey, - pInfo->tqReader->pWalReader->curVersion); + ASSERT(0); +// qDebug("queue scan tsdb return %" PRId64 " rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, +// pResult->info.rows, pResult->info.window.skey, pResult->info.window.ekey, +// pInfo->tqReader->pWalReader->curVersion); tqOffsetResetToData(&pTaskInfo->streamInfo.currentOffset, pResult->info.id.uid, pResult->info.window.ekey); return pResult; } STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - tsdbReaderClose(pTSInfo->base.dataReader); + pAPI->storeReader.storeReaderClose(pTSInfo->base.dataReader); + pTSInfo->base.dataReader = NULL; qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1); - if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) { + if (pAPI->tqReaderFn.tqReaderSeek(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) { return NULL; } @@ -1697,11 +1671,14 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__LOG) { while (1) { - bool hasResult = tqNextBlockInWal(pInfo->tqReader, id); - SSDataBlock* pRes = pInfo->tqReader->pResBlock; + bool hasResult = pAPI->tqReaderFn.tqReaderNextBlockInWal(pInfo->tqReader, id); + ASSERT(0); + + SSDataBlock* pRes = NULL; + struct SWalReader* pWalReader = pAPI->tqReaderFn.tqReaderGetWalReader(pInfo->tqReader); // curVersion move to next, so currentOffset = curVersion - 1 - tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pInfo->tqReader->pWalReader->curVersion - 1); + tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pWalReader->curVersion - 1); if (hasResult) { qDebug("doQueueScan get data from log %" PRId64 " rows, version:%" PRId64, pRes->info.rows, @@ -1737,9 +1714,10 @@ static int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, S SColumnInfoData* pDstStartCol = taosArrayGet(pDst->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pDstEndCol = taosArrayGet(pDst->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pDstUidCol = taosArrayGet(pDst->pDataBlock, UID_COLUMN_INDEX); - int32_t j = 0; + + int32_t j = 0; for (int32_t i = 0; i < rows; i++) { - if (taosHashGet(pReader->tbIdHash, &uidCol[i], sizeof(uint64_t))) { + if (pInfo->readerFn.tqReaderIsQueriedTable(pReader, uidCol[i])) { colDataSetVal(pDstStartCol, j, (const char*)&startCol[i], false); colDataSetVal(pDstEndCol, j, (const char*)&endCol[i], false); colDataSetVal(pDstUidCol, j, (const char*)&uidCol[i], false); @@ -1750,6 +1728,7 @@ static int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, S j++; } } + uint32_t cap = pDst->info.capacity; pDst->info = pSrc->info; pDst->info.rows = j; @@ -1777,7 +1756,8 @@ static void setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBlock) { if (!pInfo->igCheckUpdate && pInfo->pUpdateInfo) { - pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); + ASSERT(0); +// pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); checkUpdateData(pInfo, true, pBlock, true); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, endKey); if (pInfo->pUpdateDataRes->info.rows > 0) { @@ -1794,21 +1774,21 @@ static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBl } } -int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff) { - int32_t len = updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo); - *pBuff = taosMemoryCalloc(1, len); - updateInfoSerialize(*pBuff, len, pInfo->pUpdateInfo); - return len; -} +//int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff) { +// int32_t len = updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo); +// *pBuff = taosMemoryCalloc(1, len); +// updateInfoSerialize(*pBuff, len, pInfo->pUpdateInfo); +// return len; +//} // other properties are recovered from the execution plan -void streamScanOperatorDeocde(void* pBuff, int32_t len, SStreamScanInfo* pInfo) { +void streamScanOperatorDecode(void* pBuff, int32_t len, SStreamScanInfo* pInfo) { if (!pBuff || len == 0) { return; } - SUpdateInfo* pUpInfo = updateInfoInit(0, TSDB_TIME_PRECISION_MILLI, 0); - int32_t code = updateInfoDeserialize(pBuff, len, pUpInfo); + void* pUpInfo = pInfo->stateStore.updateInfoInit(0, TSDB_TIME_PRECISION_MILLI, 0); + int32_t code = pInfo->stateStore.updateInfoDeserialize(pBuff, len, pUpInfo); if (code == TSDB_CODE_SUCCESS) { pInfo->pUpdateInfo = pUpInfo; } @@ -1817,6 +1797,8 @@ void streamScanOperatorDeocde(void* pBuff, int32_t len, SStreamScanInfo* pInfo) static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SStreamScanInfo* pInfo = pOperator->info; qDebug("stream scan started, %s", GET_TASKID(pTaskInfo)); @@ -1839,7 +1821,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN2; } - tsdbReaderClose(pTSInfo->base.dataReader); + pAPI->storeReader.storeReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; pInfo->pTableScanOp->status = OP_OPENED; @@ -1902,7 +1884,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { calBlockTbName(pInfo, pInfo->pRecoverRes); if (pInfo->pUpdateInfo) { if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__SCAN1) { - TSKEY maxTs = updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex); + TSKEY maxTs = pAPI->stateStore.updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); } else { doCheckUpdate(pInfo, pInfo->pRecoverRes->info.window.ekey, pInfo->pRecoverRes); @@ -1919,7 +1901,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { } pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - tsdbReaderClose(pTSInfo->base.dataReader); + pAPI->storeReader.storeReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; @@ -1943,7 +1925,7 @@ FETCH_NEXT_BLOCK: SPackedData* pPacked = taosArrayGet(pInfo->pBlockLists, current); SSDataBlock* pBlock = pPacked->pDataBlock; if (pBlock->info.parTbName[0]) { - streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, pBlock->info.parTbName); + pAPI->stateStore.streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, pBlock->info.parTbName); } // TODO move into scan @@ -1951,7 +1933,8 @@ FETCH_NEXT_BLOCK: pBlock->info.calWin.ekey = INT64_MAX; pBlock->info.dataLoad = 1; if (pInfo->pUpdateInfo) { - pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); + ASSERT(0); +// pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); } blockDataUpdateTsWindow(pBlock, 0); switch (pBlock->info.type) { @@ -1964,7 +1947,7 @@ FETCH_NEXT_BLOCK: copyDataBlock(pInfo->pUpdateRes, pBlock); pInfo->updateResIndex = 0; prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); - updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo); + pAPI->stateStore.updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo); } break; case STREAM_DELETE_DATA: { printDataBlock(pBlock, "stream scan delete recv"); @@ -2081,9 +2064,9 @@ FETCH_NEXT_BLOCK: NEXT_SUBMIT_BLK: while (1) { - if (pInfo->tqReader->msg.msgStr == NULL) { + if (pInfo->readerFn.tqReaderCurrentBlockConsumed(pInfo->tqReader)) { if (pInfo->validBlockIndex >= totalBlocks) { - updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); + pAPI->stateStore.updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); doClearBufferedBlocks(pInfo); qDebug("stream scan return empty, all %d submit blocks consumed, %s", totalBlocks, id); @@ -2094,7 +2077,7 @@ FETCH_NEXT_BLOCK: SPackedData* pSubmit = taosArrayGet(pInfo->pBlockLists, current); qDebug("set %d/%d as the input submit block, %s", current, totalBlocks, id); - if (tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) { + if (pAPI->tqReaderFn.tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) { qError("submit msg messed up when initializing stream submit block %p, current %d/%d, %s", pSubmit, current, totalBlocks, id); continue; } @@ -2102,13 +2085,14 @@ FETCH_NEXT_BLOCK: blockDataCleanup(pBlock); - while (tqNextBlockImpl(pInfo->tqReader, id)) { - int32_t code = tqRetrieveDataBlock(pInfo->tqReader, id); - if (code != TSDB_CODE_SUCCESS || pInfo->tqReader->pResBlock->info.rows == 0) { + while (pAPI->tqReaderFn.tqNextBlockImpl(pInfo->tqReader, id)) { + SSDataBlock* pRes = NULL; + int32_t code = pAPI->tqReaderFn.tqRetrieveBlock(pInfo->tqReader, &pRes, id); + if (code != TSDB_CODE_SUCCESS || pRes->info.rows == 0) { continue; } - setBlockIntoRes(pInfo, pInfo->tqReader->pResBlock, false); + setBlockIntoRes(pInfo, pRes, false); if (pInfo->pCreateTbRes->info.rows > 0) { pInfo->scanMode = STREAM_SCAN_FROM_RES; @@ -2166,7 +2150,9 @@ static SArray* extractTableIdList(const STableListInfo* pTableListInfo) { static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SStreamRawScanInfo* pInfo = pOperator->info; int32_t code = TSDB_CODE_SUCCESS; pTaskInfo->streamInfo.metaRsp.metaRspLen = 0; // use metaRspLen !=0 to judge if data is meta @@ -2176,20 +2162,20 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { bool hasNext = false; if (pInfo->dataReader) { - code = tsdbNextDataBlock(pInfo->dataReader, &hasNext); + code = pAPI->storeReader.storeReaderNextDataBlock(pInfo->dataReader, &hasNext); if (code) { - tsdbReleaseDataBlock(pInfo->dataReader); + pAPI->storeReader.storeReaderReleaseDataBlock(pInfo->dataReader); T_LONG_JMP(pTaskInfo->env, code); } } if (pInfo->dataReader && hasNext) { if (isTaskKilled(pTaskInfo)) { - tsdbReleaseDataBlock(pInfo->dataReader); + pAPI->storeReader.storeReaderReleaseDataBlock(pInfo->dataReader); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } - SSDataBlock* pBlock = tsdbRetrieveDataBlock(pInfo->dataReader, NULL); + SSDataBlock* pBlock = pAPI->storeReader.storeReaderRetrieveDataBlock(pInfo->dataReader, NULL); if (pBlock == NULL) { T_LONG_JMP(pTaskInfo->env, terrno); } @@ -2199,7 +2185,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { return pBlock; } - SMetaTableInfo mtInfo = getUidfromSnapShot(pInfo->sContext); + SMetaTableInfo mtInfo = pAPI->snapshotFn.storeSSGetTableInfo(pInfo->sContext); STqOffsetVal offset = {0}; if (mtInfo.uid == 0) { // read snapshot done, change to get data from wal qDebug("tmqsnap read snapshot done, change to get data from wal"); @@ -2217,7 +2203,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { int32_t dataLen = 0; int16_t type = 0; int64_t uid = 0; - if (getMetafromSnapShot(sContext, &data, &dataLen, &type, &uid) < 0) { + if (pAPI->snapshotFn.storeSSGetMetaInfo(sContext, &data, &dataLen, &type, &uid) < 0) { qError("tmqsnap getMetafromSnapShot error"); taosMemoryFreeClear(data); return NULL; @@ -2241,8 +2227,8 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { static void destroyRawScanOperatorInfo(void* param) { SStreamRawScanInfo* pRawScan = (SStreamRawScanInfo*)param; - tsdbReaderClose(pRawScan->dataReader); - destroySnapContext(pRawScan->sContext); + pRawScan->pAPI->storeReader.storeReaderClose(pRawScan->dataReader); + pRawScan->pAPI->snapshotFn.storeDestroySnapshot(pRawScan->sContext); tableListDestroy(pRawScan->pTableListInfo); taosMemoryFree(pRawScan); } @@ -2290,7 +2276,7 @@ static void destroyStreamScanOperatorInfo(void* param) { } if (pStreamScan->tqReader) { - tqCloseReader(pStreamScan->tqReader); + pStreamScan->readerFn.tqReaderClose(pStreamScan->tqReader); } if (pStreamScan->matchInfo.pList) { taosArrayDestroy(pStreamScan->matchInfo.pList); @@ -2303,7 +2289,7 @@ static void destroyStreamScanOperatorInfo(void* param) { cleanupExprSupp(&pStreamScan->tbnameCalSup); cleanupExprSupp(&pStreamScan->tagCalSup); - updateInfoDestroy(pStreamScan->pUpdateInfo); + pStreamScan->stateStore.updateInfoDestroy(pStreamScan->pUpdateInfo); blockDataDestroy(pStreamScan->pRes); blockDataDestroy(pStreamScan->pUpdateRes); blockDataDestroy(pStreamScan->pPullDataRes); @@ -2319,6 +2305,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys SArray* pColIds = NULL; SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SStorageAPI* pAPI = &pTaskInfo->storageAPI; if (pInfo == NULL || pOperator == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -2408,7 +2395,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys if (pHandle->initTqReader) { ASSERT(pHandle->tqReader == NULL); - pInfo->tqReader = tqReaderOpen(pHandle->vnode); + pInfo->tqReader = pAPI->tqReaderFn.tqReaderOpen(pHandle->vnode); ASSERT(pInfo->tqReader); } else { ASSERT(pHandle->tqReader); @@ -2418,7 +2405,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->pUpdateInfo = NULL; pInfo->pTableScanOp = pTableScanOp; if (pInfo->pTableScanOp->pTaskInfo->streamInfo.pState) { - streamStateSetNumber(pInfo->pTableScanOp->pTaskInfo->streamInfo.pState, -1); + pAPI->stateStore.streamStateSetNumber(pInfo->pTableScanOp->pTaskInfo->streamInfo.pState, -1); } pInfo->readHandle = *pHandle; @@ -2427,9 +2414,9 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys blockDataEnsureCapacity(pInfo->pCreateTbRes, 8); // set the extract column id to streamHandle - tqReaderSetColIdList(pInfo->tqReader, pColIds); + pAPI->tqReaderFn.tqReaderSetColIdList(pInfo->tqReader, pColIds); SArray* tableIdList = extractTableIdList(((STableScanInfo*)(pInfo->pTableScanOp->info))->base.pTableListInfo); - code = tqReaderSetTbUidList(pInfo->tqReader, tableIdList); + code = pAPI->tqReaderFn.tqReaderSetTargetTableList(pInfo->tqReader, tableIdList); if (code != 0) { taosArrayDestroy(tableIdList); goto _error; @@ -2475,8 +2462,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys if (pTaskInfo->streamInfo.pState) { void* buff = NULL; int32_t len = 0; - streamStateGetInfo(pTaskInfo->streamInfo.pState, STREAM_SCAN_OP_NAME, strlen(STREAM_SCAN_OP_NAME), &buff, &len); - streamScanOperatorDeocde(buff, len, pInfo); + pAPI->stateStore.streamStateGetInfo(pTaskInfo->streamInfo.pState, STREAM_SCAN_OP_NAME, strlen(STREAM_SCAN_OP_NAME), &buff, &len); + streamScanOperatorDecode(buff, len, pInfo); } setOperatorInfo(pOperator, STREAM_SCAN_OP_NAME, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, false, OP_NOT_OPENED, pInfo, @@ -2502,18 +2489,18 @@ _error: return NULL; } -static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes, int32_t count, SMetaReader* mr) { +static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes, int32_t count, SMetaReader* mr, SStorageAPI* pAPI) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STagScanInfo* pInfo = pOperator->info; SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0]; STableKeyInfo* item = tableListGetInfo(pInfo->pTableListInfo, pInfo->curPos); - int32_t code = metaGetTableEntryByUid(mr, item->uid); + int32_t code = pAPI->metaReaderFn.getTableEntryByUid(mr, item->uid); tDecoderClear(&(*mr).coder); if (code != TSDB_CODE_SUCCESS) { qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - metaReaderClear(mr); + pAPI->metaReaderFn.clearReader(mr); T_LONG_JMP(pTaskInfo->env, terrno); } @@ -2528,7 +2515,7 @@ static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes, } else { // it is a tag value STagVal val = {0}; val.cid = pExprInfo[j].base.pParam[0].pCol->colId; - const char* p = metaGetTableTagVal((*mr).me.ctbEntry.pTags, pDst->info.type, &val); + const char* p = pAPI->metaFn.extractTagVal((*mr).me.ctbEntry.pTags, pDst->info.type, &val); char* data = NULL; if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL) { @@ -2553,6 +2540,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { } SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; STagScanInfo* pInfo = pOperator->info; SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0]; @@ -2568,10 +2556,10 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { char str[512] = {0}; int32_t count = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, pInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.meta, 0); while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { - doTagScanOneTable(pOperator, pRes, count, &mr); + doTagScanOneTable(pOperator, pRes, count, &mr, &pTaskInfo->storageAPI); ++count; if (++pInfo->curPos >= size) { setOperatorCompleted(pOperator); @@ -2589,7 +2577,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { } } - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); // qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); if (pOperator->status == OP_EXEC_DONE) { @@ -2661,6 +2649,8 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { SOperatorInfo* pOperator = source->pOperator; STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + int32_t readIdx = source->readerIdx; SSDataBlock* pBlock = source->inputBlock; int32_t code = 0; @@ -2672,7 +2662,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { SReadHandle* pHandle = &pInfo->base.readHandle; if (NULL == source->dataReader || !source->multiReader) { - code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &source->dataReader, GET_TASKID(pTaskInfo), false, NULL); + code = pAPI->storeReader.storeReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, (void**)&source->dataReader, GET_TASKID(pTaskInfo), false, NULL); if (code != 0) { T_LONG_JMP(pTaskInfo->env, code); } @@ -2684,9 +2674,9 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { qTrace("tsdb/read-table-data: %p, enter next reader", reader); while (true) { - code = tsdbNextDataBlock(reader, &hasNext); + code = pAPI->storeReader.storeReaderNextDataBlock(reader, &hasNext); if (code != 0) { - tsdbReleaseDataBlock(reader); + pAPI->storeReader.storeReaderReleaseDataBlock(reader); pInfo->base.dataReader = NULL; T_LONG_JMP(pTaskInfo->env, code); } @@ -2696,7 +2686,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } if (isTaskKilled(pTaskInfo)) { - tsdbReleaseDataBlock(reader); + pAPI->storeReader.storeReaderReleaseDataBlock(reader); pInfo->base.dataReader = NULL; T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } @@ -2736,7 +2726,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { qTrace("tsdb/read-table-data: %p, close reader", reader); if (!source->multiReader) { - tsdbReaderClose(pInfo->base.dataReader); + pAPI->storeReader.storeReaderClose(pInfo->base.dataReader); source->dataReader = NULL; } pInfo->base.dataReader = NULL; @@ -2744,7 +2734,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } if (!source->multiReader) { - tsdbReaderClose(pInfo->base.dataReader); + pAPI->storeReader.storeReaderClose(pInfo->base.dataReader); source->dataReader = NULL; } pInfo->base.dataReader = NULL; @@ -2849,6 +2839,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; int32_t numOfTable = taosArrayGetSize(pInfo->queryConds); @@ -2862,7 +2853,7 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { for (int32_t i = 0; i < numOfTable; ++i) { STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i); blockDataDestroy(param->inputBlock); - tsdbReaderClose(param->dataReader); + pAPI->storeReader.storeReaderClose(param->dataReader); param->dataReader = NULL; } taosArrayClear(pInfo->sortSourceParams); @@ -2974,11 +2965,11 @@ void destroyTableMergeScanOperatorInfo(void* param) { for (int32_t i = 0; i < numOfTable; i++) { STableMergeScanSortSourceParam* p = taosArrayGet(pTableScanInfo->sortSourceParams, i); blockDataDestroy(p->inputBlock); - tsdbReaderClose(p->dataReader); + pTableScanInfo->base.readerAPI.storeReaderClose(p->dataReader); p->dataReader = NULL; } - tsdbReaderClose(pTableScanInfo->base.dataReader); + pTableScanInfo->base.readerAPI.storeReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; taosArrayDestroy(pTableScanInfo->sortSourceParams); @@ -2991,7 +2982,7 @@ void destroyTableMergeScanOperatorInfo(void* param) { } taosArrayDestroy(pTableScanInfo->queryConds); - destroyTableScanBase(&pTableScanInfo->base); + destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock); pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock); @@ -3359,9 +3350,11 @@ static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCount const char* db = NULL; int32_t vgId = 0; char dbName[TSDB_DB_NAME_LEN] = {0}; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; // get dbname - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId); SName sn = {0}; tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); tNameGetDbName(&sn, dbName); @@ -3376,10 +3369,13 @@ static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCount static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + if (pSupp->groupByStbName) { if (pInfo->stbUidList == NULL) { pInfo->stbUidList = taosArrayInit(16, sizeof(tb_uid_t)); - if (vnodeGetStbIdList(pInfo->readHandle.vnode, 0, pInfo->stbUidList) < 0) { + if (pAPI->metaFn.storeGetTableList(pInfo->readHandle.vnode, 0, pInfo->stbUidList, TSDB_SUPER_TABLE) < 0) { qError("vgId:%d, failed to get stb id list error: %s", vgId, terrstr()); } } @@ -3398,7 +3394,9 @@ static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountSca } else { uint64_t groupId = calcGroupId(dbName, strlen(dbName)); pRes->info.id.groupId = groupId; - int64_t dbTableCount = metaGetTbNum(pInfo->readHandle.meta); + + int64_t dbTableCount = 0; + pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.meta, &dbTableCount); fillTableCountScanDataBlock(pSupp, dbName, "", dbTableCount, pRes); setOperatorCompleted(pOperator); } @@ -3406,19 +3404,25 @@ static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountSca static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + if (strlen(pSupp->dbNameFilter) != 0) { if (strlen(pSupp->stbNameFilter) != 0) { - tb_uid_t uid = metaGetTableEntryUidByName(pInfo->readHandle.meta, pSupp->stbNameFilter); + tb_uid_t uid = 0; + pAPI->metaFn.getTableUidByName(pInfo->readHandle.meta, pSupp->stbNameFilter, &uid); SMetaStbStats stats = {0}; - metaGetStbStats(pInfo->readHandle.meta, uid, &stats); + ASSERT(0); +// metaGetStbStats(pInfo->readHandle.meta, uid, &stats); int64_t ctbNum = stats.ctbNum; fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, ctbNum, pRes); } else { - int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta); + int64_t tbNumVnode = 0;//metaGetTbNum(pInfo->readHandle.meta); fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } } else { - int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta); + int64_t tbNumVnode = 0;//metaGetTbNum(pInfo->readHandle.meta); + pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode); fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } setOperatorCompleted(pOperator); @@ -3433,7 +3437,8 @@ static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, S uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); pRes->info.id.groupId = groupId; - int64_t ntbNum = metaGetNtbNum(pInfo->readHandle.meta); + int64_t ntbNum = 0;//metaGetNtbNum(pInfo->readHandle.meta); + ASSERT(0); if (ntbNum != 0) { fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes); } @@ -3442,7 +3447,8 @@ static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, S static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName, tb_uid_t stbUid) { char stbName[TSDB_TABLE_NAME_LEN] = {0}; - metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName); + ASSERT(0); +// metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName); char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; if (pSupp->groupByDbName) { @@ -3455,7 +3461,7 @@ static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, S pRes->info.id.groupId = groupId; SMetaStbStats stats = {0}; - metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats); +// metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats); int64_t ctbNum = stats.ctbNum; fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 04c6c619aa..460373b537 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -21,19 +21,19 @@ #include "querynodes.h" #include "systable.h" #include "tname.h" -#include "ttime.h" #include "tdatablock.h" #include "tmsg.h" +#include "operator.h" #include "query.h" +#include "querytask.h" +#include "storageapi.h" #include "tcompare.h" #include "thash.h" #include "ttypes.h" -#include "vnode.h" -#include "operator.h" -#include "querytask.h" - +#include "tsdstorage.h" +#include "index.h" typedef int (*__optSysFilter)(void* a, void* b, int16_t dtype); typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result); @@ -72,6 +72,7 @@ typedef struct SSysTableScanInfo { SLoadRemoteDataInfo loadInfo; SLimitInfo limitInfo; int32_t tbnameSlotId; + SStorageAPI* pAPI; } SSysTableScanInfo; typedef struct { @@ -156,7 +157,8 @@ int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) { void* pVnode = ((SSTabFltArg*)arg)->pVnode; const char* db = NULL; - vnodeGetInfo(pVnode, &db, NULL); + ASSERT(0); +// pAPI->metaFn.storeGetBasicInfo(pVnode, &db, NULL); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -183,7 +185,8 @@ int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) { void* pVnode = ((SSTabFltArg*)arg)->pVnode; int64_t vgId = 0; - vnodeGetInfo(pVnode, NULL, (int32_t*)&vgId); + ASSERT(0); +// pAPI->metaFn.storeGetBasicInfo(pVnode, NULL, (int32_t*)&vgId); SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; @@ -237,8 +240,10 @@ int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) { .equal = equal, .filterFunc = func}; - int32_t ret = metaFilterCreateTime(pMeta, ¶m, result); - return ret; + ASSERT(0); + return 0; +// int32_t ret = metaFilterCreateTime(pMeta, ¶m, result); +// return ret; } int32_t sysFilte__Ncolumn(void* arg, SNode* pNode, SArray* result) { @@ -431,8 +436,9 @@ static bool sysTableIsCondOnOneTable(SNode* pCond, char* condTable) { } static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { - qDebug("sysTableScanUserCols get cols start"); - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SSysTableScanInfo* pInfo = pOperator->info; if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -446,7 +452,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { const char* db = NULL; int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -461,18 +467,18 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { STR_TO_VARSTR(tableName, pInfo->req.filterTb); SMetaReader smrTable = {0}; - metaReaderInit(&smrTable, pInfo->readHandle.meta, 0); - int32_t code = metaGetTableEntryByName(&smrTable, pInfo->req.filterTb); + pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.meta, 0); + int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrTable, pInfo->req.filterTb); if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by metaGetTableEntryByName, therefore, return directly - metaReaderClear(&smrTable); + // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly + pAPI->metaReaderFn.clearReader(&smrTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; } if (smrTable.me.type == TSDB_SUPER_TABLE) { - metaReaderClear(&smrTable); + pAPI->metaReaderFn.clearReader(&smrTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; @@ -480,12 +486,12 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { if (smrTable.me.type == TSDB_CHILD_TABLE) { int64_t suid = smrTable.me.ctbEntry.suid; - metaReaderClear(&smrTable); - metaReaderInit(&smrTable, pInfo->readHandle.meta, 0); - code = metaGetTableEntryByUid(&smrTable, suid); + pAPI->metaReaderFn.clearReader(&smrTable); + pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.meta, 0); + code = pAPI->metaReaderFn.getTableEntryByUid(&smrTable, suid); if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by metaGetTableEntryByName, therefore, return directly - metaReaderClear(&smrTable); + // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly + pAPI->metaReaderFn.clearReader(&smrTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; @@ -503,7 +509,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { } sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName); - metaReaderClear(&smrTable); + pAPI->metaReaderFn.clearReader(&smrTable); if (numOfRows > 0) { relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); @@ -517,7 +523,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { int32_t ret = 0; if (pInfo->pCur == NULL) { - pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.meta); } if (pInfo->pSchema == NULL) { @@ -535,8 +541,12 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { int32_t restore = pInfo->restore; pInfo->restore = false; - while (restore || ((ret = metaTbCursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0)) { - if (restore) restore = false; + + while (restore || ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0)) { + if (restore) { + restore = false; + } + char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -560,12 +570,12 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { schemaRow = *(SSchemaWrapper**)schema; } else { SMetaReader smrSuperTable = {0}; - metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0); - int code = metaGetTableEntryByUid(&smrSuperTable, suid); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.meta, 0); + int code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid); if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by metaGetTableEntryByName, therefore, return directly + // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly qError("sysTableScanUserCols get meta by suid:%" PRId64 " error, code:%d", suid, code); - metaReaderClear(&smrSuperTable); + pAPI->metaReaderFn.clearReader(&smrSuperTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; @@ -573,7 +583,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { SSchemaWrapper* schemaWrapper = tCloneSSchemaWrapper(&smrSuperTable.me.stbEntry.schemaRow); taosHashPut(pInfo->pSchema, &suid, sizeof(int64_t), &schemaWrapper, POINTER_BYTES); schemaRow = schemaWrapper; - metaReaderClear(&smrSuperTable); + pAPI->metaReaderFn.clearReader(&smrSuperTable); } } else if (pInfo->pCur->mr.me.type == TSDB_NORMAL_TABLE) { qDebug("sysTableScanUserCols cursor get normal table"); @@ -605,7 +615,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { blockDataDestroy(dataBlock); if (ret != 0) { - metaCloseTbCursor(pInfo->pCur); + pAPI->metaFn.closeMetaCursor(pInfo->pCur); pInfo->pCur = NULL; setOperatorCompleted(pOperator); } @@ -618,6 +628,8 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SSysTableScanInfo* pInfo = pOperator->info; if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -631,7 +643,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { const char* db = NULL; int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -647,37 +659,37 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { STR_TO_VARSTR(tableName, condTableName); SMetaReader smrChildTable = {0}; - metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0); - int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName); + pAPI->metaReaderFn.initReader(&smrChildTable, pInfo->readHandle.meta, 0); + int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrChildTable, condTableName); if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by metaGetTableEntryByName, therefore, return directly - metaReaderClear(&smrChildTable); + // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly + pAPI->metaReaderFn.clearReader(&smrChildTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; } if (smrChildTable.me.type != TSDB_CHILD_TABLE) { - metaReaderClear(&smrChildTable); + pAPI->metaReaderFn.clearReader(&smrChildTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; } SMetaReader smrSuperTable = {0}; - metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK); - code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK); + code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by metaGetTableEntryByUid - metaReaderClear(&smrSuperTable); - metaReaderClear(&smrChildTable); + // terrno has been set by pAPI->metaReaderFn.getTableEntryByUid + pAPI->metaReaderFn.clearReader(&smrSuperTable); + pAPI->metaReaderFn.clearReader(&smrChildTable); blockDataDestroy(dataBlock); return NULL; } sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock); - metaReaderClear(&smrSuperTable); - metaReaderClear(&smrChildTable); + pAPI->metaReaderFn.clearReader(&smrSuperTable); + pAPI->metaReaderFn.clearReader(&smrChildTable); if (numOfRows > 0) { relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); @@ -691,11 +703,11 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { int32_t ret = 0; if (pInfo->pCur == NULL) { - pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.meta); } bool blockFull = false; - while ((ret = metaTbCursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) { + while ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) { if (pInfo->pCur->mr.me.type != TSDB_CHILD_TABLE) { continue; } @@ -704,27 +716,27 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); SMetaReader smrSuperTable = {0}; - metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.meta, 0); uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&smrSuperTable, suid); + int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - metaReaderClear(&smrSuperTable); - metaCloseTbCursor(pInfo->pCur); + pAPI->metaReaderFn.clearReader(&smrSuperTable); + pAPI->metaFn.closeMetaCursor(pInfo->pCur); pInfo->pCur = NULL; T_LONG_JMP(pTaskInfo->env, terrno); } if ((smrSuperTable.me.stbEntry.schemaTag.nCols + numOfRows) > pOperator->resultInfo.capacity) { - metaTbCursorPrev(pInfo->pCur, TSDB_TABLE_MAX); + pAPI->metaFn.cursorPrev(pInfo->pCur, TSDB_TABLE_MAX); blockFull = true; } else { sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, dataBlock); } - metaReaderClear(&smrSuperTable); + pAPI->metaReaderFn.clearReader(&smrSuperTable); if (blockFull || numOfRows >= pOperator->resultInfo.capacity) { relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); @@ -745,7 +757,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { blockDataDestroy(dataBlock); if (ret != 0) { - metaCloseTbCursor(pInfo->pCur); + pAPI->metaFn.closeMetaCursor(pInfo->pCur); pInfo->pCur = NULL; setOperatorCompleted(pOperator); } @@ -1090,6 +1102,8 @@ int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) { static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SSysTableScanInfo* pInfo = pOperator->info; SSysTableIndex* pIdx = pInfo->pIdx; @@ -1100,7 +1114,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { const char* db = NULL; int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -1118,10 +1132,10 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { tb_uid_t* uid = taosArrayGet(pIdx->uids, i); SMetaReader mr = {0}; - metaReaderInit(&mr, pInfo->readHandle.meta, 0); - ret = metaGetTableEntryByUid(&mr, *uid); + pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.meta, 0); + ret = pAPI->metaReaderFn.getTableEntryByUid(&mr, *uid); if (ret < 0) { - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); continue; } STR_TO_VARSTR(n, mr.me.name); @@ -1146,15 +1160,15 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false); SMetaReader mr1 = {0}; - metaReaderInit(&mr1, pInfo->readHandle.meta, META_READER_NOLOCK); + pAPI->metaReaderFn.initReader(&mr1, pInfo->readHandle.meta, META_READER_NOLOCK); int64_t suid = mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&mr1, suid); + int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr1, suid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - metaReaderClear(&mr1); - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr1); + pAPI->metaReaderFn.clearReader(&mr); T_LONG_JMP(pTaskInfo->env, terrno); } pColInfoData = taosArrayGet(p->pDataBlock, 3); @@ -1164,7 +1178,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { STR_TO_VARSTR(n, mr1.me.name); pColInfoData = taosArrayGet(p->pDataBlock, 4); colDataSetVal(pColInfoData, numOfRows, n, false); - metaReaderClear(&mr1); + pAPI->metaReaderFn.clearReader(&mr1); // table comment pColInfoData = taosArrayGet(p->pDataBlock, 8); @@ -1229,7 +1243,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { // impl later } - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); pColInfoData = taosArrayGet(p->pDataBlock, 9); colDataSetVal(pColInfoData, numOfRows, n, false); @@ -1275,10 +1289,11 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; SSysTableScanInfo* pInfo = pOperator->info; if (pInfo->pCur == NULL) { - pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.meta); } blockDataCleanup(pInfo->pRes); @@ -1286,7 +1301,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { const char* db = NULL; int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -1301,7 +1316,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t ret = 0; - while ((ret = metaTbCursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) { + while ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) { STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); // table name @@ -1324,15 +1339,15 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false); SMetaReader mr = {0}; - metaReaderInit(&mr, pInfo->readHandle.meta, META_READER_NOLOCK); + pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.meta, META_READER_NOLOCK); uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&mr, suid); + int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr, suid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - metaReaderClear(&mr); - metaCloseTbCursor(pInfo->pCur); + pAPI->metaReaderFn.clearReader(&mr); + pAPI->metaFn.closeMetaCursor(pInfo->pCur); pInfo->pCur = NULL; T_LONG_JMP(pTaskInfo->env, terrno); } @@ -1345,7 +1360,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { STR_TO_VARSTR(n, mr.me.name); pColInfoData = taosArrayGet(p->pDataBlock, 4); colDataSetVal(pColInfoData, numOfRows, n, false); - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); // table comment pColInfoData = taosArrayGet(p->pDataBlock, 8); @@ -1442,7 +1457,8 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found if (ret != 0) { - metaCloseTbCursor(pInfo->pCur); + pAPI->metaFn.closeMetaCursor(pInfo->pCur); + pAPI->metaFn.closeMetaCursor(pInfo->pCur); pInfo->pCur = NULL; setOperatorCompleted(pOperator); } @@ -1708,7 +1724,7 @@ static SSDataBlock* sysTableScanFromMNode(SOperatorInfo* pOperator, SSysTableSca SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo) { - int32_t code = TDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -1798,7 +1814,7 @@ void destroySysScanOperator(void* param) { if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { - metaCloseTbCursor(pInfo->pCur); + pInfo->pAPI->metaFn.closeMetaCursor(pInfo->pCur); pInfo->pCur = NULL; } if (pInfo->pIdx) { @@ -2124,15 +2140,15 @@ static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) { return -1; } -static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, const char* idstr) { +static int32_t doGetTableRowSize(SReadHandle *pHandle, uint64_t uid, int32_t* rowLen, const char* idstr) { *rowLen = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, pMeta, 0); - int32_t code = metaGetTableEntryByUid(&mr, uid); + pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0); + int32_t code = pHandle->api.metaReaderFn.getTableEntryByUid(&mr, uid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", uid, tstrerror(terrno), idstr); - metaReaderClear(&mr); + pHandle->api.metaReaderFn.clearReader(&mr); return terrno; } @@ -2144,10 +2160,10 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, con } else if (mr.me.type == TSDB_CHILD_TABLE) { uint64_t suid = mr.me.ctbEntry.suid; tDecoderClear(&mr.coder); - code = metaGetTableEntryByUid(&mr, suid); + code = pHandle->api.metaReaderFn.getTableEntryByUid(&mr, suid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), idstr); - metaReaderClear(&mr); + pHandle->api.metaReaderFn.clearReader(&mr); return terrno; } @@ -2163,7 +2179,7 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, con } } - metaReaderClear(&mr); + pHandle->api.metaReaderFn.clearReader(&mr); return TSDB_CODE_SUCCESS; } @@ -2174,16 +2190,17 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { SBlockDistInfo* pBlockScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN}; - int32_t code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid, + int32_t code = doGetTableRowSize(&pBlockScanInfo->readHandle, pBlockScanInfo->uid, (int32_t*)&blockDistInfo.rowSize, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } - tsdbGetFileBlocksDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); - blockDistInfo.numOfInmemRows = (int32_t)tsdbGetNumOfRowsInMemTable(pBlockScanInfo->pHandle); + pAPI->storeReader.storeReaderGetDataBlockDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); + blockDistInfo.numOfInmemRows = (int32_t) pAPI->metaFn.getNumOfRowsInMem(pBlockScanInfo->pHandle); SSDataBlock* pBlock = pBlockScanInfo->pResBlock; @@ -2213,7 +2230,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { static void destroyBlockDistScanOperatorInfo(void* param) { SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param; blockDataDestroy(pDistInfo->pResBlock); - tsdbReaderClose(pDistInfo->pHandle); + pDistInfo->readHandle.api.storeReader.storeReaderClose(pDistInfo->pHandle); tableListDestroy(pDistInfo->pTableListInfo); taosMemoryFreeClear(param); } @@ -2268,7 +2285,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi size_t num = tableListGetSize(pTableListInfo); void* pList = tableListGetInfo(pTableListInfo, 0); - code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str, false, NULL); + code = readHandle->api.storeReader.storeReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, (void**)&pInfo->pHandle, pTaskInfo->id.str, false, NULL); cleanupQueryTableDataCond(&cond); if (code != 0) { goto _error; diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 2698b0bfd9..209e5cb8fc 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -18,6 +18,7 @@ #include "functionMgt.h" #include "operator.h" #include "querytask.h" +#include "storageapi.h" #include "tcommon.h" #include "tcompare.h" #include "tdatablock.h" diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 7a9efde4f3..b4fd571e23 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1331,10 +1331,12 @@ static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, } static bool doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SStreamIntervalOperatorInfo* pInfo = pOperator->info; SWinKey key = {.ts = ts, .groupId = groupId}; tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey)); - streamStateDel(pInfo->pState, &key); + pAPI->stateStore.streamStateDel(pInfo->pState, &key); return true; } @@ -1471,7 +1473,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin for (int32_t i = *index; i < size; i++) { SWinKey* pWin = taosArrayGet(pWins, i); void* tbname = NULL; - streamStateGetParName(pInfo->pState, pWin->groupId, &tbname); + pInfo->statestore.streamStateGetParName(pInfo->pState, pWin->groupId, &tbname); if (tbname == NULL) { appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, NULL); } else { @@ -1479,7 +1481,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, parTbName); } - streamFreeVal(tbname); + pInfo->statestore.streamStateFreeVal(tbname); (*index)++; } } @@ -1533,7 +1535,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { blockDataDestroy(pInfo->pPullDataRes); taosArrayDestroy(pInfo->pDelWins); blockDataDestroy(pInfo->pDelRes); - streamFileStateDestroy(pInfo->pState->pFileState); + pInfo->statestore.streamFileStateDestroy(pInfo->pState->pFileState); taosMemoryFreeClear(pInfo->pState); nodesDestroyNode((SNode*)pInfo->pPhyNode); @@ -1608,16 +1610,20 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt } void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamIntervalOperatorInfo* pInfo) { + SStateStore* pAPI = &downstream->pTaskInfo->storageAPI.stateStore; + if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { initIntervalDownStream(downstream->pDownstream[0], type, pInfo); return; } + SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->windowSup.parentType = type; pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup; if (!pScanInfo->igCheckUpdate && !pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark); + pScanInfo->pUpdateInfo = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark); } + pScanInfo->interval = pInfo->interval; pScanInfo->twAggSup = pInfo->twAggSup; pScanInfo->pState = pInfo->pState; @@ -2045,33 +2051,33 @@ void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3 } } -bool hasIntervalWindow(SStreamState* pState, SWinKey* pKey) { return streamStateCheck(pState, pKey); } +bool hasIntervalWindow(void* pState, SWinKey* pKey, SStateStore* pStore) { return pStore->streamStateCheck(pState, pKey); } -int32_t setIntervalOutputBuf(SStreamState* pState, STimeWindow* win, SRowBuffPos** pResult, int64_t groupId, +int32_t setIntervalOutputBuf(void* pState, STimeWindow* win, SRowBuffPos** pResult, int64_t groupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, - SAggSupporter* pAggSup) { - SWinKey key = { - .ts = win->skey, - .groupId = groupId, - }; + SAggSupporter* pAggSup, SStateStore* pStore) { + + SWinKey key = { .ts = win->skey, .groupId = groupId }; char* value = NULL; int32_t size = pAggSup->resultRowSize; - if (streamStateAddIfNotExist(pState, &key, (void**)&value, &size) < 0) { + if (pStore->streamStateAddIfNotExist(pState, &key, (void**)&value, &size) < 0) { return TSDB_CODE_OUT_OF_MEMORY; } + *pResult = (SRowBuffPos*)value; SResultRow* res = (SResultRow*)((*pResult)->pRowBuff); + // set time window for current result res->win = (*win); setResultRowInitCtx(res, pCtx, numOfOutput, rowEntryInfoOffset); return TSDB_CODE_SUCCESS; } -bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup) { +bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup, SStateStore* pStore) { if (pTwSup->maxTs != INT64_MIN && pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) { SWinKey key = {.ts = pWin->skey, .groupId = groupId}; - if (!hasIntervalWindow(pState, &key)) { + if (!hasIntervalWindow(pState, &key, pStore)) { return true; } return false; @@ -2102,7 +2108,7 @@ static void clearStreamIntervalOperator(SStreamIntervalOperatorInfo* pInfo) { clearDiskbasedBuf(pInfo->aggSup.pResultBuf); initResultRowInfo(&pInfo->binfo.resultRowInfo); pInfo->aggSup.currentPageId = -1; - streamStateClear(pInfo->pState); + pInfo->statestore.streamStateClear(pInfo->pState); } static void clearSpecialDataBlock(SSDataBlock* pBlock) { @@ -2198,13 +2204,15 @@ static void clearFunctionContext(SExprSupp* pSup) { } } -int32_t getOutputBuf(SStreamState* pState, SRowBuffPos* pPos, SResultRow** pResult) { - return streamStateGetByPos(pState, pPos, (void**)pResult); +int32_t getOutputBuf(void* pState, SRowBuffPos* pPos, SResultRow** pResult, SStateStore* pStore) { + return pStore->streamStateGetByPos(pState, pPos, (void**)pResult); } -int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup, +int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SExprInfo* pExprInfo = pSup->pExprInfo; int32_t numOfExprs = pSup->numOfExprs; int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; @@ -2215,7 +2223,7 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) { SRowBuffPos* pPos = *(SRowBuffPos**)taosArrayGet(pGroupResInfo->pRows, i); SResultRow* pRow = NULL; - int32_t code = getOutputBuf(pState, pPos, &pRow); + int32_t code = getOutputBuf(pState, pPos, &pRow, &pAPI->stateStore); uint64_t groupId = ((SWinKey*)pPos->pKey)->groupId; ASSERT(code == 0); doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); @@ -2227,12 +2235,12 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat if (pBlock->info.id.groupId == 0) { pBlock->info.id.groupId = groupId; void* tbname = NULL; - if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { + if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { pBlock->info.parTbName[0] = 0; } else { memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); } - streamFreeVal(tbname); + pAPI->stateStore.streamStateFreeVal(tbname); } else { // current value belongs to different group, it can't be packed into one datablock if (pBlock->info.id.groupId != groupId) { @@ -2279,7 +2287,7 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat return TSDB_CODE_SUCCESS; } -void doBuildStreamIntervalResult(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, +void doBuildStreamIntervalResult(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; // set output datablock version @@ -2354,7 +2362,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p .groupId = groupId, }; void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey)); - if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup) && isClosed && !chIds) { + if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup, &pInfo->statestore) && isClosed && !chIds) { SPullWindowInfo pull = { .window = nextWin, .groupId = groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; // add pull data request @@ -2385,7 +2393,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p } int32_t code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput, - pSup->rowEntryInfoOffset, &pInfo->aggSup); + pSup->rowEntryInfoOffset, &pInfo->aggSup, &pInfo->statestore); pResult = (SResultRow*)pResPos->pRowBuff; if (code != TSDB_CODE_SUCCESS || pResult == NULL) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); @@ -2467,6 +2475,8 @@ static inline int winPosCmprImpl(const void* pKey1, const void* pKey2) { static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SOperatorInfo* downstream = pOperator->pDownstream[0]; SExprSupp* pSup = &pOperator->exprSupp; @@ -2505,8 +2515,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } else { if (pInfo->twAggSup.maxTs > 0 && pInfo->twAggSup.maxTs - pInfo->twAggSup.checkPointInterval > pInfo->twAggSup.checkPointTs) { - streamStateCommit(pInfo->pState); - streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark); + pAPI->stateStore.streamStateCommit(pInfo->pState); + pAPI->stateStore.streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark); pInfo->twAggSup.checkPointTs = pInfo->twAggSup.maxTs; } qDebug("===stream===interval final close"); @@ -2707,6 +2717,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, } pOperator->pTaskInfo = pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval, .sliding = pIntervalPhyNode->sliding, .intervalUnit = pIntervalPhyNode->intervalUnit, @@ -2743,9 +2755,9 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); initBasicInfo(&pInfo->binfo, pResBlock); - pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); + pInfo->pState = taosMemoryCalloc(1, sizeof(void)); *(pInfo->pState) = *(pTaskInfo->streamInfo.pState); - streamStateSetNumber(pInfo->pState, -1); + pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1); int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, pInfo->pState); if (code != TSDB_CODE_SUCCESS) { @@ -2753,10 +2765,9 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, } initStreamFunciton(pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs); - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); - initResultRowInfo(&pInfo->binfo.resultRowInfo); + pInfo->numOfChild = numOfChild; pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode); @@ -2773,6 +2784,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, if (!IS_FINAL_OP(pInfo) || numOfChild == 0) { pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; } + pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo)); pInfo->pullIndex = 0; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); @@ -2789,7 +2801,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->pUpdated = NULL; pInfo->pUpdatedMap = NULL; int32_t funResSize= getMaxFunResSize(&pOperator->exprSupp, numOfCols); - pInfo->pState->pFileState = streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, + pInfo->pState->pFileState = pAPI->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, compareTs, pInfo->pState, pInfo->twAggSup.deleteMark); pInfo->dataVersion = 0; @@ -2887,7 +2899,7 @@ void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uin pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type}; pScanInfo->pState = pAggSup->pState; if ((!pScanInfo->igCheckUpdate || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) && !pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark); + pScanInfo->pUpdateInfo = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark); } pScanInfo->twAggSup = *pTwSup; } @@ -2905,9 +2917,9 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, } initDummyFunction(pSup->pDummyCtx, pCtx, numOfOutput); - pSup->pState = taosMemoryCalloc(1, sizeof(SStreamState)); + pSup->pState = taosMemoryCalloc(1, sizeof(void)); *(pSup->pState) = *pState; - streamStateSetNumber(pSup->pState, -1); + pSup->stateStore.streamStateSetNumber(pSup->pState, -1); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pSup->pResultRows = tSimpleHashInit(32, hashFn); @@ -2950,7 +2962,7 @@ void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT pKey->win.skey = startTs; pKey->win.ekey = endTs; pKey->groupId = groupId; - int32_t code = streamStateSessionGetKeyByRange(pAggSup->pState, pKey, pKey); + int32_t code = pAggSup->stateStore.streamStateSessionGetKeyByRange(pAggSup->pState, pKey, pKey); if (code != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_KEY_INVALID(pKey); } @@ -2964,8 +2976,8 @@ void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT pCurWin->sessionWin.win.skey = startTs; pCurWin->sessionWin.win.ekey = endTs; int32_t size = pAggSup->resultRowSize; - int32_t code = - streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, pAggSup->gap, &pCurWin->pOutputBuf, &size); + int32_t code = pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, + pAggSup->gap, &pCurWin->pOutputBuf, &size); if (code == TSDB_CODE_SUCCESS) { pCurWin->isOutput = true; } else { @@ -2976,11 +2988,12 @@ void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT int32_t getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResultWindowInfo* pWinInfo) { int32_t size = 0; - int32_t code = streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, &pWinInfo->pOutputBuf, &size); + int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, &pWinInfo->pOutputBuf, &size); if (code != TSDB_CODE_SUCCESS) { return code; } - streamStateCurNext(pAggSup->pState, pCur); + + pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pCur); return TSDB_CODE_SUCCESS; } void saveDeleteInfo(SArray* pWins, SSessionKey key) { @@ -3065,7 +3078,7 @@ static int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindo } static bool doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) { - streamStateSessionDel(pAggSup->pState, pKey); + pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, pKey); SSessionKey hashKey = {0}; getSessionHashKey(pKey, &hashKey); tSimpleHashRemove(pAggSup->pResultRows, &hashKey, sizeof(SSessionKey)); @@ -3083,12 +3096,12 @@ static int32_t setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* SStreamStateCur* getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin) { - SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->sessionWin); + SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->sessionWin); pNextWin->isOutput = true; setSessionWinOutputInfo(pStUpdated, pNextWin); int32_t size = 0; pNextWin->sessionWin = pCurWin->sessionWin; - int32_t code = streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin, &pNextWin->pOutputBuf, &size); + int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin, &pNextWin->pOutputBuf, &size); if (code != TSDB_CODE_SUCCESS) { taosMemoryFreeClear(pNextWin->pOutputBuf); SET_SESSION_WIN_INVALID(*pNextWin); @@ -3100,6 +3113,8 @@ static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pC SSHashObj* pStDeleted) { SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SResultRow* pCurResult = NULL; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; @@ -3111,7 +3126,7 @@ static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pC SStreamStateCur* pCur = getNextSessionWinInfo(pAggSup, pStUpdated, pCurWin, &winInfo); if (!IS_VALID_SESSION_WIN(winInfo) || !isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap)) { taosMemoryFree(winInfo.pOutputBuf); - streamStateFreeCur(pCur); + pAPI->stateStore.streamStateFreeCur(pCur); break; } SResultRow* pWinResult = NULL; @@ -3125,13 +3140,13 @@ static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pC } removeSessionResult(pStUpdated, pAggSup->pResultRows, winInfo.sessionWin); doDeleteSessionWindow(pAggSup, &winInfo.sessionWin); - streamStateFreeCur(pCur); + pAPI->stateStore.streamStateFreeCur(pCur); taosMemoryFree(winInfo.pOutputBuf); } } int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo) { - saveSessionDiscBuf(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pOutputBuf, pAggSup->resultRowSize); + saveSessionDiscBuf(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pOutputBuf, pAggSup->resultRowSize, &pAggSup->stateStore); return TSDB_CODE_SUCCESS; } @@ -3251,6 +3266,8 @@ static int32_t copyUpdateResult(SSHashObj* pStUpdated, SArray* pUpdated) { } void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) { + SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI; + blockDataCleanup(pBlock); int32_t size = tSimpleHashGetSize(pStDeleted); if (size == 0) { @@ -3279,14 +3296,14 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); void* tbname = NULL; - streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, res->groupId, &tbname); + pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, res->groupId, &tbname); if (tbname == NULL) { colDataSetNULL(pTableCol, pBlock->info.rows); } else { char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); - streamFreeVal(tbname); + pAPI->stateStore.streamStateFreeVal(tbname); } pBlock->info.rows += 1; } @@ -3298,6 +3315,8 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SSHashObj* pStUpdated) { SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + int32_t size = taosArrayGetSize(pWinArray); SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; @@ -3314,7 +3333,7 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS SStreamAggSupporter* pChAggSup = &pChInfo->streamAggSup; SSessionKey chWinKey = {0}; getSessionHashKey(pWinKey, &chWinKey); - SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pChAggSup->pState, &chWinKey); + SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pChAggSup->pState, &chWinKey); SResultRow* pResult = NULL; SResultRow* pChResult = NULL; while (1) { @@ -3341,7 +3360,7 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS break; } } - streamStateFreeCur(pCur); + pAPI->stateStore.streamStateFreeCur(pCur); } if (num > 0) { saveSessionOutputBuf(pAggSup, &parentWin); @@ -3406,7 +3425,7 @@ void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayL pGroupResInfo->pBuf = NULL; } -void doBuildSessionResult(SOperatorInfo* pOperator, SStreamState* pState, SGroupResInfo* pGroupResInfo, +void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; // set output datablock version @@ -3636,7 +3655,7 @@ _error: static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) { tSimpleHashClear(pInfo->streamAggSup.pResultRows); - streamStateSessionClear(pInfo->streamAggSup.pState); + pInfo->streamAggSup.stateStore.streamStateSessionClear(pInfo->streamAggSup.pState); } static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { @@ -3762,6 +3781,7 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream goto _error; } + SStorageAPI* pAPI = &pTaskInfo->storageAPI; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; pInfo->isFinal = (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION); @@ -3786,7 +3806,7 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream } SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info; pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; - streamStateSetNumber(pChInfo->streamAggSup.pState, i); + pAPI->stateStore.streamStateSetNumber(pChInfo->streamAggSup.pState, i); taosArrayPush(pInfo->pChildren, &pChildOp); } } @@ -3849,7 +3869,7 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, pCurWin->winInfo.sessionWin.win.skey = ts; pCurWin->winInfo.sessionWin.win.ekey = ts; int32_t code = - streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData, pAggSup->stateKeySize, + pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData, pAggSup->stateKeySize, compareStateKey, &pCurWin->winInfo.pOutputBuf, &size); pCurWin->pStateKey = (SStateKeys*)((char*)pCurWin->winInfo.pOutputBuf + (pAggSup->resultRowSize - pAggSup->stateKeySize)); @@ -3870,12 +3890,12 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin; pNextWin->winInfo.pOutputBuf = NULL; - SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin); - code = streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, NULL, 0); + SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin); + code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, NULL, 0); if (code != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_INVALID(pNextWin->winInfo); } - streamStateFreeCur(pCur); + pAggSup->stateStore.streamStateFreeCur(pCur); } int32_t updateStateWindowInfo(SStateWindowInfo* pWinInfo, SStateWindowInfo* pNextWin, TSKEY* pTs, uint64_t groupId, @@ -3914,7 +3934,9 @@ int32_t updateStateWindowInfo(SStateWindowInfo* pWinInfo, SStateWindowInfo* pNex static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated, SSHashObj* pStDeleted) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SStreamStateAggOperatorInfo* pInfo = pOperator->info; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; uint64_t groupId = pSDataBlock->info.id.groupId; @@ -3956,7 +3978,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin); - releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)curWin.winInfo.pOutputBuf); + releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)curWin.winInfo.pOutputBuf, &pAPI->stateStore); continue; } code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput, @@ -4736,6 +4758,8 @@ _error: static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SExprSupp* pSup = &pOperator->exprSupp; if (pOperator->status == OP_EXEC_DONE) { @@ -4757,8 +4781,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { setOperatorCompleted(pOperator); if (pInfo->twAggSup.maxTs > 0 && pInfo->twAggSup.maxTs - pInfo->twAggSup.checkPointInterval > pInfo->twAggSup.checkPointTs) { - streamStateCommit(pInfo->pState); - streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark); + pAPI->stateStore.streamStateCommit(pInfo->pState); + pAPI->stateStore.streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark); setStreamDataVersion(pTaskInfo, pInfo->dataVersion, pInfo->pState->checkPointId); pInfo->twAggSup.checkPointTs = pInfo->twAggSup.maxTs; } @@ -4900,6 +4924,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay"); pOperator->pTaskInfo = pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; pInfo->ignoreExpiredDataSaved = false; pInfo->isFinal = false; @@ -4912,9 +4938,9 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; initResultSizeInfo(&pOperator->resultInfo, 4096); - pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); + pInfo->pState = taosMemoryCalloc(1, sizeof(void)); *(pInfo->pState) = *(pTaskInfo->streamInfo.pState); - streamStateSetNumber(pInfo->pState, -1); + pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1); size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, @@ -4952,7 +4978,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pUpdated = NULL; pInfo->pUpdatedMap = NULL; int32_t funResSize= getMaxFunResSize(pSup, numOfCols); - pInfo->pState->pFileState = streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, + pInfo->pState->pFileState = pTaskInfo->storageAPI.stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, compareTs, pInfo->pState, pInfo->twAggSup.deleteMark); setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index f23b4d3e87..39ebee4fb1 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -33,7 +33,6 @@ target_link_libraries( PRIVATE qcom PRIVATE scalar PRIVATE transport - PRIVATE stream ${LINK_JEMALLOC} PUBLIC uv_a ) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 1a43802e6b..f83873848e 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -18,7 +18,7 @@ #include "function.h" #include "query.h" #include "querynodes.h" -#include "streamState.h" +//#include "streamState.h" #include "tcompare.h" #include "tdatablock.h" #include "tdigest.h" @@ -3155,9 +3155,9 @@ static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, releaseBufPage(pHandle->pBuf, pPage); } else { // other tuple save policy - if (streamStateFuncPut(pHandle->pState, key, pBuf, length) >= 0) { - p.streamTupleKey = *key; - } +// if (streamStateFuncPut(pHandle->pState, key, pBuf, length) >= 0) { +// p.streamTupleKey = *key; +// } } *pPos = p; @@ -3192,7 +3192,7 @@ static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); } else { - streamStateFuncPut(pHandle->pState, &pPos->streamTupleKey, pBuf, length); +// streamStateFuncPut(pHandle->pState, &pPos->streamTupleKey, pBuf, length); } return TSDB_CODE_SUCCESS; @@ -3217,7 +3217,7 @@ static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPo } else { void* value = NULL; int32_t vLen; - streamStateFuncGet(pHandle->pState, &pPos->streamTupleKey, &value, &vLen); +// streamStateFuncGet(pHandle->pState, &pPos->streamTupleKey, &value, &vLen); return (char*)value; } } diff --git a/source/libs/index/CMakeLists.txt b/source/libs/index/CMakeLists.txt index 0c2ce37c40..8ef134444f 100644 --- a/source/libs/index/CMakeLists.txt +++ b/source/libs/index/CMakeLists.txt @@ -12,7 +12,7 @@ target_link_libraries( PUBLIC os PUBLIC util PUBLIC common - PUBLIC vnode +# PUBLIC vnode PUBLIC nodes PUBLIC scalar PUBLIC function diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 02ed0d2d05..7b3013362d 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -20,7 +20,7 @@ #include "querynodes.h" #include "scalar.h" #include "tdatablock.h" -#include "vnode.h" +#include "tsdstorage.h" // clang-format off #define SIF_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) @@ -79,6 +79,7 @@ typedef struct SIFParam { char colName[TSDB_COL_NAME_LEN * 2 + 4]; SIndexMetaArg arg; + SStoreAPI api; } SIFParam; typedef struct SIFCtx { @@ -659,7 +660,7 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP } else { if (sifSetFltParam(left, right, &typedata, ¶m) != 0) return -1; } - ret = metaFilterTableIds(arg->metaEx, ¶m, output->result); + ret = left->api.metaFilterTableIds(arg->metaEx, ¶m, output->result); } return ret; } diff --git a/source/libs/scalar/CMakeLists.txt b/source/libs/scalar/CMakeLists.txt index 193a6971e5..30c68cb512 100644 --- a/source/libs/scalar/CMakeLists.txt +++ b/source/libs/scalar/CMakeLists.txt @@ -14,6 +14,7 @@ target_link_libraries(scalar PRIVATE nodes PRIVATE function PRIVATE qcom + PRIVATE parser ) if(${BUILD_TEST}) diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 0f39cf817b..1cbd7b042c 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -17,7 +17,6 @@ #define _STREAM_BACKEDN_ROCKSDB_H_ #include "rocksdb/c.h" -// #include "streamInc.h" #include "streamState.h" #include "tcoding.h" #include "tcommon.h" diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 16ba81c74a..0015fea10f 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -214,8 +214,7 @@ int streamGetInit(const char* funcName); // |key|-----value------| // |key|ttl|len|userData| -static rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, - rocksdb_snapshot_t** snapshot, rocksdb_readoptions_t** readOpt); +static rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, void** snapshot, void** readOpt); int defaultKeyComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen) { int ret = memcmp(aBuf, bBuf, aLen); @@ -799,10 +798,10 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { if (ppInst != NULL && *ppInst != NULL) { RocksdbCfInst* inst = *ppInst; pState->pTdbState->rocksdb = inst->db; - pState->pTdbState->pHandle = inst->pHandle; + pState->pTdbState->pHandle = (void**)inst->pHandle; pState->pTdbState->writeOpts = inst->wOpt; pState->pTdbState->readOpts = inst->rOpt; - pState->pTdbState->cfOpts = inst->cfOpt; + pState->pTdbState->cfOpts = (void**)inst->cfOpt; pState->pTdbState->dbOpt = handle->dbOpt; pState->pTdbState->param = inst->param; pState->pTdbState->pBackendHandle = handle; @@ -810,6 +809,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { taosThreadMutexUnlock(&handle->cfMutex); return 0; } + taosThreadMutexUnlock(&handle->cfMutex); char* err = NULL; @@ -850,10 +850,10 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { } } pState->pTdbState->rocksdb = handle->db; - pState->pTdbState->pHandle = cfHandle; + pState->pTdbState->pHandle = (void**)cfHandle; pState->pTdbState->writeOpts = rocksdb_writeoptions_create(); pState->pTdbState->readOpts = rocksdb_readoptions_create(); - pState->pTdbState->cfOpts = (rocksdb_options_t**)cfOpt; + pState->pTdbState->cfOpts = (void**)(rocksdb_options_t**)cfOpt; pState->pTdbState->dbOpt = handle->dbOpt; pState->pTdbState->param = param; pState->pTdbState->pBackendHandle = handle; @@ -954,8 +954,7 @@ bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len } return true; } -rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, rocksdb_snapshot_t** snapshot, - rocksdb_readoptions_t** readOpt) { +rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, void** snapshot, void** readOpt) { int idx = streamGetInit(cfName); if (snapshot != NULL) { @@ -1896,7 +1895,7 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co rocksdb_snapshot_t* snapshot = NULL; rocksdb_readoptions_t* readopts = NULL; - rocksdb_iterator_t* pIter = streamStateIterCreate(pState, "default", &snapshot, &readopts); + rocksdb_iterator_t* pIter = streamStateIterCreate(pState, "default", (void**)&snapshot, (void**)&readopts); if (pIter == NULL) { return -1; } @@ -1966,6 +1965,7 @@ char* streamDefaultIterVal_rocksdb(void* iter, int32_t* len) { } return dst; } + // batch func void* streamStateCreateBatch() { rocksdb_writebatch_t* pBatch = rocksdb_writebatch_create(); diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index ad66bb5a27..f531f65565 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -21,6 +21,7 @@ #include "tcommon.h" #include "thash.h" #include "tsimplehash.h" +#include "storageapi.h" #define FLUSH_RATIO 0.5 #define FLUSH_NUM 4 From c66c0c4dfd415b0815dba3dce4fc7d44980be5ee Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 May 2023 18:42:52 +0800 Subject: [PATCH 18/97] refactor: do some internal refactor. --- include/libs/executor/storageapi.h | 7 +++---- include/libs/index/index.h | 1 - include/libs/scalar/filter.h | 10 ---------- source/libs/executor/src/aggregateoperator.c | 1 - source/libs/executor/src/sysscanoperator.c | 1 - source/libs/index/src/indexFilter.c | 1 - 6 files changed, 3 insertions(+), 18 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 9b53f11ad9..7bcf6f8fd4 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -18,7 +18,6 @@ #include "tsimplehash.h" #include "tscalablebf.h" -//#include "tdb.h" #include "taosdef.h" #include "tmsg.h" #include "tcommon.h" @@ -35,8 +34,6 @@ extern "C" { #define CACHESCAN_RETRIEVE_LAST_ROW 0x4 #define CACHESCAN_RETRIEVE_LAST 0x8 -typedef struct SMeta SMeta; - typedef struct SMetaEntry { int64_t version; int8_t type; @@ -138,8 +135,10 @@ typedef struct SMetaTableInfo { char tbName[TSDB_TABLE_NAME_LEN]; } SMetaTableInfo; +typedef struct SMeta SMeta; + typedef struct SSnapContext { - SMeta * pMeta; + SMeta * pMeta; // todo remove it int64_t snapVersion; struct TBC *pCur; int64_t suid; diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 9260ec4b1e..fd71c531fa 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -21,7 +21,6 @@ #include "taoserror.h" #include "tarray.h" #include "tglobal.h" -#include "tsdstorage.h" #ifdef __cplusplus extern "C" { diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h index 7c5ca628cd..f20ba287de 100644 --- a/include/libs/scalar/filter.h +++ b/include/libs/scalar/filter.h @@ -55,16 +55,6 @@ extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pColsAgg, int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond, SNode **pOtherCond); -typedef struct SMetaFltParam { - tb_uid_t suid; - int16_t cid; - int16_t type; - void *val; - bool reverse; - bool equal; - int (*filterFunc)(void *a, void *b, int16_t type); -} SMetaFltParam; - #ifdef __cplusplus } #endif diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index 918038635b..00aa52b593 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -21,7 +21,6 @@ #include "tname.h" #include "executorInt.h" -#include "tsdstorage.h" #include "operator.h" #include "query.h" #include "querytask.h" diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 460373b537..5bbfb188ed 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -32,7 +32,6 @@ #include "tcompare.h" #include "thash.h" #include "ttypes.h" -#include "tsdstorage.h" #include "index.h" typedef int (*__optSysFilter)(void* a, void* b, int16_t dtype); diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 2dce931ad4..b90959579d 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -21,7 +21,6 @@ #include "querynodes.h" #include "scalar.h" #include "tdatablock.h" -#include "tsdstorage.h" // clang-format off #define SIF_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) From a63473b1cc5688c89e82542cd73a487d58bd01be Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 May 2023 18:54:32 +0800 Subject: [PATCH 19/97] refacotor: do some internal refactor. --- include/libs/executor/storageapi.h | 125 ++++++++++----------- source/libs/executor/inc/executorInt.h | 6 +- source/libs/executor/src/executor.c | 22 ++-- source/libs/executor/src/executorInt.c | 2 +- source/libs/executor/src/operator.c | 4 +- source/libs/executor/src/scanoperator.c | 84 +++++++------- source/libs/executor/src/sysscanoperator.c | 6 +- 7 files changed, 120 insertions(+), 129 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 7bcf6f8fd4..67c0eacd26 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -34,6 +34,11 @@ extern "C" { #define CACHESCAN_RETRIEVE_LAST_ROW 0x4 #define CACHESCAN_RETRIEVE_LAST 0x8 +#define META_READER_NOLOCK 0x1 + +typedef struct SMeta SMeta; +typedef TSKEY (*GetTsFun)(void*); + typedef struct SMetaEntry { int64_t version; int8_t type; @@ -70,6 +75,32 @@ typedef struct SMetaEntry { uint8_t *pBuf; } SMetaEntry; +typedef struct SMetaReader { + int32_t flags; + void * pMeta; + SDecoder coder; + SMetaEntry me; + void * pBuf; + int32_t szBuf; + struct SStorageAPI *storageAPI; +} SMetaReader; + +typedef struct SMTbCursor { + struct TBC *pDbc; + void * pKey; + void * pVal; + int32_t kLen; + int32_t vLen; + SMetaReader mr; +} SMTbCursor; + +typedef struct SRowBuffPos { + void* pRowBuff; + void* pKey; + bool beFlushed; + bool beUsed; +} SRowBuffPos; + // int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables); // int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, // SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr); @@ -135,8 +166,6 @@ typedef struct SMetaTableInfo { char tbName[TSDB_TABLE_NAME_LEN]; } SMetaTableInfo; -typedef struct SMeta SMeta; - typedef struct SSnapContext { SMeta * pMeta; // todo remove it int64_t snapVersion; @@ -162,23 +191,18 @@ typedef struct { // int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); // bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid); // bool tqCurrentBlockConsumed(const STqReader* pReader); -// // int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); // bool tqNextBlockInWal(STqReader* pReader, const char* idstr); // bool tqNextBlockImpl(STqReader *pReader, const char* idstr); - // int32_t getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); // SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx); // int32_t setForSnapShot(SSnapContext *ctx, int64_t uid); // int32_t destroySnapContext(SSnapContext *ctx); - // SMTbCursor *metaOpenTbCursor(SMeta *pMeta); // void metaCloseTbCursor(SMTbCursor *pTbCur); // int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); // int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); -#define META_READER_NOLOCK 0x1 - /*-------------------------------------------------new api format---------------------------------------------------*/ // typedef int32_t (*__store_reader_(STsdbReader *pReader, const void *pTableList, int32_t num); @@ -201,7 +225,7 @@ typedef int32_t (*__store_reader_open_fn_t)(void *pVnode, SQueryTableDataCond *p int32_t numOfTables, SSDataBlock *pResBlock, void **ppReader, const char *idstr, bool countOnly, SHashObj **pIgnoreTables); -typedef struct SStoreDataReaderFn { +typedef struct SStoreTSDReader { __store_reader_open_fn_t storeReaderOpen; void (*storeReaderClose)(); void (*setReaderId)(void *pReader, const char *pId); @@ -216,7 +240,7 @@ typedef struct SStoreDataReaderFn { void (*storeReaderGetDataBlockDistInfo)(); void (*storeReaderGetNumOfInMemRows)(); void (*storeReaderNotifyClosing)(); -} SStoreDataReaderFn; +} SStoreTSDReader; /** * int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables); @@ -226,14 +250,14 @@ int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32 SArray *pTableUids); void *tsdbCacherowsReaderClose(void *pReader); */ -typedef struct SStoreCachedDataReaderFn { +typedef struct SStoreCacheReader { int32_t (*openReader)(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr); void *(*closeReader)(void *pReader); int32_t (*retrieveRows)(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, SArray *pTableUidList); void (*reuseReader)(void *pReader, void *pTableIdList, int32_t numOfTables); -} SStoreCachedDataReaderFn; +} SStoreCacheReader; /*------------------------------------------------------------------------------------------------------------------*/ /* @@ -259,7 +283,7 @@ SWalReader* tqGetWalReader(STqReader* pReader); int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); */ // todo rename -typedef struct SStoreTqReaderFn { +typedef struct SStoreTqReader { void *(*tqReaderOpen)(); void (*tqReaderClose)(); @@ -282,7 +306,7 @@ typedef struct SStoreTqReaderFn { int32_t (*tqReaderSetSubmitMsg)(); // todo remove it void (*tqReaderNextBlockFilterOut)(); -} SStoreTqReaderFn; +} SStoreTqReader; typedef struct SStoreSnapshotFn { /* @@ -291,10 +315,10 @@ typedef struct SStoreSnapshotFn { int32_t setForSnapShot(SSnapContext *ctx, int64_t uid); int32_t destroySnapContext(SSnapContext *ctx); */ - int32_t (*storeCreateSnapshot)(); - void (*storeDestroySnapshot)(); - SMetaTableInfo (*storeSSGetTableInfo)(); - int32_t (*storeSSGetMetaInfo)(); + int32_t (*createSnapshot)(); + void (*destroySnapshot)(); + SMetaTableInfo (*getTableInfoFromSnapshot)(); + int32_t (*getMetaInfoFromSnapshot)(); } SStoreSnapshotFn; /** @@ -322,17 +346,9 @@ int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen); */ -typedef struct SMetaReader { - int32_t flags; - void * pMeta; - SDecoder coder; - SMetaEntry me; - void * pBuf; - int32_t szBuf; - struct SStorageAPI *storageAPI; -} SMetaReader; -typedef struct SStoreMetaReaderFn { + +typedef struct SStoreMetaReader { void (*initReader)(void *pReader, void *pMeta, int32_t flags); void *(*clearReader)(); @@ -341,9 +357,9 @@ typedef struct SStoreMetaReaderFn { int32_t (*getTableEntryByUid)(); int32_t (*getTableEntryByName)(); int32_t (*readerGetEntryGetUidCache)(SMetaReader *pReader, tb_uid_t uid); -} SStoreMetaReaderFn; +} SStoreMetaReader; -typedef struct SStoreMetaFn { +typedef struct SStoreMeta { /* SMTbCursor *metaOpenTbCursor(SMeta *pMeta); void metaCloseTbCursor(SMTbCursor *pTbCur); @@ -398,7 +414,7 @@ int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list); int32_t vnodeGetCtbIdListByFilter(void *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg); int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list); */ -} SStoreMetaFn; +} SStoreMeta; typedef struct STdbState { void* rocksdb; @@ -450,21 +466,14 @@ typedef struct SUpdateInfo { } SUpdateInfo; typedef struct { - void* iter; - void* snapshot; - void* readOpt; - void* db; -// rocksdb_iterator_t* iter; -// rocksdb_snapshot_t* snapshot; -// rocksdb_readoptions_t* readOpt; -// rocksdb_t* db; - - void* pCur; + void* iter; // rocksdb_iterator_t* iter; + void* snapshot; // rocksdb_snapshot_t* snapshot; + void* readOpt; // rocksdb_readoptions_t* readOpt; + void* db; // rocksdb_t* db; + void* pCur; int64_t number; } SStreamStateCur; -typedef TSKEY (*GetTsFun)(void*); - typedef struct SStateStore { int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname); int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal); @@ -541,33 +550,15 @@ typedef struct SStateStore { } SStateStore; typedef struct SStorageAPI { - SStoreMetaFn metaFn; // todo: refactor - SStoreDataReaderFn storeReader; - SStoreMetaReaderFn metaReaderFn; - SStoreCachedDataReaderFn cacheFn; - SStoreSnapshotFn snapshotFn; - SStoreTqReaderFn tqReaderFn; - SStateStore stateStore; + SStoreMeta metaFn; // todo: refactor + SStoreTSDReader tsdReader; + SStoreMetaReader metaReaderFn; + SStoreCacheReader cacheFn; + SStoreSnapshotFn snapshotFn; + SStoreTqReader tqReaderFn; + SStateStore stateStore; } SStorageAPI; -typedef struct SMTbCursor { - struct TBC *pDbc; - void * pKey; - void * pVal; - int32_t kLen; - int32_t vLen; - SMetaReader mr; -} SMTbCursor; - -typedef struct SRowBuffPos { - void* pRowBuff; - void* pKey; - bool beFlushed; - bool beUsed; -} SRowBuffPos; - - - #ifdef __cplusplus } #endif diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 57b3bc3292..4c5ea4c242 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -208,7 +208,7 @@ typedef struct STableScanBase { SLimitInfo limitInfo; // there are more than one table list exists in one task, if only one vnode exists. STableListInfo* pTableListInfo; - SStoreDataReaderFn readerAPI; + SStoreTSDReader readerAPI; } STableScanBase; typedef struct STableScanInfo { @@ -224,7 +224,7 @@ typedef struct STableScanInfo { int8_t assignBlockUid; bool hasGroupByTag; bool countOnly; - SStoreDataReaderFn readerAPI; + SStoreTSDReader readerAPI; } STableScanInfo; typedef struct STableMergeScanInfo { @@ -372,7 +372,7 @@ typedef struct SStreamScanInfo { int8_t igCheckUpdate; int8_t igExpired; void* pState; //void - SStoreTqReaderFn readerFn; + SStoreTqReader readerFn; SStateStore stateStore; } SStreamScanInfo; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index e89db5b59c..aa51b52c80 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -166,7 +166,7 @@ void doSetTaskId(SOperatorInfo* pOperator, SStorageAPI *pAPI) { if (pStreamScanInfo->pTableScanOp != NULL) { STableScanInfo* pScanInfo = pStreamScanInfo->pTableScanOp->info; if (pScanInfo->base.dataReader != NULL) { - pAPI->storeReader.setReaderId(pScanInfo->base.dataReader, pTaskInfo->id.str); + pAPI->tsdReader.setReaderId(pScanInfo->base.dataReader, pTaskInfo->id.str); } } } else { @@ -1087,7 +1087,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT if (pOffset->type == TMQ_OFFSET__LOG) { // todo refactor: move away - pTaskInfo->storageAPI.storeReader.storeReaderClose(pScanBaseInfo->dataReader); + pTaskInfo->storageAPI.tsdReader.storeReaderClose(pScanBaseInfo->dataReader); pScanBaseInfo->dataReader = NULL; ASSERT(0); @@ -1148,7 +1148,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT pScanInfo->scanTimes = 0; if (pScanBaseInfo->dataReader == NULL) { - int32_t code = pTaskInfo->storageAPI.storeReader.storeReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, + int32_t code = pTaskInfo->storageAPI.tsdReader.storeReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, pScanInfo->pResBlock, (void**) &pScanBaseInfo->dataReader, id, false, NULL); if (code != TSDB_CODE_SUCCESS) { qError("prepare read tsdb snapshot failed, uid:%" PRId64 ", code:%s %s", pOffset->uid, tstrerror(code), id); @@ -1159,8 +1159,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts:%" PRId64 " table index:%d, total:%d, %s", uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); } else { - pTaskInfo->storageAPI.storeReader.storeReaderSetTableList(pScanBaseInfo->dataReader, &keyInfo, 1); - pTaskInfo->storageAPI.storeReader.storeReaderResetStatus(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); + pTaskInfo->storageAPI.tsdReader.storeReaderSetTableList(pScanBaseInfo->dataReader, &keyInfo, 1); + pTaskInfo->storageAPI.tsdReader.storeReaderResetStatus(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 " table index:%d numOfTable:%d, %s", uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); } @@ -1182,14 +1182,14 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT SOperatorInfo* p = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, id); STableListInfo* pTableListInfo = ((SStreamRawScanInfo*)(p->info))->pTableListInfo; - if (pAPI->snapshotFn.storeCreateSnapshot(sContext, pOffset->uid) != 0) { + if (pAPI->snapshotFn.createSnapshot(sContext, pOffset->uid) != 0) { qError("setDataForSnapShot error. uid:%" PRId64 " , %s", pOffset->uid, id); terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } - SMetaTableInfo mtInfo = pTaskInfo->storageAPI.snapshotFn.storeSSGetTableInfo(sContext); - pTaskInfo->storageAPI.storeReader.storeReaderClose(pInfo->dataReader); + SMetaTableInfo mtInfo = pTaskInfo->storageAPI.snapshotFn.getTableInfoFromSnapshot(sContext); + pTaskInfo->storageAPI.tsdReader.storeReaderClose(pInfo->dataReader); pInfo->dataReader = NULL; cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); @@ -1207,7 +1207,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0); int32_t size = tableListGetSize(pTableListInfo); - pTaskInfo->storageAPI.storeReader.storeReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, (void**) &pInfo->dataReader, NULL, + pTaskInfo->storageAPI.tsdReader.storeReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, (void**) &pInfo->dataReader, NULL, false, NULL); cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); @@ -1219,7 +1219,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) { SStreamRawScanInfo* pInfo = pOperator->info; SSnapContext* sContext = pInfo->sContext; - if (pTaskInfo->storageAPI.snapshotFn.storeCreateSnapshot(sContext, pOffset->uid) != 0) { + if (pTaskInfo->storageAPI.snapshotFn.createSnapshot(sContext, pOffset->uid) != 0) { qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version); terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; @@ -1228,7 +1228,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT id); } else if (pOffset->type == TMQ_OFFSET__LOG) { SStreamRawScanInfo* pInfo = pOperator->info; - pTaskInfo->storageAPI.storeReader.storeReaderClose(pInfo->dataReader); + pTaskInfo->storageAPI.tsdReader.storeReaderClose(pInfo->dataReader); pInfo->dataReader = NULL; qDebug("tmqsnap qStreamPrepareScan snapshot log, %s", id); } diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index c11c5be1ae..240803e243 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -1195,7 +1195,7 @@ void qStreamCloseTsdbReader(void* task) { qDebug("wait for the reader stopping"); } - pTaskInfo->storageAPI.storeReader.storeReaderClose(pTSInfo->base.dataReader); + pTaskInfo->storageAPI.tsdReader.storeReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; // restore the status, todo refactor. diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 1b85aedc1e..8d62b7a65f 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -239,7 +239,7 @@ static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, STableScanInfo* pInfo = pOperator->info; if (pInfo->base.dataReader != NULL) { - pAPI->storeReader.storeReaderNotifyClosing(pInfo->base.dataReader); + pAPI->tsdReader.storeReaderNotifyClosing(pInfo->base.dataReader); } return OPTR_FN_RET_ABORT; } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { @@ -248,7 +248,7 @@ static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, if (pInfo->pTableScanOp != NULL) { STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; if (pTableScanInfo != NULL && pTableScanInfo->base.dataReader != NULL) { - pAPI->storeReader.storeReaderNotifyClosing(pTableScanInfo->base.dataReader); + pAPI->tsdReader.storeReaderNotifyClosing(pTableScanInfo->base.dataReader); } } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ac8ca675ce..4bed2d3a5d 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -267,7 +267,7 @@ static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SStorageAPI* pAPI = &pTaskInfo->storageAPI; bool allColumnsHaveAgg = true; - int32_t code = pAPI->storeReader.storeReaderRetrieveBlockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); + int32_t code = pAPI->tsdReader.storeReaderRetrieveBlockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -350,7 +350,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->filterOutBlocks += 1; pCost->totalRows += pBlock->info.rows; - pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", uid:%" PRIu64, @@ -358,7 +358,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pBlockInfo->id.uid); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); pCost->skipBlocks += 1; - pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_SMA_LOAD) { pCost->loadBlockStatis += 1; @@ -368,7 +368,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); - pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else { qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo)); @@ -390,7 +390,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->filterOutBlocks += 1; (*status) = FUNC_DATA_REQUIRED_FILTEROUT; - pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } } @@ -405,7 +405,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; - pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); STableScanInfo* p1 = pOperator->info; if (taosHashGetSize(p1->pIgnoreTables) == taosArrayGetSize(p1->base.pTableListInfo->pTableList)) { @@ -419,7 +419,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->totalCheckedRows += pBlock->info.rows; pCost->loadBlocks += 1; - SSDataBlock* p = pAPI->storeReader.storeReaderRetrieveDataBlock(pTableScanInfo->dataReader, NULL); + SSDataBlock* p = pAPI->tsdReader.storeReaderRetrieveDataBlock(pTableScanInfo->dataReader, NULL); if (p == NULL) { return terrno; } @@ -693,9 +693,9 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { int64_t st = taosGetTimestampUs(); while (true) { - code = pAPI->storeReader.storeReaderNextDataBlock(pTableScanInfo->base.dataReader, &hasNext); + code = pAPI->tsdReader.storeReaderNextDataBlock(pTableScanInfo->base.dataReader, &hasNext); if (code) { - pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); + pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); T_LONG_JMP(pTaskInfo->env, code); } @@ -704,12 +704,12 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { } if (isTaskKilled(pTaskInfo)) { - pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); + pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } if (pOperator->status == OP_EXEC_DONE) { - pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); + pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); break; } @@ -774,7 +774,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { qDebug("start to repeat ascending order scan data blocks due to query func required, %s", GET_TASKID(pTaskInfo)); // do prepare for the next round table scan operation - pAPI->storeReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pAPI->tsdReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); } } @@ -782,7 +782,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { if (pTableScanInfo->scanTimes < total) { if (pTableScanInfo->base.cond.order == TSDB_ORDER_ASC) { prepareForDescendingScan(&pTableScanInfo->base, pOperator->exprSupp.pCtx, 0); - pAPI->storeReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pAPI->tsdReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); } @@ -800,7 +800,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { pTableScanInfo->base.scanFlag = MAIN_SCAN; qDebug("%s start to repeat descending order scan data blocks", GET_TASKID(pTaskInfo)); - pAPI->storeReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pAPI->tsdReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); } } } @@ -839,11 +839,11 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable); taosRUnLockLatch(&pTaskInfo->lock); - pAPI->storeReader.storeReaderSetTableList(pInfo->base.dataReader, &tInfo, 1); + pAPI->tsdReader.storeReaderSetTableList(pInfo->base.dataReader, &tInfo, 1); qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", tInfo.uid, numOfTables, pInfo->currentTable, numOfTables, GET_TASKID(pTaskInfo)); - pAPI->storeReader.storeReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); + pAPI->tsdReader.storeReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; } } else { // scan table group by group sequentially @@ -858,7 +858,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); ASSERT(pInfo->base.dataReader == NULL); - int32_t code = pAPI->storeReader.storeReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, + int32_t code = pAPI->tsdReader.storeReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly, &pInfo->pIgnoreTables); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); @@ -887,8 +887,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableKeyInfo* pList = NULL; tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); - pAPI->storeReader.storeReaderSetTableList(pInfo->base.dataReader, pList, num); - pAPI->storeReader.storeReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); + pAPI->tsdReader.storeReaderSetTableList(pInfo->base.dataReader, pList, num); + pAPI->tsdReader.storeReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; result = doGroupedTableScan(pOperator); @@ -910,7 +910,7 @@ static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptr return 0; } -static void destroyTableScanBase(STableScanBase* pBase, SStoreDataReaderFn* pAPI) { +static void destroyTableScanBase(STableScanBase* pBase, SStoreTSDReader* pAPI) { cleanupQueryTableDataCond(&pBase->cond); pAPI->storeReaderClose(pBase->dataReader); @@ -1094,7 +1094,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU SSDataBlock* pBlock = pTableScanInfo->pResBlock; STsdbReader* pReader = NULL; - int32_t code = pAPI->storeReader.storeReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, + int32_t code = pAPI->tsdReader.storeReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, (void**)&pReader, GET_TASKID(pTaskInfo), false, NULL); if (code != TSDB_CODE_SUCCESS) { terrno = code; @@ -1103,7 +1103,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } bool hasNext = false; - code = pAPI->storeReader.storeReaderNextDataBlock(pReader, &hasNext); + code = pAPI->tsdReader.storeReaderNextDataBlock(pReader, &hasNext); if (code != TSDB_CODE_SUCCESS) { terrno = code; T_LONG_JMP(pTaskInfo->env, code); @@ -1111,12 +1111,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } if (hasNext) { - /*SSDataBlock* p = */ pAPI->storeReader.storeReaderRetrieveDataBlock(pReader, NULL); + /*SSDataBlock* p = */ pAPI->tsdReader.storeReaderRetrieveDataBlock(pReader, NULL); doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); pBlock->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); } - pAPI->storeReader.storeReaderClose(pReader); + pAPI->tsdReader.storeReaderClose(pReader); qDebug("retrieve prev rows:%" PRId64 ", skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64 ", suid:%" PRIu64, pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid); @@ -1657,7 +1657,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { return pResult; } STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - pAPI->storeReader.storeReaderClose(pTSInfo->base.dataReader); + pAPI->tsdReader.storeReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1); @@ -1821,7 +1821,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN2; } - pAPI->storeReader.storeReaderClose(pTSInfo->base.dataReader); + pAPI->tsdReader.storeReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; pInfo->pTableScanOp->status = OP_OPENED; @@ -1901,7 +1901,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { } pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - pAPI->storeReader.storeReaderClose(pTSInfo->base.dataReader); + pAPI->tsdReader.storeReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; @@ -2162,20 +2162,20 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { bool hasNext = false; if (pInfo->dataReader) { - code = pAPI->storeReader.storeReaderNextDataBlock(pInfo->dataReader, &hasNext); + code = pAPI->tsdReader.storeReaderNextDataBlock(pInfo->dataReader, &hasNext); if (code) { - pAPI->storeReader.storeReaderReleaseDataBlock(pInfo->dataReader); + pAPI->tsdReader.storeReaderReleaseDataBlock(pInfo->dataReader); T_LONG_JMP(pTaskInfo->env, code); } } if (pInfo->dataReader && hasNext) { if (isTaskKilled(pTaskInfo)) { - pAPI->storeReader.storeReaderReleaseDataBlock(pInfo->dataReader); + pAPI->tsdReader.storeReaderReleaseDataBlock(pInfo->dataReader); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } - SSDataBlock* pBlock = pAPI->storeReader.storeReaderRetrieveDataBlock(pInfo->dataReader, NULL); + SSDataBlock* pBlock = pAPI->tsdReader.storeReaderRetrieveDataBlock(pInfo->dataReader, NULL); if (pBlock == NULL) { T_LONG_JMP(pTaskInfo->env, terrno); } @@ -2185,7 +2185,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { return pBlock; } - SMetaTableInfo mtInfo = pAPI->snapshotFn.storeSSGetTableInfo(pInfo->sContext); + SMetaTableInfo mtInfo = pAPI->snapshotFn.getTableInfoFromSnapshot(pInfo->sContext); STqOffsetVal offset = {0}; if (mtInfo.uid == 0) { // read snapshot done, change to get data from wal qDebug("tmqsnap read snapshot done, change to get data from wal"); @@ -2203,7 +2203,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { int32_t dataLen = 0; int16_t type = 0; int64_t uid = 0; - if (pAPI->snapshotFn.storeSSGetMetaInfo(sContext, &data, &dataLen, &type, &uid) < 0) { + if (pAPI->snapshotFn.getMetaInfoFromSnapshot(sContext, &data, &dataLen, &type, &uid) < 0) { qError("tmqsnap getMetafromSnapShot error"); taosMemoryFreeClear(data); return NULL; @@ -2227,8 +2227,8 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { static void destroyRawScanOperatorInfo(void* param) { SStreamRawScanInfo* pRawScan = (SStreamRawScanInfo*)param; - pRawScan->pAPI->storeReader.storeReaderClose(pRawScan->dataReader); - pRawScan->pAPI->snapshotFn.storeDestroySnapshot(pRawScan->sContext); + pRawScan->pAPI->tsdReader.storeReaderClose(pRawScan->dataReader); + pRawScan->pAPI->snapshotFn.destroySnapshot(pRawScan->sContext); tableListDestroy(pRawScan->pTableListInfo); taosMemoryFree(pRawScan); } @@ -2662,7 +2662,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { SReadHandle* pHandle = &pInfo->base.readHandle; if (NULL == source->dataReader || !source->multiReader) { - code = pAPI->storeReader.storeReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, (void**)&source->dataReader, GET_TASKID(pTaskInfo), false, NULL); + code = pAPI->tsdReader.storeReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, (void**)&source->dataReader, GET_TASKID(pTaskInfo), false, NULL); if (code != 0) { T_LONG_JMP(pTaskInfo->env, code); } @@ -2674,9 +2674,9 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { qTrace("tsdb/read-table-data: %p, enter next reader", reader); while (true) { - code = pAPI->storeReader.storeReaderNextDataBlock(reader, &hasNext); + code = pAPI->tsdReader.storeReaderNextDataBlock(reader, &hasNext); if (code != 0) { - pAPI->storeReader.storeReaderReleaseDataBlock(reader); + pAPI->tsdReader.storeReaderReleaseDataBlock(reader); pInfo->base.dataReader = NULL; T_LONG_JMP(pTaskInfo->env, code); } @@ -2686,7 +2686,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } if (isTaskKilled(pTaskInfo)) { - pAPI->storeReader.storeReaderReleaseDataBlock(reader); + pAPI->tsdReader.storeReaderReleaseDataBlock(reader); pInfo->base.dataReader = NULL; T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } @@ -2726,7 +2726,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { qTrace("tsdb/read-table-data: %p, close reader", reader); if (!source->multiReader) { - pAPI->storeReader.storeReaderClose(pInfo->base.dataReader); + pAPI->tsdReader.storeReaderClose(pInfo->base.dataReader); source->dataReader = NULL; } pInfo->base.dataReader = NULL; @@ -2734,7 +2734,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } if (!source->multiReader) { - pAPI->storeReader.storeReaderClose(pInfo->base.dataReader); + pAPI->tsdReader.storeReaderClose(pInfo->base.dataReader); source->dataReader = NULL; } pInfo->base.dataReader = NULL; @@ -2853,7 +2853,7 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { for (int32_t i = 0; i < numOfTable; ++i) { STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i); blockDataDestroy(param->inputBlock); - pAPI->storeReader.storeReaderClose(param->dataReader); + pAPI->tsdReader.storeReaderClose(param->dataReader); param->dataReader = NULL; } taosArrayClear(pInfo->sortSourceParams); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 5bbfb188ed..0e2ea77b86 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -2198,7 +2198,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, code); } - pAPI->storeReader.storeReaderGetDataBlockDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); + pAPI->tsdReader.storeReaderGetDataBlockDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); blockDistInfo.numOfInmemRows = (int32_t) pAPI->metaFn.getNumOfRowsInMem(pBlockScanInfo->pHandle); SSDataBlock* pBlock = pBlockScanInfo->pResBlock; @@ -2229,7 +2229,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { static void destroyBlockDistScanOperatorInfo(void* param) { SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param; blockDataDestroy(pDistInfo->pResBlock); - pDistInfo->readHandle.api.storeReader.storeReaderClose(pDistInfo->pHandle); + pDistInfo->readHandle.api.tsdReader.storeReaderClose(pDistInfo->pHandle); tableListDestroy(pDistInfo->pTableListInfo); taosMemoryFreeClear(param); } @@ -2284,7 +2284,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi size_t num = tableListGetSize(pTableListInfo); void* pList = tableListGetInfo(pTableListInfo, 0); - code = readHandle->api.storeReader.storeReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, (void**)&pInfo->pHandle, pTaskInfo->id.str, false, NULL); + code = readHandle->api.tsdReader.storeReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, (void**)&pInfo->pHandle, pTaskInfo->id.str, false, NULL); cleanupQueryTableDataCond(&cond); if (code != 0) { goto _error; From 85afbf0d353ed460e502b067c7bda282922418e7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 May 2023 18:58:54 +0800 Subject: [PATCH 20/97] refactor: do some internal refactor. --- include/libs/executor/storageapi.h | 31 ++++---- source/libs/executor/inc/executorInt.h | 4 +- source/libs/executor/src/executor.c | 16 ++-- source/libs/executor/src/executorInt.c | 2 +- source/libs/executor/src/operator.c | 4 +- source/libs/executor/src/scanoperator.c | 88 +++++++++++----------- source/libs/executor/src/sysscanoperator.c | 6 +- 7 files changed, 76 insertions(+), 75 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 67c0eacd26..8c79ba0500 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -225,22 +225,23 @@ typedef int32_t (*__store_reader_open_fn_t)(void *pVnode, SQueryTableDataCond *p int32_t numOfTables, SSDataBlock *pResBlock, void **ppReader, const char *idstr, bool countOnly, SHashObj **pIgnoreTables); -typedef struct SStoreTSDReader { - __store_reader_open_fn_t storeReaderOpen; - void (*storeReaderClose)(); - void (*setReaderId)(void *pReader, const char *pId); - void (*storeReaderSetTableList)(); - int32_t (*storeReaderNextDataBlock)(); - int32_t (*storeReaderRetrieveBlockSMA)(); +typedef struct TsdReader { + __store_reader_open_fn_t tsdReaderOpen; + void (*tsdReaderClose)(); + void (*tsdSetReaderTaskId)(void *pReader, const char *pId); + void (*tsdSetQueryTableList)(); + int32_t (*tsdReaderNextDataBlock)(); - SSDataBlock *(*storeReaderRetrieveDataBlock)(); - void (*storeReaderReleaseDataBlock)(); + int32_t (*tsdReaderRetrieveBlockSMAInfo)(); + SSDataBlock *(*tsdReaderRetrieveDataBlock)(); - void (*storeReaderResetStatus)(); - void (*storeReaderGetDataBlockDistInfo)(); - void (*storeReaderGetNumOfInMemRows)(); - void (*storeReaderNotifyClosing)(); -} SStoreTSDReader; + void (*tsdReaderReleaseDataBlock)(); + + void (*tsdReaderResetStatus)(); + void (*tsdReaderGetDataBlockDistInfo)(); + void (*tsdReaderGetNumOfInMemRows)(); + void (*tsdReaderNotifyClosing)(); +} TsdReader; /** * int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables); @@ -551,7 +552,7 @@ typedef struct SStateStore { typedef struct SStorageAPI { SStoreMeta metaFn; // todo: refactor - SStoreTSDReader tsdReader; + TsdReader tsdReader; SStoreMetaReader metaReaderFn; SStoreCacheReader cacheFn; SStoreSnapshotFn snapshotFn; diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 4c5ea4c242..eb585e7046 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -208,7 +208,7 @@ typedef struct STableScanBase { SLimitInfo limitInfo; // there are more than one table list exists in one task, if only one vnode exists. STableListInfo* pTableListInfo; - SStoreTSDReader readerAPI; + TsdReader readerAPI; } STableScanBase; typedef struct STableScanInfo { @@ -224,7 +224,7 @@ typedef struct STableScanInfo { int8_t assignBlockUid; bool hasGroupByTag; bool countOnly; - SStoreTSDReader readerAPI; + TsdReader readerAPI; } STableScanInfo; typedef struct STableMergeScanInfo { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index aa51b52c80..73a6532f73 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -166,7 +166,7 @@ void doSetTaskId(SOperatorInfo* pOperator, SStorageAPI *pAPI) { if (pStreamScanInfo->pTableScanOp != NULL) { STableScanInfo* pScanInfo = pStreamScanInfo->pTableScanOp->info; if (pScanInfo->base.dataReader != NULL) { - pAPI->tsdReader.setReaderId(pScanInfo->base.dataReader, pTaskInfo->id.str); + pAPI->tsdReader.tsdSetReaderTaskId(pScanInfo->base.dataReader, pTaskInfo->id.str); } } } else { @@ -1087,7 +1087,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT if (pOffset->type == TMQ_OFFSET__LOG) { // todo refactor: move away - pTaskInfo->storageAPI.tsdReader.storeReaderClose(pScanBaseInfo->dataReader); + pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pScanBaseInfo->dataReader); pScanBaseInfo->dataReader = NULL; ASSERT(0); @@ -1148,7 +1148,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT pScanInfo->scanTimes = 0; if (pScanBaseInfo->dataReader == NULL) { - int32_t code = pTaskInfo->storageAPI.tsdReader.storeReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, + int32_t code = pTaskInfo->storageAPI.tsdReader.tsdReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, pScanInfo->pResBlock, (void**) &pScanBaseInfo->dataReader, id, false, NULL); if (code != TSDB_CODE_SUCCESS) { qError("prepare read tsdb snapshot failed, uid:%" PRId64 ", code:%s %s", pOffset->uid, tstrerror(code), id); @@ -1159,8 +1159,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts:%" PRId64 " table index:%d, total:%d, %s", uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); } else { - pTaskInfo->storageAPI.tsdReader.storeReaderSetTableList(pScanBaseInfo->dataReader, &keyInfo, 1); - pTaskInfo->storageAPI.tsdReader.storeReaderResetStatus(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); + pTaskInfo->storageAPI.tsdReader.tsdSetQueryTableList(pScanBaseInfo->dataReader, &keyInfo, 1); + pTaskInfo->storageAPI.tsdReader.tsdReaderResetStatus(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 " table index:%d numOfTable:%d, %s", uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); } @@ -1189,7 +1189,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } SMetaTableInfo mtInfo = pTaskInfo->storageAPI.snapshotFn.getTableInfoFromSnapshot(sContext); - pTaskInfo->storageAPI.tsdReader.storeReaderClose(pInfo->dataReader); + pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pInfo->dataReader); pInfo->dataReader = NULL; cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); @@ -1207,7 +1207,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0); int32_t size = tableListGetSize(pTableListInfo); - pTaskInfo->storageAPI.tsdReader.storeReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, (void**) &pInfo->dataReader, NULL, + pTaskInfo->storageAPI.tsdReader.tsdReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, (void**) &pInfo->dataReader, NULL, false, NULL); cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); @@ -1228,7 +1228,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT id); } else if (pOffset->type == TMQ_OFFSET__LOG) { SStreamRawScanInfo* pInfo = pOperator->info; - pTaskInfo->storageAPI.tsdReader.storeReaderClose(pInfo->dataReader); + pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pInfo->dataReader); pInfo->dataReader = NULL; qDebug("tmqsnap qStreamPrepareScan snapshot log, %s", id); } diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 240803e243..45478cf837 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -1195,7 +1195,7 @@ void qStreamCloseTsdbReader(void* task) { qDebug("wait for the reader stopping"); } - pTaskInfo->storageAPI.tsdReader.storeReaderClose(pTSInfo->base.dataReader); + pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; // restore the status, todo refactor. diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 8d62b7a65f..8521fb623e 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -239,7 +239,7 @@ static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, STableScanInfo* pInfo = pOperator->info; if (pInfo->base.dataReader != NULL) { - pAPI->tsdReader.storeReaderNotifyClosing(pInfo->base.dataReader); + pAPI->tsdReader.tsdReaderNotifyClosing(pInfo->base.dataReader); } return OPTR_FN_RET_ABORT; } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { @@ -248,7 +248,7 @@ static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, if (pInfo->pTableScanOp != NULL) { STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; if (pTableScanInfo != NULL && pTableScanInfo->base.dataReader != NULL) { - pAPI->tsdReader.storeReaderNotifyClosing(pTableScanInfo->base.dataReader); + pAPI->tsdReader.tsdReaderNotifyClosing(pTableScanInfo->base.dataReader); } } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 4bed2d3a5d..8c880bb9b1 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -267,7 +267,7 @@ static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SStorageAPI* pAPI = &pTaskInfo->storageAPI; bool allColumnsHaveAgg = true; - int32_t code = pAPI->tsdReader.storeReaderRetrieveBlockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); + int32_t code = pAPI->tsdReader.tsdReaderRetrieveBlockSMAInfo(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -350,7 +350,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->filterOutBlocks += 1; pCost->totalRows += pBlock->info.rows; - pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", uid:%" PRIu64, @@ -358,7 +358,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pBlockInfo->id.uid); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); pCost->skipBlocks += 1; - pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_SMA_LOAD) { pCost->loadBlockStatis += 1; @@ -368,7 +368,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); - pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else { qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo)); @@ -390,7 +390,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->filterOutBlocks += 1; (*status) = FUNC_DATA_REQUIRED_FILTEROUT; - pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } } @@ -405,7 +405,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; - pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader); STableScanInfo* p1 = pOperator->info; if (taosHashGetSize(p1->pIgnoreTables) == taosArrayGetSize(p1->base.pTableListInfo->pTableList)) { @@ -419,7 +419,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->totalCheckedRows += pBlock->info.rows; pCost->loadBlocks += 1; - SSDataBlock* p = pAPI->tsdReader.storeReaderRetrieveDataBlock(pTableScanInfo->dataReader, NULL); + SSDataBlock* p = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pTableScanInfo->dataReader, NULL); if (p == NULL) { return terrno; } @@ -693,9 +693,9 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { int64_t st = taosGetTimestampUs(); while (true) { - code = pAPI->tsdReader.storeReaderNextDataBlock(pTableScanInfo->base.dataReader, &hasNext); + code = pAPI->tsdReader.tsdReaderNextDataBlock(pTableScanInfo->base.dataReader, &hasNext); if (code) { - pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->base.dataReader); T_LONG_JMP(pTaskInfo->env, code); } @@ -704,12 +704,12 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { } if (isTaskKilled(pTaskInfo)) { - pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->base.dataReader); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } if (pOperator->status == OP_EXEC_DONE) { - pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->base.dataReader); break; } @@ -774,7 +774,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { qDebug("start to repeat ascending order scan data blocks due to query func required, %s", GET_TASKID(pTaskInfo)); // do prepare for the next round table scan operation - pAPI->tsdReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pAPI->tsdReader.tsdReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); } } @@ -782,7 +782,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { if (pTableScanInfo->scanTimes < total) { if (pTableScanInfo->base.cond.order == TSDB_ORDER_ASC) { prepareForDescendingScan(&pTableScanInfo->base, pOperator->exprSupp.pCtx, 0); - pAPI->tsdReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pAPI->tsdReader.tsdReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); } @@ -800,7 +800,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { pTableScanInfo->base.scanFlag = MAIN_SCAN; qDebug("%s start to repeat descending order scan data blocks", GET_TASKID(pTaskInfo)); - pAPI->tsdReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pAPI->tsdReader.tsdReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); } } } @@ -839,11 +839,11 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable); taosRUnLockLatch(&pTaskInfo->lock); - pAPI->tsdReader.storeReaderSetTableList(pInfo->base.dataReader, &tInfo, 1); + pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, &tInfo, 1); qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", tInfo.uid, numOfTables, pInfo->currentTable, numOfTables, GET_TASKID(pTaskInfo)); - pAPI->tsdReader.storeReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); + pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; } } else { // scan table group by group sequentially @@ -858,7 +858,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); ASSERT(pInfo->base.dataReader == NULL); - int32_t code = pAPI->tsdReader.storeReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, + int32_t code = pAPI->tsdReader.tsdReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly, &pInfo->pIgnoreTables); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); @@ -887,8 +887,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableKeyInfo* pList = NULL; tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); - pAPI->tsdReader.storeReaderSetTableList(pInfo->base.dataReader, pList, num); - pAPI->tsdReader.storeReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); + pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, pList, num); + pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; result = doGroupedTableScan(pOperator); @@ -910,10 +910,10 @@ static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptr return 0; } -static void destroyTableScanBase(STableScanBase* pBase, SStoreTSDReader* pAPI) { +static void destroyTableScanBase(STableScanBase* pBase, TsdReader* pAPI) { cleanupQueryTableDataCond(&pBase->cond); - pAPI->storeReaderClose(pBase->dataReader); + pAPI->tsdReaderClose(pBase->dataReader); pBase->dataReader = NULL; if (pBase->matchInfo.pList != NULL) { @@ -1074,7 +1074,7 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint6 pTableScanInfo->base.cond.endVersion = version; pTableScanInfo->scanTimes = 0; pTableScanInfo->currentGroupId = -1; - pTableScanInfo->readerAPI.storeReaderClose(pTableScanInfo->base.dataReader); + pTableScanInfo->readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; } @@ -1094,7 +1094,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU SSDataBlock* pBlock = pTableScanInfo->pResBlock; STsdbReader* pReader = NULL; - int32_t code = pAPI->tsdReader.storeReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, + int32_t code = pAPI->tsdReader.tsdReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, (void**)&pReader, GET_TASKID(pTaskInfo), false, NULL); if (code != TSDB_CODE_SUCCESS) { terrno = code; @@ -1103,7 +1103,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } bool hasNext = false; - code = pAPI->tsdReader.storeReaderNextDataBlock(pReader, &hasNext); + code = pAPI->tsdReader.tsdReaderNextDataBlock(pReader, &hasNext); if (code != TSDB_CODE_SUCCESS) { terrno = code; T_LONG_JMP(pTaskInfo->env, code); @@ -1111,12 +1111,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } if (hasNext) { - /*SSDataBlock* p = */ pAPI->tsdReader.storeReaderRetrieveDataBlock(pReader, NULL); + /*SSDataBlock* p = */ pAPI->tsdReader.tsdReaderRetrieveDataBlock(pReader, NULL); doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); pBlock->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); } - pAPI->tsdReader.storeReaderClose(pReader); + pAPI->tsdReader.tsdReaderClose(pReader); qDebug("retrieve prev rows:%" PRId64 ", skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64 ", suid:%" PRIu64, pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid); @@ -1249,7 +1249,7 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 *pRowIndex = 0; pInfo->updateWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - pTableScanInfo->readerAPI.storeReaderClose(pTableScanInfo->base.dataReader); + pTableScanInfo->readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; return NULL; } @@ -1657,7 +1657,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { return pResult; } STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - pAPI->tsdReader.storeReaderClose(pTSInfo->base.dataReader); + pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1); @@ -1821,7 +1821,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN2; } - pAPI->tsdReader.storeReaderClose(pTSInfo->base.dataReader); + pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; pInfo->pTableScanOp->status = OP_OPENED; @@ -1901,7 +1901,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { } pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - pAPI->tsdReader.storeReaderClose(pTSInfo->base.dataReader); + pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; @@ -2162,20 +2162,20 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { bool hasNext = false; if (pInfo->dataReader) { - code = pAPI->tsdReader.storeReaderNextDataBlock(pInfo->dataReader, &hasNext); + code = pAPI->tsdReader.tsdReaderNextDataBlock(pInfo->dataReader, &hasNext); if (code) { - pAPI->tsdReader.storeReaderReleaseDataBlock(pInfo->dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pInfo->dataReader); T_LONG_JMP(pTaskInfo->env, code); } } if (pInfo->dataReader && hasNext) { if (isTaskKilled(pTaskInfo)) { - pAPI->tsdReader.storeReaderReleaseDataBlock(pInfo->dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pInfo->dataReader); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } - SSDataBlock* pBlock = pAPI->tsdReader.storeReaderRetrieveDataBlock(pInfo->dataReader, NULL); + SSDataBlock* pBlock = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pInfo->dataReader, NULL); if (pBlock == NULL) { T_LONG_JMP(pTaskInfo->env, terrno); } @@ -2227,7 +2227,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { static void destroyRawScanOperatorInfo(void* param) { SStreamRawScanInfo* pRawScan = (SStreamRawScanInfo*)param; - pRawScan->pAPI->tsdReader.storeReaderClose(pRawScan->dataReader); + pRawScan->pAPI->tsdReader.tsdReaderClose(pRawScan->dataReader); pRawScan->pAPI->snapshotFn.destroySnapshot(pRawScan->sContext); tableListDestroy(pRawScan->pTableListInfo); taosMemoryFree(pRawScan); @@ -2662,7 +2662,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { SReadHandle* pHandle = &pInfo->base.readHandle; if (NULL == source->dataReader || !source->multiReader) { - code = pAPI->tsdReader.storeReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, (void**)&source->dataReader, GET_TASKID(pTaskInfo), false, NULL); + code = pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, (void**)&source->dataReader, GET_TASKID(pTaskInfo), false, NULL); if (code != 0) { T_LONG_JMP(pTaskInfo->env, code); } @@ -2674,9 +2674,9 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { qTrace("tsdb/read-table-data: %p, enter next reader", reader); while (true) { - code = pAPI->tsdReader.storeReaderNextDataBlock(reader, &hasNext); + code = pAPI->tsdReader.tsdReaderNextDataBlock(reader, &hasNext); if (code != 0) { - pAPI->tsdReader.storeReaderReleaseDataBlock(reader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(reader); pInfo->base.dataReader = NULL; T_LONG_JMP(pTaskInfo->env, code); } @@ -2686,7 +2686,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } if (isTaskKilled(pTaskInfo)) { - pAPI->tsdReader.storeReaderReleaseDataBlock(reader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(reader); pInfo->base.dataReader = NULL; T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } @@ -2726,7 +2726,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { qTrace("tsdb/read-table-data: %p, close reader", reader); if (!source->multiReader) { - pAPI->tsdReader.storeReaderClose(pInfo->base.dataReader); + pAPI->tsdReader.tsdReaderClose(pInfo->base.dataReader); source->dataReader = NULL; } pInfo->base.dataReader = NULL; @@ -2734,7 +2734,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } if (!source->multiReader) { - pAPI->tsdReader.storeReaderClose(pInfo->base.dataReader); + pAPI->tsdReader.tsdReaderClose(pInfo->base.dataReader); source->dataReader = NULL; } pInfo->base.dataReader = NULL; @@ -2853,7 +2853,7 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { for (int32_t i = 0; i < numOfTable; ++i) { STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i); blockDataDestroy(param->inputBlock); - pAPI->tsdReader.storeReaderClose(param->dataReader); + pAPI->tsdReader.tsdReaderClose(param->dataReader); param->dataReader = NULL; } taosArrayClear(pInfo->sortSourceParams); @@ -2965,11 +2965,11 @@ void destroyTableMergeScanOperatorInfo(void* param) { for (int32_t i = 0; i < numOfTable; i++) { STableMergeScanSortSourceParam* p = taosArrayGet(pTableScanInfo->sortSourceParams, i); blockDataDestroy(p->inputBlock); - pTableScanInfo->base.readerAPI.storeReaderClose(p->dataReader); + pTableScanInfo->base.readerAPI.tsdReaderClose(p->dataReader); p->dataReader = NULL; } - pTableScanInfo->base.readerAPI.storeReaderClose(pTableScanInfo->base.dataReader); + pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; taosArrayDestroy(pTableScanInfo->sortSourceParams); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 0e2ea77b86..76302a4eb4 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -2198,7 +2198,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, code); } - pAPI->tsdReader.storeReaderGetDataBlockDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); + pAPI->tsdReader.tsdReaderGetDataBlockDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); blockDistInfo.numOfInmemRows = (int32_t) pAPI->metaFn.getNumOfRowsInMem(pBlockScanInfo->pHandle); SSDataBlock* pBlock = pBlockScanInfo->pResBlock; @@ -2229,7 +2229,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { static void destroyBlockDistScanOperatorInfo(void* param) { SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param; blockDataDestroy(pDistInfo->pResBlock); - pDistInfo->readHandle.api.tsdReader.storeReaderClose(pDistInfo->pHandle); + pDistInfo->readHandle.api.tsdReader.tsdReaderClose(pDistInfo->pHandle); tableListDestroy(pDistInfo->pTableListInfo); taosMemoryFreeClear(param); } @@ -2284,7 +2284,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi size_t num = tableListGetSize(pTableListInfo); void* pList = tableListGetInfo(pTableListInfo, 0); - code = readHandle->api.tsdReader.storeReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, (void**)&pInfo->pHandle, pTaskInfo->id.str, false, NULL); + code = readHandle->api.tsdReader.tsdReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, (void**)&pInfo->pHandle, pTaskInfo->id.str, false, NULL); cleanupQueryTableDataCond(&cond); if (code != 0) { goto _error; From 66477a28ca8552fae482a527bf631a7aec2d738d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 May 2023 19:10:50 +0800 Subject: [PATCH 21/97] refactor: do some internal refactor. --- include/libs/executor/executor.h | 4 +- include/libs/executor/storageapi.h | 4 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 206 ++++++++++----------- source/libs/executor/src/executil.c | 2 +- source/libs/executor/src/executor.c | 6 +- source/libs/executor/src/querytask.c | 2 +- source/libs/executor/src/scanoperator.c | 22 +-- source/libs/executor/src/sysscanoperator.c | 26 +-- source/libs/qworker/src/qwMsg.c | 6 +- 9 files changed, 137 insertions(+), 141 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 7f86b8f5e2..c4c2c89fdb 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -42,7 +42,7 @@ typedef struct { typedef struct { void* tqReader; - void* meta; +// void* meta; void* config; void* vnode; void* mnd; @@ -54,7 +54,7 @@ typedef struct { int32_t numOfVgroups; void* sContext; // SSnapContext* - void* pStateBackend; + void* pStateBackend; struct SStorageAPI api; } SReadHandle; diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 8c79ba0500..c0fd6c34fc 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -243,6 +243,7 @@ typedef struct TsdReader { void (*tsdReaderNotifyClosing)(); } TsdReader; + /** * int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, @@ -262,7 +263,6 @@ typedef struct SStoreCacheReader { /*------------------------------------------------------------------------------------------------------------------*/ /* - * void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList); int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList); @@ -552,7 +552,7 @@ typedef struct SStateStore { typedef struct SStorageAPI { SStoreMeta metaFn; // todo: refactor - TsdReader tsdReader; + TsdReader tsdReader; SStoreMetaReader metaReaderFn; SStoreCacheReader cacheFn; SStoreSnapshotFn snapshotFn; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index fe1ccf90c8..0f08b63294 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -19,23 +19,23 @@ #include "vnode.h" #include "vnodeInt.h" -static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodePreprocessCreateTableReq(SVnode *pVnode, SDecoder *pCoder, int64_t ctime, int64_t *pUid) { int32_t code = 0; @@ -301,32 +301,32 @@ _exit: return code; } -int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp) { +int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg *pRsp) { void *ptr = NULL; void *pReq; int32_t len; int32_t ret; - if (version <= pVnode->state.applied) { - vError("vgId:%d, duplicate write request. version: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), version, + if (ver <= pVnode->state.applied) { + vError("vgId:%d, duplicate write request. ver: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), ver, pVnode->state.applied); terrno = TSDB_CODE_VND_DUP_REQUEST; return -1; } vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), - version); + ver); ASSERT(pVnode->state.applyTerm <= pMsg->info.conn.applyTerm); - ASSERT(pVnode->state.applied + 1 == version); + ASSERT(pVnode->state.applied + 1 == ver); - atomic_store_64(&pVnode->state.applied, version); + atomic_store_64(&pVnode->state.applied, ver); atomic_store_64(&pVnode->state.applyTerm, pMsg->info.conn.applyTerm); if (!syncUtilUserCommit(pMsg->msgType)) goto _exit; if (pMsg->msgType == TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE || pMsg->msgType == TDMT_STREAM_TASK_CHECK_RSP) { - if (tqCheckLogInWal(pVnode->pTq, version)) return 0; + if (tqCheckLogInWal(pVnode->pTq, ver)) return 0; } // skip header @@ -337,123 +337,123 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp switch (pMsg->msgType) { /* META */ case TDMT_VND_CREATE_STB: - if (vnodeProcessCreateStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessCreateStbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_ALTER_STB: - if (vnodeProcessAlterStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessAlterStbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DROP_STB: - if (vnodeProcessDropStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessDropStbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_CREATE_TABLE: - if (vnodeProcessCreateTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessCreateTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_ALTER_TABLE: - if (vnodeProcessAlterTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessAlterTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DROP_TABLE: - if (vnodeProcessDropTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessDropTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DROP_TTL_TABLE: - if (vnodeProcessDropTtlTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessDropTtlTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_TRIM: - if (vnodeProcessTrimReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessTrimReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_CREATE_SMA: - if (vnodeProcessCreateTSmaReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessCreateTSmaReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; /* TSDB */ case TDMT_VND_SUBMIT: - if (vnodeProcessSubmitReq(pVnode, version, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err; + if (vnodeProcessSubmitReq(pVnode, ver, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err; break; case TDMT_VND_DELETE: - if (vnodeProcessDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessDeleteReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_BATCH_DEL: - if (vnodeProcessBatchDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessBatchDeleteReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; /* TQ */ case TDMT_VND_TMQ_SUBSCRIBE: - if (tqProcessSubscribeReq(pVnode->pTq, version, pReq, len) < 0) { + if (tqProcessSubscribeReq(pVnode->pTq, ver, pReq, len) < 0) { goto _err; } break; case TDMT_VND_TMQ_DELETE_SUB: - if (tqProcessDeleteSubReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { + if (tqProcessDeleteSubReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) { goto _err; } break; case TDMT_VND_TMQ_COMMIT_OFFSET: - if (tqProcessOffsetCommitReq(pVnode->pTq, version, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) { + if (tqProcessOffsetCommitReq(pVnode->pTq, ver, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) { goto _err; } break; case TDMT_VND_TMQ_SEEK_TO_OFFSET: - if (tqProcessSeekReq(pVnode->pTq, version, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) { + if (tqProcessSeekReq(pVnode->pTq, ver, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) { goto _err; } break; case TDMT_VND_TMQ_ADD_CHECKINFO: - if (tqProcessAddCheckInfoReq(pVnode->pTq, version, pReq, len) < 0) { + if (tqProcessAddCheckInfoReq(pVnode->pTq, ver, pReq, len) < 0) { goto _err; } break; case TDMT_VND_TMQ_DEL_CHECKINFO: - if (tqProcessDelCheckInfoReq(pVnode->pTq, version, pReq, len) < 0) { + if (tqProcessDelCheckInfoReq(pVnode->pTq, ver, pReq, len) < 0) { goto _err; } break; case TDMT_STREAM_TASK_DEPLOY: { - if (pVnode->restored && tqProcessTaskDeployReq(pVnode->pTq, version, pReq, len) < 0) { + if (pVnode->restored && tqProcessTaskDeployReq(pVnode->pTq, ver, pReq, len) < 0) { goto _err; } } break; case TDMT_STREAM_TASK_DROP: { - if (tqProcessTaskDropReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { + if (tqProcessTaskDropReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) { goto _err; } } break; case TDMT_STREAM_TASK_PAUSE: { - if (pVnode->restored && tqProcessTaskPauseReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { + if (pVnode->restored && tqProcessTaskPauseReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) { goto _err; } } break; case TDMT_STREAM_TASK_RESUME: { - if (pVnode->restored && tqProcessTaskResumeReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { + if (pVnode->restored && tqProcessTaskResumeReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) { goto _err; } } break; case TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE: { - if (tqProcessTaskRecover2Req(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { + if (tqProcessTaskRecover2Req(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) { goto _err; } } break; case TDMT_STREAM_TASK_CHECK_RSP: { - if (tqProcessStreamTaskCheckRsp(pVnode->pTq, version, pReq, len) < 0) { + if (tqProcessStreamTaskCheckRsp(pVnode->pTq, ver, pReq, len) < 0) { goto _err; } } break; case TDMT_VND_ALTER_CONFIRM: needCommit = pVnode->config.hashChange; - if (vnodeProcessAlterConfirmReq(pVnode, version, pReq, len, pRsp) < 0) { + if (vnodeProcessAlterConfirmReq(pVnode, ver, pReq, len, pRsp) < 0) { goto _err; } break; case TDMT_VND_ALTER_CONFIG: - vnodeProcessAlterConfigReq(pVnode, version, pReq, len, pRsp); + vnodeProcessAlterConfigReq(pVnode, ver, pReq, len, pRsp); break; case TDMT_VND_COMMIT: needCommit = true; break; case TDMT_VND_CREATE_INDEX: - vnodeProcessCreateIndexReq(pVnode, version, pReq, len, pRsp); + vnodeProcessCreateIndexReq(pVnode, ver, pReq, len, pRsp); break; case TDMT_VND_DROP_INDEX: - vnodeProcessDropIndexReq(pVnode, version, pReq, len, pRsp); + vnodeProcessDropIndexReq(pVnode, ver, pReq, len, pRsp); break; case TDMT_VND_COMPACT: - vnodeProcessCompactVnodeReq(pVnode, version, pReq, len, pRsp); + vnodeProcessCompactVnodeReq(pVnode, ver, pReq, len, pRsp); goto _exit; default: vError("vgId:%d, unprocessed msg, %d", TD_VID(pVnode), pMsg->msgType); @@ -461,18 +461,18 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp } vTrace("vgId:%d, process %s request, code:0x%x index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), pRsp->code, - version); + ver); - walApplyVer(pVnode->pWal, version); + walApplyVer(pVnode->pWal, ver); - if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) { + if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, ver) < 0) { vError("vgId:%d, failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } // commit if need if (needCommit) { - vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); + vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), ver); if (vnodeAsyncCommit(pVnode) < 0) { vError("vgId:%d, failed to vnode async commit since %s.", TD_VID(pVnode), tstrerror(terrno)); goto _err; @@ -489,8 +489,8 @@ _exit: return 0; _err: - vError("vgId:%d, process %s request failed since %s, version:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), - tstrerror(terrno), version); + vError("vgId:%d, process %s request failed since %s, ver:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), + tstrerror(terrno), ver); return -1; } @@ -514,7 +514,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { return 0; } - SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; + SReadHandle handle = {.config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; switch (pMsg->msgType) { case TDMT_SCH_QUERY: case TDMT_SCH_MERGE_QUERY: @@ -603,7 +603,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { } extern int32_t vnodeAsyncRentention(SVnode *pVnode, int64_t now); -static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; SVTrimDbReq trimReq = {0}; @@ -624,7 +624,7 @@ _exit: return code; } -static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SArray *tbUids = taosArrayInit(8, sizeof(int64_t)); if (tbUids == NULL) return TSDB_CODE_OUT_OF_MEMORY; @@ -650,7 +650,7 @@ end: return ret; } -static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVCreateStbReq req = {0}; SDecoder coder; @@ -667,7 +667,7 @@ static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *p goto _err; } - if (metaCreateSTable(pVnode->pMeta, version, &req) < 0) { + if (metaCreateSTable(pVnode->pMeta, ver, &req) < 0) { pRsp->code = terrno; goto _err; } @@ -685,7 +685,7 @@ _err: return -1; } -static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SDecoder decoder = {0}; SEncoder encoder = {0}; int32_t rcode = 0; @@ -742,7 +742,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pR } // do create table - if (metaCreateTable(pVnode->pMeta, version, pCreateReq, &cRsp.pMeta) < 0) { + if (metaCreateTable(pVnode->pMeta, ver, pCreateReq, &cRsp.pMeta) < 0) { if (pCreateReq->flags & TD_CREATE_IF_NOT_EXISTS && terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { cRsp.code = TSDB_CODE_SUCCESS; } else { @@ -790,7 +790,7 @@ _exit: return rcode; } -static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVCreateStbReq req = {0}; SDecoder dc = {0}; @@ -808,7 +808,7 @@ static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pR return -1; } - if (metaAlterSTable(pVnode->pMeta, version, &req) < 0) { + if (metaAlterSTable(pVnode->pMeta, ver, &req) < 0) { pRsp->code = terrno; tDecoderClear(&dc); return -1; @@ -819,7 +819,7 @@ static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pR return 0; } -static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVDropStbReq req = {0}; int32_t rcode = TSDB_CODE_SUCCESS; SDecoder decoder = {0}; @@ -839,7 +839,7 @@ static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pRe // process request tbUidList = taosArrayInit(8, sizeof(int64_t)); if (tbUidList == NULL) goto _exit; - if (metaDropSTable(pVnode->pMeta, version, &req, tbUidList) < 0) { + if (metaDropSTable(pVnode->pMeta, ver, &req, tbUidList) < 0) { rcode = terrno; goto _exit; } @@ -862,7 +862,7 @@ _exit: return 0; } -static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVAlterTbReq vAlterTbReq = {0}; SVAlterTbRsp vAlterTbRsp = {0}; SDecoder dc = {0}; @@ -887,7 +887,7 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pRe } // process - if (metaAlterTable(pVnode->pMeta, version, &vAlterTbReq, &vMetaRsp) < 0) { + if (metaAlterTable(pVnode->pMeta, ver, &vAlterTbReq, &vMetaRsp) < 0) { vAlterTbRsp.code = terrno; tDecoderClear(&dc); rcode = -1; @@ -912,7 +912,7 @@ _exit: return 0; } -static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVDropTbBatchReq req = {0}; SVDropTbBatchRsp rsp = {0}; SDecoder decoder = {0}; @@ -946,7 +946,7 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq tb_uid_t tbUid = 0; /* code */ - ret = metaDropTable(pVnode->pMeta, version, pDropTbReq, tbUids, &tbUid); + ret = metaDropTable(pVnode->pMeta, ver, pDropTbReq, tbUids, &tbUid); if (ret < 0) { if (pDropTbReq->igNotExists && terrno == TSDB_CODE_TDB_TABLE_NOT_EXIST) { dropTbRsp.code = TSDB_CODE_SUCCESS; @@ -1189,7 +1189,7 @@ static int32_t vnodeRebuildSubmitReqMsg(SSubmitReq2 *pSubmitReq, void **ppMsg) { return code; } -static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; terrno = 0; @@ -1203,7 +1203,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq void *pAllocMsg = NULL; SSubmitReq2Msg *pMsg = (SSubmitReq2Msg *)pReq; - if (0 == pMsg->version) { + if (0 == pMsg->ver) { code = vnodeSubmitReqConvertToSubmitReq2(pVnode, (SSubmitReq *)pMsg, pSubmitReq); if (TSDB_CODE_SUCCESS == code) { code = vnodeRebuildSubmitReqMsg(pSubmitReq, &pReq); @@ -1251,7 +1251,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq for (int32_t iRow = 0; iRow < pColData->nVal; iRow++) { if (aKey[iRow] < minKey || aKey[iRow] > maxKey || (iRow > 0 && aKey[iRow] <= aKey[iRow - 1])) { code = TSDB_CODE_INVALID_MSG; - vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), version); + vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), ver); goto _exit; } } @@ -1263,7 +1263,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq for (int32_t iRow = 0; iRow < nRow; ++iRow) { if (aRow[iRow]->ts < minKey || aRow[iRow]->ts > maxKey || (iRow > 0 && aRow[iRow]->ts <= aRow[iRow - 1]->ts)) { code = TSDB_CODE_INVALID_MSG; - vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), version); + vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), ver); goto _exit; } } @@ -1350,7 +1350,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq SVCreateTbRsp *pCreateTbRsp = taosArrayReserve(pSubmitRsp->aCreateTbRsp, 1); // create table - if (metaCreateTable(pVnode->pMeta, version, pSubmitTbData->pCreateTbReq, &pCreateTbRsp->pMeta) == 0) { + if (metaCreateTable(pVnode->pMeta, ver, pSubmitTbData->pCreateTbReq, &pCreateTbRsp->pMeta) == 0) { // create table success if (newTbUids == NULL && @@ -1376,7 +1376,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq // insert data int32_t affectedRows; - code = tsdbInsertTableData(pVnode->pTsdb, version, pSubmitTbData, &affectedRows); + code = tsdbInsertTableData(pVnode->pTsdb, ver, pSubmitTbData, &affectedRows); if (code) goto _exit; pSubmitRsp->affectedRows += affectedRows; @@ -1404,7 +1404,7 @@ _exit: atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1); if (code == 0) { atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1); - tdProcessRSmaSubmit(pVnode->pSma, version, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT); + tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT); } // clear @@ -1419,7 +1419,7 @@ _exit: return code; } -static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVCreateTSmaReq req = {0}; SDecoder coder = {0}; @@ -1439,20 +1439,20 @@ static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void * goto _err; } - if (tdProcessTSmaCreate(pVnode->pSma, version, (const char *)&req) < 0) { + if (tdProcessTSmaCreate(pVnode->pSma, ver, (const char *)&req) < 0) { if (pRsp) pRsp->code = terrno; goto _err; } tDecoderClear(&coder); vDebug("vgId:%d, success to create tsma %s:%" PRIi64 " version %" PRIi64 " for table %" PRIi64, TD_VID(pVnode), - req.indexName, req.indexUid, version, req.tableUid); + req.indexName, req.indexUid, ver, req.tableUid); return 0; _err: tDecoderClear(&coder); vError("vgId:%d, failed to create tsma %s:%" PRIi64 " version %" PRIi64 "for table %" PRIi64 " since %s", - TD_VID(pVnode), req.indexName, req.indexUid, version, req.tableUid, terrstr()); + TD_VID(pVnode), req.indexName, req.indexUid, ver, req.tableUid, terrstr()); return -1; } @@ -1468,28 +1468,28 @@ int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen) { return vnodeProcessCreateTSmaReq(pVnode, 1, pCont, contLen, NULL); } -static int32_t vnodeConsolidateAlterHashRange(SVnode *pVnode, int64_t version) { +static int32_t vnodeConsolidateAlterHashRange(SVnode *pVnode, int64_t ver) { int32_t code = TSDB_CODE_SUCCESS; vInfo("vgId:%d, trim meta of tables per hash range [%" PRIu32 ", %" PRIu32 "]. apply-index:%" PRId64, TD_VID(pVnode), - pVnode->config.hashBegin, pVnode->config.hashEnd, version); + pVnode->config.hashBegin, pVnode->config.hashEnd, ver); // TODO: trim meta of tables from TDB per hash range [pVnode->config.hashBegin, pVnode->config.hashEnd] return code; } -static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { vInfo("vgId:%d, vnode handle msgType:alter-confirm, alter confim msg is processed", TD_VID(pVnode)); int32_t code = TSDB_CODE_SUCCESS; if (!pVnode->config.hashChange) { goto _exit; } - code = vnodeConsolidateAlterHashRange(pVnode, version); + code = vnodeConsolidateAlterHashRange(pVnode, ver); if (code < 0) { vError("vgId:%d, failed to consolidate alter hashrange since %s. version:%" PRId64, TD_VID(pVnode), terrstr(), - version); + ver); goto _exit; } pVnode->config.hashChange = false; @@ -1503,7 +1503,7 @@ _exit: return code; } -static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { bool walChanged = false; bool tsdbChanged = false; @@ -1606,7 +1606,7 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void return 0; } -static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SBatchDeleteReq deleteReq; SDecoder decoder; tDecoderInit(&decoder, pReq, len); @@ -1626,7 +1626,7 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void int64_t uid = mr.me.uid; - int32_t code = tsdbDeleteTableData(pVnode->pTsdb, version, deleteReq.suid, uid, pOneReq->startTs, pOneReq->endTs); + int32_t code = tsdbDeleteTableData(pVnode->pTsdb, ver, deleteReq.suid, uid, pOneReq->startTs, pOneReq->endTs); if (code < 0) { terrno = code; vError("vgId:%d, delete error since %s, suid:%" PRId64 ", uid:%" PRId64 ", start ts:%" PRId64 ", end ts:%" PRId64, @@ -1640,7 +1640,7 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void return 0; } -static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; SDecoder *pCoder = &(SDecoder){0}; SDeleteRes *pRes = &(SDeleteRes){0}; @@ -1661,7 +1661,7 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq ASSERT(taosArrayGetSize(pRes->uidList) == 0 || (pRes->skey != 0 && pRes->ekey != 0)); for (int32_t iUid = 0; iUid < taosArrayGetSize(pRes->uidList); iUid++) { - code = tsdbDeleteTableData(pVnode->pTsdb, version, pRes->suid, *(uint64_t *)taosArrayGet(pRes->uidList, iUid), + code = tsdbDeleteTableData(pVnode->pTsdb, ver, pRes->suid, *(uint64_t *)taosArrayGet(pRes->uidList, iUid), pRes->skey, pRes->ekey); if (code) goto _err; } @@ -1682,7 +1682,7 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq _err: return code; } -static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVCreateStbReq req = {0}; SDecoder dc = {0}; @@ -1698,7 +1698,7 @@ static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t version, void tDecoderClear(&dc); return -1; } - if (metaAddIndexToSTable(pVnode->pMeta, version, &req) < 0) { + if (metaAddIndexToSTable(pVnode->pMeta, ver, &req) < 0) { pRsp->code = terrno; goto _err; } @@ -1708,7 +1708,7 @@ _err: tDecoderClear(&dc); return -1; } -static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SDropIndexReq req = {0}; pRsp->msgType = TDMT_VND_DROP_INDEX_RSP; pRsp->code = TSDB_CODE_SUCCESS; @@ -1720,21 +1720,21 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *p return -1; } - if (metaDropIndexFromSTable(pVnode->pMeta, version, &req) < 0) { + if (metaDropIndexFromSTable(pVnode->pMeta, ver, &req) < 0) { pRsp->code = terrno; return -1; } return TSDB_CODE_SUCCESS; } -extern int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +extern int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - return vnodeProcessCompactVnodeReqImpl(pVnode, version, pReq, len, pRsp); +static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { + return vnodeProcessCompactVnodeReqImpl(pVnode, ver, pReq, len, pRsp); } #ifndef TD_ENTERPRISE -int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { return 0; } #endif diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index ec23f7b724..b24fcade79 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2055,7 +2055,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pTableListInfo->numOfOuputGroups = 1; } } else { - code = getColInfoResultForGroupby(pHandle->meta, group, pTableListInfo, digest, pAPI); + code = getColInfoResultForGroupby(pHandle->vnode, group, pTableListInfo, digest, pAPI); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 73a6532f73..cad98348d0 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -340,7 +340,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S // let's discard the tables those are not created according to the queried super table. SMetaReader mr = {0}; - pAPI->metaReaderFn.initReader(&mr, pScanInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&mr, pScanInfo->readHandle.vnode, 0); for (int32_t i = 0; i < numOfUids; ++i) { uint64_t* id = (uint64_t*)taosArrayGet(tableIdList, i); @@ -372,7 +372,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S if (pScanInfo->pTagCond != NULL) { bool qualified = false; STableKeyInfo info = {.groupId = 0, .uid = mr.me.uid}; - code = isQualifiedTable(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified, pAPI); + code = isQualifiedTable(&info, pScanInfo->pTagCond, pScanInfo->readHandle.vnode, &qualified, pAPI); if (code != TSDB_CODE_SUCCESS) { qError("failed to filter new table, uid:0x%" PRIx64 ", %s", info.uid, idstr); continue; @@ -437,7 +437,7 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI if (assignUid) { keyInfo.groupId = keyInfo.uid; } else { - code = getGroupIdFromTagsVal(pScanInfo->readHandle.meta, keyInfo.uid, pScanInfo->pGroupTags, keyBuf, + code = getGroupIdFromTagsVal(pScanInfo->readHandle.vnode, keyInfo.uid, pScanInfo->pGroupTags, keyBuf, &keyInfo.groupId, &pTaskInfo->storageAPI); if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(keyBuf); diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c index a48de1d158..a684a6bf94 100644 --- a/source/libs/executor/src/querytask.c +++ b/source/libs/executor/src/querytask.c @@ -122,7 +122,7 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo SStorageAPI* pAPI = &pTaskInfo->storageAPI; - pAPI->metaReaderFn.initReader(&mr, pHandle->meta, 0); + pAPI->metaReaderFn.initReader(&mr, pHandle->vnode, 0); int32_t code = pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, pScanNode->uid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get the table meta, uid:0x%" PRIx64 ", suid:0x%" PRIx64 ", %s", pScanNode->uid, pScanNode->suid, diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8c880bb9b1..23e029e780 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -531,7 +531,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int // 1. check if it is existed in meta cache if (pCache == NULL) { - pHandle->api.metaReaderFn.initReader(&mr, pHandle->meta, 0); + pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0); code = pHandle->api.metaReaderFn.readerGetEntryGetUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { // when encounter the TSDB_CODE_PAR_TABLE_NOT_EXIST error, we proceed. @@ -560,7 +560,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int h = taosLRUCacheLookup(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid)); if (h == NULL) { - pHandle->api.metaReaderFn.initReader(&mr, pHandle->meta, 0); + pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0); code = pHandle->api.metaReaderFn.readerGetEntryGetUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { @@ -2556,7 +2556,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { char str[512] = {0}; int32_t count = 0; SMetaReader mr = {0}; - pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, 0); while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { doTagScanOneTable(pOperator, pRes, count, &mr, &pTaskInfo->storageAPI); @@ -3396,7 +3396,7 @@ static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountSca pRes->info.id.groupId = groupId; int64_t dbTableCount = 0; - pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.meta, &dbTableCount); + pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &dbTableCount); fillTableCountScanDataBlock(pSupp, dbName, "", dbTableCount, pRes); setOperatorCompleted(pOperator); } @@ -3410,18 +3410,18 @@ static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanO if (strlen(pSupp->dbNameFilter) != 0) { if (strlen(pSupp->stbNameFilter) != 0) { tb_uid_t uid = 0; - pAPI->metaFn.getTableUidByName(pInfo->readHandle.meta, pSupp->stbNameFilter, &uid); + pAPI->metaFn.getTableUidByName(pInfo->readHandle.vnode, pSupp->stbNameFilter, &uid); SMetaStbStats stats = {0}; ASSERT(0); -// metaGetStbStats(pInfo->readHandle.meta, uid, &stats); +// metaGetStbStats(pInfo->readHandle.vnode, uid, &stats); int64_t ctbNum = stats.ctbNum; fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, ctbNum, pRes); } else { - int64_t tbNumVnode = 0;//metaGetTbNum(pInfo->readHandle.meta); + int64_t tbNumVnode = 0;//metaGetTbNum(pInfo->readHandle.vnode); fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } } else { - int64_t tbNumVnode = 0;//metaGetTbNum(pInfo->readHandle.meta); + int64_t tbNumVnode = 0;//metaGetTbNum(pInfo->readHandle.vnode); pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode); fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } @@ -3437,7 +3437,7 @@ static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, S uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); pRes->info.id.groupId = groupId; - int64_t ntbNum = 0;//metaGetNtbNum(pInfo->readHandle.meta); + int64_t ntbNum = 0;//metaGetNtbNum(pInfo->readHandle.vnode); ASSERT(0); if (ntbNum != 0) { fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes); @@ -3448,7 +3448,7 @@ static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, S SSDataBlock* pRes, char* dbName, tb_uid_t stbUid) { char stbName[TSDB_TABLE_NAME_LEN] = {0}; ASSERT(0); -// metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName); +// metaGetTableSzNameByUid(pInfo->readHandle.vnode, stbUid, stbName); char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; if (pSupp->groupByDbName) { @@ -3461,7 +3461,7 @@ static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, S pRes->info.id.groupId = groupId; SMetaStbStats stats = {0}; -// metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats); +// metaGetStbStats(pInfo->readHandle.vnode, stbUid, &stats); int64_t ctbNum = stats.ctbNum; fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 76302a4eb4..a9450fb183 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -466,7 +466,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { STR_TO_VARSTR(tableName, pInfo->req.filterTb); SMetaReader smrTable = {0}; - pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, 0); int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrTable, pInfo->req.filterTb); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly @@ -486,7 +486,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { if (smrTable.me.type == TSDB_CHILD_TABLE) { int64_t suid = smrTable.me.ctbEntry.suid; pAPI->metaReaderFn.clearReader(&smrTable); - pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, 0); code = pAPI->metaReaderFn.getTableEntryByUid(&smrTable, suid); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly @@ -522,7 +522,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { int32_t ret = 0; if (pInfo->pCur == NULL) { - pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.meta); + pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.vnode); } if (pInfo->pSchema == NULL) { @@ -569,7 +569,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { schemaRow = *(SSchemaWrapper**)schema; } else { SMetaReader smrSuperTable = {0}; - pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, 0); int code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly @@ -658,7 +658,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { STR_TO_VARSTR(tableName, condTableName); SMetaReader smrChildTable = {0}; - pAPI->metaReaderFn.initReader(&smrChildTable, pInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&smrChildTable, pInfo->readHandle.vnode, 0); int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrChildTable, condTableName); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly @@ -676,7 +676,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { } SMetaReader smrSuperTable = {0}; - pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, META_READER_NOLOCK); code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByUid @@ -702,7 +702,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { int32_t ret = 0; if (pInfo->pCur == NULL) { - pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.meta); + pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.vnode); } bool blockFull = false; @@ -715,7 +715,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); SMetaReader smrSuperTable = {0}; - pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, 0); uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid); if (code != TSDB_CODE_SUCCESS) { @@ -1131,7 +1131,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { tb_uid_t* uid = taosArrayGet(pIdx->uids, i); SMetaReader mr = {0}; - pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, 0); ret = pAPI->metaReaderFn.getTableEntryByUid(&mr, *uid); if (ret < 0) { pAPI->metaReaderFn.clearReader(&mr); @@ -1159,7 +1159,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false); SMetaReader mr1 = {0}; - pAPI->metaReaderFn.initReader(&mr1, pInfo->readHandle.meta, META_READER_NOLOCK); + pAPI->metaReaderFn.initReader(&mr1, pInfo->readHandle.vnode, META_READER_NOLOCK); int64_t suid = mr.me.ctbEntry.suid; int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr1, suid); @@ -1292,7 +1292,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { SSysTableScanInfo* pInfo = pOperator->info; if (pInfo->pCur == NULL) { - pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.meta); + pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.vnode); } blockDataCleanup(pInfo->pRes); @@ -1338,7 +1338,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false); SMetaReader mr = {0}; - pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.meta, META_READER_NOLOCK); + pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, META_READER_NOLOCK); uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr, suid); @@ -1486,7 +1486,7 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { } else { if (pInfo->showRewrite == false) { if (pCondition != NULL && pInfo->pIdx == NULL) { - SSTabFltArg arg = {.pMeta = pInfo->readHandle.meta, .pVnode = pInfo->readHandle.vnode}; + SSTabFltArg arg = {.pMeta = pInfo->readHandle.vnode, .pVnode = pInfo->readHandle.vnode}; SSysTableIndex* idx = taosMemoryMalloc(sizeof(SSysTableIndex)); idx->init = 0; diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 1a3a740b34..231e597724 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -440,11 +440,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int int64_t rId = msg.refId; int32_t eId = msg.execId; - SQWMsg qwMsg = {.node = node, - .msg = msg.msg, - .msgLen = msg.msgLen, - .connInfo = pMsg->info, - .msgType = pMsg->msgType}; + SQWMsg qwMsg = {.node = node, .msg = msg.msg, .msgLen = msg.msgLen, .connInfo = pMsg->info, .msgType = pMsg->msgType}; qwMsg.msgInfo.explain = msg.explain; qwMsg.msgInfo.taskType = msg.taskType; qwMsg.msgInfo.needFetch = msg.needFetch; From 39a1fa8f782c8be0c4c00d6fc5f0ef4353f046f8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 May 2023 19:20:25 +0800 Subject: [PATCH 22/97] refactor: do some internal refactor. --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/meta/metaSnapshot.c | 6 +++--- source/dnode/vnode/src/sma/smaRollup.c | 14 ++------------ source/dnode/vnode/src/tq/tq.c | 10 ++++------ source/dnode/vnode/src/tq/tqMeta.c | 7 +++---- source/dnode/vnode/src/vnd/vnodeSvr.c | 4 ++-- 6 files changed, 15 insertions(+), 28 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index ddaf0d9bbc..f6d731f2b4 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -291,7 +291,7 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *pSnapshot); int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData); -int32_t buildSnapContext(SMeta *pMeta, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta, +int32_t buildSnapContext(SVnode *pVnode, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta, SSnapContext **ctxRet); int32_t getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx); diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index d3b6471282..ff4355fdf8 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -260,12 +260,12 @@ static void saveSuperTableInfoForChildTable(SMetaEntry* me, SHashObj* suidInfo) taosHashPut(suidInfo, &me->uid, sizeof(tb_uid_t), &dataTmp, sizeof(STableInfoForChildTable)); } -int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta, +int32_t buildSnapContext(SVnode* pVnode, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta, SSnapContext** ctxRet) { SSnapContext* ctx = taosMemoryCalloc(1, sizeof(SSnapContext)); if (ctx == NULL) return -1; *ctxRet = ctx; - ctx->pMeta = pMeta; + ctx->pMeta = pVnode->pMeta; ctx->snapVersion = snapVersion; ctx->suid = suid; ctx->subType = subType; @@ -301,7 +301,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t continue; } - if (tdbTbGet(pMeta->pUidIdx, &tmp->uid, sizeof(tb_uid_t), NULL, NULL) < + if (tdbTbGet(ctx->pMeta->pUidIdx, &tmp->uid, sizeof(tb_uid_t), NULL, NULL) < 0) { // check if table exist for now, need optimize later continue; } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index ccc00ce25e..f66f85cc88 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -276,12 +276,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat return TSDB_CODE_FAILED; } - SReadHandle handle = { - .meta = pVnode->pMeta, - .vnode = pVnode, - .initTqReader = 1, - .pStateBackend = pStreamState, - }; + SReadHandle handle = { .vnode = pVnode, .initTqReader = 1, .pStateBackend = pStreamState }; pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle, TD_VID(pVnode)); if (!pRSmaInfo->taskInfo[idx]) { terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE; @@ -853,12 +848,7 @@ static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t code = qSerializeTaskStatus(srcTaskInfo, &pOutput, &len); TSDB_CHECK_CODE(code, lino, _exit); - SReadHandle handle = { - .meta = pVnode->pMeta, - .vnode = pVnode, - .initTqReader = 1, - }; - + SReadHandle handle = { .vnode = pVnode, .initTqReader = 1 }; if (ASSERTS(!dstTaskInfo, "dstTaskInfo:%p is not NULL", dstTaskInfo)) { code = TSDB_CODE_APP_ERROR; TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 74361ec319..6befccdab2 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -670,8 +670,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg int64_t ver = pRef->refVer; pHandle->pRef = pRef; - SReadHandle handle = { - .meta = pVnode->pMeta, .vnode = pVnode, .initTableReader = true, .initTqReader = true, .version = ver}; + SReadHandle handle = {.vnode = pVnode, .initTableReader = true, .initTqReader = true, .version = ver}; pHandle->snapshotVer = ver; if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { @@ -689,7 +688,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg pHandle->execHandle.execDb.pFilterOutTbUid = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); - buildSnapContext(handle.meta, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta, + buildSnapContext(handle.vnode, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta, (SSnapContext**)(&handle.sContext)); pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId); @@ -708,7 +707,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); - buildSnapContext(handle.meta, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta, + buildSnapContext(handle.vnode, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta, (SSnapContext**)(&handle.sContext)); pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId); } @@ -787,8 +786,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { return -1; } - SReadHandle handle = { - .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, .initTqReader = 1, .pStateBackend = pTask->pState}; + SReadHandle handle = {.vnode = pTq->pVnode, .initTqReader = 1, .pStateBackend = pTask->pState}; pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId); if (pTask->exec.pExecutor == NULL) { diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 5654147b6d..d5cfa86e4c 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -298,11 +298,10 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { walSetRefVer(handle.pRef, handle.snapshotVer); SReadHandle reader = { - .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, .initTableReader = true, .initTqReader = true, - .version = handle.snapshotVer, + .version = handle.snapshotVer }; if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { @@ -330,7 +329,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); handle.execHandle.pTqReader = tqReaderOpen(pTq->pVnode); - buildSnapContext(reader.meta, reader.version, 0, handle.execHandle.subType, handle.fetchMeta, + buildSnapContext(reader.vnode, reader.version, 0, handle.execHandle.subType, handle.fetchMeta, (SSnapContext**)(&reader.sContext)); handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0); } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__TABLE) { @@ -347,7 +346,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { tqReaderSetTbUidList(handle.execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); - buildSnapContext(reader.meta, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType, + buildSnapContext(reader.vnode, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType, handle.fetchMeta, (SSnapContext**)(&reader.sContext)); handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0); } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 0f08b63294..268d51cd16 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -250,7 +250,7 @@ static int32_t vnodePreProcessDeleteMsg(SVnode *pVnode, SRpcMsg *pMsg) { uint8_t *pCont; SEncoder *pCoder = &(SEncoder){0}; SDeleteRes res = {0}; - SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; + SReadHandle handle = {.config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; code = qWorkerProcessDeleteMsg(&handle, pVnode->pQuery, pMsg, &res); if (code) goto _exit; @@ -1203,7 +1203,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in void *pAllocMsg = NULL; SSubmitReq2Msg *pMsg = (SSubmitReq2Msg *)pReq; - if (0 == pMsg->ver) { + if (0 == pMsg->version) { code = vnodeSubmitReqConvertToSubmitReq2(pVnode, (SSubmitReq *)pMsg, pSubmitReq); if (TSDB_CODE_SUCCESS == code) { code = vnodeRebuildSubmitReqMsg(pSubmitReq, &pReq); From 2e4c14c18ce9a87f38eabe8c249fa26d04f70dbe Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 23 May 2023 21:30:41 +0800 Subject: [PATCH 23/97] fix: fix bugs --- source/libs/scalar/src/filter.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index e7f88908e5..264e41b006 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3514,10 +3514,7 @@ int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { if (val2->kind == FLT_SCL_DATUM_KIND_NULL || val2->kind == FLT_SCL_DATUM_KIND_MIN || val2->kind == FLT_SCL_DATUM_KIND_MAX) { - if (val1->kind == FLT_SCL_DATUM_KIND_NULL || val1->kind == FLT_SCL_DATUM_KIND_MIN || - val1->kind == FLT_SCL_DATUM_KIND_MAX) { return (val1->kind < val2->kind) ? -1 : ((val1->kind > val2->kind) ? 1 : 0); - } } switch (val2->kind) { @@ -3690,7 +3687,12 @@ int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum *datum, uint8_t type, int64_t val) { switch (type) { - case TSDB_DATA_TYPE_BOOL: { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: { datum->kind = FLT_SCL_DATUM_KIND_INT64; datum->i = val; break; @@ -3711,7 +3713,8 @@ int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum *datum, uint8_t type, int } // TODO:varchar/nchar/json default: { - qError("not supported"); + datum->kind = FLT_SCL_DATUM_KIND_NULL; + qError("not supported type when build datum from block sma value"); break; } } From 26bfa4363d23d2c545d07fbc00a339976723cf4b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 24 May 2023 11:01:47 +0800 Subject: [PATCH 24/97] fix(cache/read): associate tbname with primary ts column --- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index b93294e46f..0e0b7a2ffa 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -343,6 +343,14 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 hasRes = true; p->ts = pColVal->ts; + if (k == 0) { + if (TARRAY_SIZE(pTableUidList) == 0) { + taosArrayPush(pTableUidList, &pKeyInfo->uid); + } else { + taosArraySet(pTableUidList, 0, &pKeyInfo->uid); + } + } + if (pColVal->ts < singleTableLastTs && HASTYPE(pr->type, CACHESCAN_RETRIEVE_LAST)) { singleTableLastTs = pColVal->ts; } @@ -373,12 +381,6 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } } - if (TARRAY_SIZE(pTableUidList) == 0) { - taosArrayPush(pTableUidList, &pKeyInfo->uid); - } else { - taosArraySet(pTableUidList, 0, &pKeyInfo->uid); - } - // taosArrayClearEx(pRow, freeItem); taosArrayClear(pRow); } From 383c828c856df00a752b9843a615c468993f9ef3 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 24 May 2023 11:18:44 +0800 Subject: [PATCH 25/97] fix: before review --- source/libs/scalar/src/filter.c | 90 +++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 32 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 264e41b006..ecf621a726 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3472,7 +3472,7 @@ int32_t fltSclCompareWithFloat64(SFltSclDatum *val1, SFltSclDatum *val2) { } // TODO: varchar, nchar default: - qError("not support comparsion. %d, %d", val1->kind, val2->kind); + qError("not supported comparsion. kind1 %d, kind2 %d", val1->kind, val2->kind); return (val1->kind - val2->kind); } } @@ -3489,7 +3489,7 @@ int32_t fltSclCompareWithInt64(SFltSclDatum *val1, SFltSclDatum *val2) { } // TODO: varchar, nchar default: - qError("not support comparsion. %d, %d", val1->kind, val2->kind); + qError("not supported comparsion. kind1 %d, kind2 %d", val1->kind, val2->kind); return (val1->kind - val2->kind); } } @@ -3506,7 +3506,7 @@ int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { } // TODO: varchar, nchar default: - qError("not support comparsion. %d, %d", val1->kind, val2->kind); + qError("not supported comparsion. kind1 %d, kind2 %d", val1->kind, val2->kind); return (val1->kind - val2->kind); } } @@ -3514,7 +3514,7 @@ int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { if (val2->kind == FLT_SCL_DATUM_KIND_NULL || val2->kind == FLT_SCL_DATUM_KIND_MIN || val2->kind == FLT_SCL_DATUM_KIND_MAX) { - return (val1->kind < val2->kind) ? -1 : ((val1->kind > val2->kind) ? 1 : 0); + return (val1->kind < val2->kind) ? -1 : ((val1->kind > val2->kind) ? 1 : 0); } switch (val2->kind) { @@ -3529,7 +3529,7 @@ int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { } // TODO: varchar/nchar default: - fltError("not supported kind. just return 0"); + qError("not supported kind when compare datum. kind2 : %d", val2->kind); return 0; break; } @@ -3585,7 +3585,6 @@ int32_t fltSclMergeSort(SArray *pts1, SArray *pts2, SArray *result) { return 0; } -// pts1 and pts2 must be ordered and de-duplicated and each range can not be a range of another range int32_t fltSclMerge(SArray *pts1, SArray *pts2, bool isUnion, SArray *merged) { size_t len1 = taosArrayGetSize(pts1); size_t len2 = taosArrayGetSize(pts2); @@ -3616,7 +3615,6 @@ int32_t fltSclIntersect(SArray *pts1, SArray *pts2, SArray *merged) { return flt int32_t fltSclUnion(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, true, merged); } -// TODO: column, constant typedef struct { SColumnNode *colNode; SValueNode *valNode; @@ -3675,9 +3673,9 @@ int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) datum->d = valNode->datum.d; break; } - // TODO:varchar/nchar/json + // TODO:varchar/nchar/json default: { - qError("not supported"); + qError("not supported type %d when build datum from value node", valNode->node.resType.type); break; } } @@ -3687,11 +3685,11 @@ int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum *datum, uint8_t type, int64_t val) { switch (type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { datum->kind = FLT_SCL_DATUM_KIND_INT64; datum->i = val; @@ -3711,10 +3709,10 @@ int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum *datum, uint8_t type, int datum->d = *(double *)&val; break; } - // TODO:varchar/nchar/json + // TODO:varchar/nchar/json default: { datum->kind = FLT_SCL_DATUM_KIND_NULL; - qError("not supported type when build datum from block sma value"); + qError("not supported type %d when build datum from block sma value", type); break; } } @@ -3732,6 +3730,13 @@ int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange *colRange, SColumnDataAg taosArrayPush(points, &endPt); return TSDB_CODE_SUCCESS; } + if (pAgg->numOfNull > 0) { + SFltSclDatum nullDatum = {.kind = FLT_SCL_DATUM_KIND_NULL}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = nullDatum}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = nullDatum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + } SFltSclDatum min; fltSclBuildDatumFromBlockSmaValue(&min, colRange->colNode->node.resType.type, pAgg->min); SFltSclPoint minPt = {.excl = false, .start = true, .val = min}; @@ -4357,8 +4362,25 @@ int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { } break; } + case OP_TYPE_IS_NULL: { + SFltSclDatum nullDatum = {.kind = FLT_SCL_DATUM_KIND_NULL}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = nullDatum}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = nullDatum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + case OP_TYPE_IS_NOT_NULL: { + SFltSclDatum minDatum = {.kind = FLT_SCL_DATUM_KIND_MIN, .type = oper->colNode->node.resType}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = minDatum}; + SFltSclDatum maxDatum = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = maxDatum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } default: { - qError("not supported op"); + qError("not supported operator type : %d when build range points", oper->type); break; } } @@ -4391,26 +4413,30 @@ static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) { if (nodeType(pNode) != QUERY_NODE_OPERATOR) { return TSDB_CODE_SUCCESS; } + SOperatorNode *pOper = (SOperatorNode *)pNode; - // TODO: left value node, right column node - // TODO: datatype - // TODO: operator if (pOper->pLeft == NULL || pOper->pRight == NULL) { return TSDB_CODE_SUCCESS; } - if (nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_VALUE && - (pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL || - pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL || - pOper->opType == OP_TYPE_NOT_EQUAL || pOper->opType == OP_TYPE_EQUAL)) { - SValueNode *valNode = (SValueNode *)pOper->pRight; - if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { - SFltSclOperator sclOp = { - .colNode = (SColumnNode*)nodesCloneNode(pOper->pLeft), - .valNode = (SValueNode*)nodesCloneNode(pOper->pRight), - .type = pOper->opType}; - taosArrayPush(sclOpList, &sclOp); - } + + if (!(pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL || + pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL || + pOper->opType == OP_TYPE_NOT_EQUAL || pOper->opType == OP_TYPE_EQUAL)) { + return TSDB_CODE_SUCCESS; } + + if (!(nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_VALUE)) { + return TSDB_CODE_SUCCESS; + } + + SValueNode *valNode = (SValueNode *)pOper->pRight; + if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { + SFltSclOperator sclOp = {.colNode = (SColumnNode *)nodesCloneNode(pOper->pLeft), + .valNode = (SValueNode *)nodesCloneNode(pOper->pRight), + .type = pOper->opType}; + taosArrayPush(sclOpList, &sclOp); + } + return TSDB_CODE_SUCCESS; } @@ -4446,8 +4472,8 @@ int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) for (int32_t i = 0; i < taosArrayGetSize(sclOpList); ++i) { SFltSclOperator *sclOp = taosArrayGet(sclOpList, i); - nodesDestroyNode((SNode*)sclOp->colNode); - nodesDestroyNode((SNode*)sclOp->valNode); + nodesDestroyNode((SNode *)sclOp->colNode); + nodesDestroyNode((SNode *)sclOp->valNode); } taosArrayDestroy(sclOpList); return TSDB_CODE_SUCCESS; From 1f7f326bedeee288a89c23ecade532d2ce3736d7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 May 2023 13:22:05 +0800 Subject: [PATCH 26/97] refactor: do some internal refactor and set the api function ptr. --- include/libs/executor/storageapi.h | 134 ++++++-------- include/libs/stream/streamState.h | 2 +- source/dnode/vnode/inc/vnode.h | 30 +-- source/dnode/vnode/src/inc/vnodeInt.h | 4 + source/dnode/vnode/src/meta/metaQuery.c | 144 ++++---------- source/dnode/vnode/src/meta/metaSma.c | 2 +- source/dnode/vnode/src/meta/metaTable.c | 4 +- source/dnode/vnode/src/sma/smaRollup.c | 4 +- source/dnode/vnode/src/tq/tq.c | 5 +- source/dnode/vnode/src/tq/tqRead.c | 8 +- source/dnode/vnode/src/tq/tqScan.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 +- source/dnode/vnode/src/vnd/vnodeOpen.c | 175 ++++++++++++++++++ source/dnode/vnode/src/vnd/vnodeQuery.c | 42 +++-- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 + source/libs/executor/inc/querytask.h | 2 +- source/libs/executor/src/executil.c | 9 +- source/libs/executor/src/executor.c | 7 +- source/libs/executor/src/querytask.c | 10 +- source/libs/executor/src/scanoperator.c | 36 ++-- source/libs/executor/src/sysscanoperator.c | 33 ++-- source/libs/executor/src/timewindowoperator.c | 4 +- source/libs/stream/src/streamState.c | 29 +-- 24 files changed, 413 insertions(+), 281 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index c0fd6c34fc..8790c302ce 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -136,8 +136,8 @@ typedef struct SRowBuffPos { // void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); // void metaReaderReleaseLock(SMetaReader *pReader); // void metaReaderClear(SMetaReader *pReader); -// int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); -// int32_t metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); +// int32_t metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); +// int32_t metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); // int metaGetTableEntryByName(SMetaReader *pReader, const char *name); // int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList); // int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList); @@ -191,17 +191,13 @@ typedef struct { // int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); // bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid); // bool tqCurrentBlockConsumed(const STqReader* pReader); -// int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); +// int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id); // bool tqNextBlockInWal(STqReader* pReader, const char* idstr); // bool tqNextBlockImpl(STqReader *pReader, const char* idstr); // int32_t getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); // SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx); // int32_t setForSnapShot(SSnapContext *ctx, int64_t uid); // int32_t destroySnapContext(SSnapContext *ctx); -// SMTbCursor *metaOpenTbCursor(SMeta *pMeta); -// void metaCloseTbCursor(SMTbCursor *pTbCur); -// int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); -// int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); /*-------------------------------------------------new api format---------------------------------------------------*/ @@ -229,17 +225,17 @@ typedef struct TsdReader { __store_reader_open_fn_t tsdReaderOpen; void (*tsdReaderClose)(); void (*tsdSetReaderTaskId)(void *pReader, const char *pId); - void (*tsdSetQueryTableList)(); - int32_t (*tsdReaderNextDataBlock)(); + int32_t (*tsdSetQueryTableList)(); + int32_t (*tsdNextDataBlock)(); int32_t (*tsdReaderRetrieveBlockSMAInfo)(); SSDataBlock *(*tsdReaderRetrieveDataBlock)(); void (*tsdReaderReleaseDataBlock)(); - void (*tsdReaderResetStatus)(); - void (*tsdReaderGetDataBlockDistInfo)(); - void (*tsdReaderGetNumOfInMemRows)(); + int32_t (*tsdReaderResetStatus)(); + int32_t (*tsdReaderGetDataBlockDistInfo)(); + int64_t (*tsdReaderGetNumOfInMemRows)(); void (*tsdReaderNotifyClosing)(); } TsdReader; @@ -270,13 +266,13 @@ int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid); bool tqCurrentBlockConsumed(const STqReader* pReader); -int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); +int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id); bool tqNextBlockInWal(STqReader* pReader, const char* idstr); bool tqNextBlockImpl(STqReader *pReader, const char* idstr); int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char* idstr); STqReader *tqReaderOpen(void *pVnode); -void tqCloseReader(STqReader *); +void tqReaderClose(STqReader *); int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); @@ -285,7 +281,7 @@ int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas */ // todo rename typedef struct SStoreTqReader { - void *(*tqReaderOpen)(); + struct STqReader* (*tqReaderOpen)(); void (*tqReaderClose)(); int32_t (*tqReaderSeek)(); @@ -294,7 +290,7 @@ typedef struct SStoreTqReader { bool (*tqNextBlockImpl)(); // todo remove it void (*tqReaderSetColIdList)(); - int32_t (*tqReaderSetTargetTableList)(); + int32_t (*tqReaderSetQueryTableList)(); int32_t (*tqReaderAddTables)(); int32_t (*tqReaderRemoveTables)(); @@ -303,10 +299,10 @@ typedef struct SStoreTqReader { bool (*tqReaderCurrentBlockConsumed)(); struct SWalReader *(*tqReaderGetWalReader)(); // todo remove it - void (*tqReaderRetrieveTaosXBlock)(); // todo remove it + int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it int32_t (*tqReaderSetSubmitMsg)(); // todo remove it - void (*tqReaderNextBlockFilterOut)(); + bool (*tqReaderNextBlockFilterOut)(); } SStoreTqReader; typedef struct SStoreSnapshotFn { @@ -326,8 +322,8 @@ typedef struct SStoreSnapshotFn { void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); void metaReaderReleaseLock(SMetaReader *pReader); void metaReaderClear(SMetaReader *pReader); -int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); -int32_t metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); +int32_t metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); +int32_t metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); int metaGetTableEntryByName(SMetaReader *pReader, const char *name); int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList); const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); @@ -347,39 +343,29 @@ int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen); */ - - typedef struct SStoreMetaReader { - void (*initReader)(void *pReader, void *pMeta, int32_t flags); - void *(*clearReader)(); - - void (*readerReleaseLock)(); - - int32_t (*getTableEntryByUid)(); - int32_t (*getTableEntryByName)(); - int32_t (*readerGetEntryGetUidCache)(SMetaReader *pReader, tb_uid_t uid); + void (*initReader)(SMetaReader *pReader, void *pMeta, int32_t flags); + void (*clearReader)(SMetaReader *pReader); + void (*readerReleaseLock)(SMetaReader *pReader); + int32_t (*getTableEntryByUid)(SMetaReader *pReader, tb_uid_t uid); + int32_t (*getTableEntryByName)(SMetaReader *pReader, const char *name); + int32_t (*getEntryGetUidCache)(SMetaReader *pReader, tb_uid_t uid); } SStoreMetaReader; typedef struct SStoreMeta { - /* -SMTbCursor *metaOpenTbCursor(SMeta *pMeta); -void metaCloseTbCursor(SMTbCursor *pTbCur); -int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); -int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); - */ - void *(*openMetaCursor)(); - void (*closeMetaCursor)(); - int32_t (*cursorNext)(); - void (*cursorPrev)(); + SMTbCursor *(*openTableMetaCursor)(); // metaOpenTbCursor + void (*closeTableMetaCursor)(); // metaCloseTbCursor + int32_t (*cursorNext)(); // metaTbCursorNext + int32_t (*cursorPrev)(); // metaTbCursorPrev int32_t (*getTableTags)(void *pVnode, uint64_t suid, SArray *uidList); int32_t (*getTableTagsByUid)(); - const char *(*extractTagVal)(const void *tag, int16_t type, STagVal *tagVal); // todo remove it + const void *(*extractTagVal)(const void *tag, int16_t type, STagVal *tagVal); // todo remove it int32_t (*getTableUidByName)(void *pVnode, char *tbName, uint64_t *uid); int32_t (*getTableTypeByName)(void *pVnode, char *tbName, ETableType *tbType); int32_t (*getTableNameByUid)(void *pVnode, uint64_t uid, char *tbName); - bool (*isTableExisted)(void *pVnode, uint64_t uid); + bool (*isTableExisted)(void *pVnode, tb_uid_t uid); /** * int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, @@ -397,8 +383,8 @@ int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, in * */ void *(*storeGetIndexInfo)(); - void *(*storeGetInvertIndex)(); - void (*storeGetChildTableList)(); // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] + void *(*getInvertIndex)(void* pVnode); + int32_t (*getChildTableList)(void *pVnode, int64_t suid, SArray *list); // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] int32_t (*storeGetTableList)(); // vnodeGetStbIdList & vnodeGetAllTableList void *storeGetVersionRange; void *storeGetLastTimestamp; @@ -406,8 +392,8 @@ int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, in int32_t (*getTableSchema)(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); // tsdbGetTableSchema // db name, vgId, numOfTables, numOfSTables - void (*storeGetNumOfChildTables)(); // int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); - void (*storeGetBasicInfo)(); // vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) & metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta); + int32_t (*getNumOfChildTables)(void* pVnode, int64_t uid, int64_t* numOfTables); // int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); + void (*getBasicInfo)(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables);// vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) & metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta); int64_t (*getNumOfRowsInMem)(); /** @@ -497,50 +483,50 @@ typedef struct SStateStore { int32_t (*streamStateFillGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t (*streamStateFillDel)(SStreamState* pState, const SWinKey* key); - int32_t (*streamStateCurNext)(SStreamState* pState, void* pCur); - int32_t (*streamStateCurPrev)(SStreamState* pState, void* pCur); + int32_t (*streamStateCurNext)(SStreamState* pState, SStreamStateCur* pCur); + int32_t (*streamStateCurPrev)(SStreamState* pState, SStreamStateCur* pCur); - void* (*streamStateGetAndCheckCur)(SStreamState* pState, SWinKey* key); - void* (*streamStateSeekKeyNext)(SStreamState* pState, const SWinKey* key); - void* (*streamStateFillSeekKeyNext)(SStreamState* pState, const SWinKey* key); - void* (*streamStateFillSeekKeyPrev)(SStreamState* pState, const SWinKey* key); - void (*streamStateFreeCur)(void* pCur); + SStreamStateCur* (*streamStateGetAndCheckCur)(SStreamState* pState, SWinKey* key); + SStreamStateCur* (*streamStateSeekKeyNext)(SStreamState* pState, const SWinKey* key); + SStreamStateCur* (*streamStateFillSeekKeyNext)(SStreamState* pState, const SWinKey* key); + SStreamStateCur* (*streamStateFillSeekKeyPrev)(SStreamState* pState, const SWinKey* key); + void (*streamStateFreeCur)(SStreamStateCur* pCur); - int32_t (*streamStateGetGroupKVByCur)(void* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); - int32_t (*streamStateGetKVByCur)(void* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); + int32_t (*streamStateGetGroupKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); + int32_t (*streamStateGetKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen); int32_t (*streamStateSessionPut)(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen); int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); int32_t (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key); int32_t (*streamStateSessionClear)(SStreamState* pState); - int32_t (*streamStateSessionGetKVByCur)(void* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); + int32_t (*streamStateSessionGetKVByCur)(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t (*streamStateStateAddIfNotExist)(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); - int32_t (*streamStateSessionGetKeyByRange)(void* pState, const SSessionKey* range, SSessionKey* curKey); + int32_t (*streamStateSessionGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); - void* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark); - TSKEY (*updateInfoFillBlockData)(void *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol); - bool (*updateInfoIsUpdated)(void *pInfo, uint64_t tableId, TSKEY ts); - bool (*updateInfoIsTableInserted)(void *pInfo, int64_t tbUid); - void (*updateInfoDestroy)(void *pInfo); + SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark); + TSKEY (*updateInfoFillBlockData)(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol); + bool (*updateInfoIsUpdated)(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts); + bool (*updateInfoIsTableInserted)(SUpdateInfo *pInfo, int64_t tbUid); + void (*updateInfoDestroy)(SUpdateInfo *pInfo); SUpdateInfo* (*updateInfoInitP)(SInterval *pInterval, int64_t watermark); - void (*updateInfoAddCloseWindowSBF)(void *pInfo); - void (*updateInfoDestoryColseWinSBF)(void *pInfo); - int32_t (*updateInfoSerialize)(void *buf, int32_t bufLen, const void *pInfo); - int32_t (*updateInfoDeserialize)(void *buf, int32_t bufLen, void *pInfo); + void (*updateInfoAddCloseWindowSBF)(SUpdateInfo *pInfo); + void (*updateInfoDestoryColseWinSBF)(SUpdateInfo *pInfo); + int32_t (*updateInfoSerialize)(void *buf, int32_t bufLen, const SUpdateInfo *pInfo); + int32_t (*updateInfoDeserialize)(void *buf, int32_t bufLen, SUpdateInfo *pInfo); - void* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key); - void* (*streamStateSessionSeekKeyCurrentPrev)(SStreamState* pState, const SSessionKey* key); - void* (*streamStateSessionSeekKeyCurrentNext)(SStreamState* pState, const SSessionKey* key); + SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key); + SStreamStateCur* (*streamStateSessionSeekKeyCurrentPrev)(SStreamState* pState, const SSessionKey* key); + SStreamStateCur* (*streamStateSessionSeekKeyCurrentNext)(SStreamState* pState, const SSessionKey* key); - void* (*streamFileStateInit)(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, GetTsFun fp, - void* pFile, TSKEY delMark); + struct SStreamFileState* (*streamFileStateInit)(int64_t memSize, uint32_t keySize, uint32_t rowSize, + uint32_t selectRowSize, GetTsFun fp, void* pFile, TSKEY delMark); - void (*streamFileStateDestroy)(void* pFileState); - void (*streamFileStateClear)(void* pFileState); - bool (*needClearDiskBuff)(void* pFileState); + void (*streamFileStateDestroy)(struct SStreamFileState* pFileState); + void (*streamFileStateClear)(struct SStreamFileState* pFileState); + bool (*needClearDiskBuff)(struct SStreamFileState* pFileState); SStreamState* (*streamStateOpen)(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages); void (*streamStateClose)(SStreamState* pState, bool remove); diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index f9fffef7c2..08357b8251 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -59,7 +59,7 @@ extern "C" { // TXN* txn; //} STdbState; -SStreamState* streamStateOpen(char* path, struct SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages); +SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages); void streamStateClose(SStreamState* pState, bool remove); int32_t streamStateBegin(SStreamState* pState); int32_t streamStateCommit(SStreamState* pState); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index f6d731f2b4..95fe150443 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -67,17 +67,17 @@ int32_t vnodeStart(SVnode *pVnode); void vnodeStop(SVnode *pVnode); int64_t vnodeGetSyncHandle(SVnode *pVnode); void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot); -void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId); +void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables); int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen); int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list); int32_t vnodeIsCatchUp(SVnode *pVnode); ESyncRole vnodeGetRole(SVnode *pVnode); -int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list); +int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list); int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg); int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list); -void *vnodeGetIdx(SVnode *pVnode); -void *vnodeGetIvtIdx(SVnode *pVnode); +void *vnodeGetIdx(void *pVnode); +void *vnodeGetIvtIdx(void *pVnode); int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num); int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num); @@ -105,22 +105,22 @@ typedef struct SMetaEntry SMetaEntry; #define META_READER_NOLOCK 0x1 -void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); +void _metaReaderInit(SMetaReader *pReader, void *pVnode, int32_t flags); void metaReaderReleaseLock(SMetaReader *pReader); void metaReaderClear(SMetaReader *pReader); -int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); -int32_t metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); +int32_t metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); +int32_t metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); int metaGetTableEntryByName(SMetaReader *pReader, const char *name); int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList); int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList); int32_t metaReadNext(SMetaReader *pReader); -const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); +const void *metaGetTableTagVal(const void *tag, int16_t type, STagVal *tagVal); int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName); int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid); int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); -bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid); +bool metaIsTableExist(void* pVnode, tb_uid_t uid); int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList, bool *acquired); int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, @@ -138,7 +138,7 @@ int64_t metaGetNtbNum(SMeta *pMeta); // int64_t uid; // int64_t ctbNum; //} SMetaStbStats; -int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); +int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables); //typedef struct SMetaFltParam { // tb_uid_t suid; @@ -163,7 +163,7 @@ typedef SVCreateTSmaReq SSmaCfg; typedef struct SMTbCursor SMTbCursor; -SMTbCursor *metaOpenTbCursor(SMeta *pMeta); +SMTbCursor *metaOpenTbCursor(void *pVnode); void metaCloseTbCursor(SMTbCursor *pTbCur); int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); @@ -260,16 +260,20 @@ typedef struct STqReader { } STqReader; STqReader *tqReaderOpen(SVnode *pVnode); -void tqCloseReader(STqReader *); +void tqReaderClose(STqReader *); void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList); int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList); int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); -int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); +bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid); +bool tqCurrentBlockConsumed(const STqReader* pReader); + +int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id); bool tqNextBlockInWal(STqReader *pReader, const char *idstr); bool tqNextBlockImpl(STqReader *pReader, const char *idstr); +SWalReader* tqGetWalReader(STqReader* pReader); int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, const char *id); int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 230e8b8f88..735ff1700c 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -161,6 +161,8 @@ void* metaGetIdx(SMeta* pMeta); void* metaGetIvtIdx(SMeta* pMeta); int metaTtlSmaller(SMeta* pMeta, uint64_t time, SArray* uidList); +void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); + int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg); int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); @@ -469,6 +471,8 @@ struct SCompactInfo { STimeWindow tw; }; +void initStorageAPI(SStorageAPI* pAPI); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 48bfc5d86f..40f4b6fa67 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -17,6 +17,11 @@ #include "osMemory.h" #include "tencode.h" +void _metaReaderInit(SMetaReader* pReader, void* pVnode, int32_t flags) { + SMeta* pMeta = ((SVnode*)pVnode)->pMeta; + metaReaderInit(pReader, pMeta, flags); +} + void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) { memset(pReader, 0, sizeof(*pReader)); pReader->flags = flags; @@ -64,96 +69,20 @@ _err: return -1; } -// int metaGetTableEntryByUidTest(void* meta, SArray *uidList) { -// -// SArray* readerList = taosArrayInit(taosArrayGetSize(uidList), sizeof(SMetaReader)); -// SArray* uidVersion = taosArrayInit(taosArrayGetSize(uidList), sizeof(STbDbKey)); -// SMeta *pMeta = meta; -// int64_t version; -// SHashObj *uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); -// -// int64_t stt1 = taosGetTimestampUs(); -// for(int i = 0; i < taosArrayGetSize(uidList); i++) { -// void* ppVal = NULL; -// int vlen = 0; -// uint64_t * uid = taosArrayGet(uidList, i); -// // query uid.idx -// if (tdbTbGet(pMeta->pUidIdx, uid, sizeof(*uid), &ppVal, &vlen) < 0) { -// continue; -// } -// version = *(int64_t *)ppVal; -// -// STbDbKey tbDbKey = {.version = version, .uid = *uid}; -// taosArrayPush(uidVersion, &tbDbKey); -// taosHashPut(uHash, uid, sizeof(int64_t), ppVal, sizeof(int64_t)); -// } -// int64_t stt2 = taosGetTimestampUs(); -// qDebug("metaGetTableEntryByUidTest1 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt2-stt1); -// -// TBC *pCur = NULL; -// tdbTbcOpen(pMeta->pTbDb, &pCur, NULL); -// tdbTbcMoveToFirst(pCur); -// void *pKey = NULL; -// int kLen = 0; -// -// while(1){ -// SMetaReader pReader = {0}; -// int32_t ret = tdbTbcNext(pCur, &pKey, &kLen, &pReader.pBuf, &pReader.szBuf); -// if (ret < 0) break; -// STbDbKey *tmp = (STbDbKey*)pKey; -// int64_t *ver = (int64_t*)taosHashGet(uHash, &tmp->uid, sizeof(int64_t)); -// if(ver == NULL || *ver != tmp->version) continue; -// taosArrayPush(readerList, &pReader); -// } -// tdbTbcClose(pCur); -// -// taosArrayClear(readerList); -// int64_t stt3 = taosGetTimestampUs(); -// qDebug("metaGetTableEntryByUidTest2 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt3-stt2); -// for(int i = 0; i < taosArrayGetSize(uidVersion); i++) { -// SMetaReader pReader = {0}; -// -// STbDbKey *tbDbKey = taosArrayGet(uidVersion, i); -// // query table.db -// if (tdbTbGet(pMeta->pTbDb, tbDbKey, sizeof(STbDbKey), &pReader.pBuf, &pReader.szBuf) < 0) { -// continue; -// } -// taosArrayPush(readerList, &pReader); -// } -// int64_t stt4 = taosGetTimestampUs(); -// qDebug("metaGetTableEntryByUidTest3 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt4-stt3); -// -// for(int i = 0; i < taosArrayGetSize(readerList); i++){ -// SMetaReader* pReader = taosArrayGet(readerList, i); -// metaReaderInit(pReader, meta, 0); -// // decode the entry -// tDecoderInit(&pReader->coder, pReader->pBuf, pReader->szBuf); -// -// if (metaDecodeEntry(&pReader->coder, &pReader->me) < 0) { -// } -// metaReaderClear(pReader); -// } -// int64_t stt5 = taosGetTimestampUs(); -// qDebug("metaGetTableEntryByUidTest4 rows:%d, cost:%ld us", taosArrayGetSize(readerList), stt5-stt4); -// return 0; -// } - -bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid) { - // query uid.idx - metaRLock(pMeta); - - if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), NULL, NULL) < 0) { - metaULock(pMeta); +bool metaIsTableExist(void *pVnode, tb_uid_t uid) { + SVnode* pVnodeObj = pVnode; + metaRLock(pVnodeObj->pMeta); // query uid.idx + if (tdbTbGet(pVnodeObj->pMeta->pUidIdx, &uid, sizeof(uid), NULL, NULL) < 0) { + metaULock(pVnodeObj->pMeta); return false; } - metaULock(pMeta); - + metaULock(pVnodeObj->pMeta); return true; } -int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { +int metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { SMeta *pMeta = pReader->pMeta; int64_t version1; @@ -167,7 +96,7 @@ int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { return metaGetTableEntryByVersion(pReader, version1, uid); } -int metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid) { +int metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid) { SMeta *pMeta = pReader->pMeta; SMetaInfo info; @@ -190,7 +119,7 @@ int metaGetTableEntryByName(SMetaReader *pReader, const char *name) { } uid = *(tb_uid_t *)pReader->pBuf; - return metaGetTableEntryByUid(pReader, uid); + return metaReaderGetTableEntryByUid(pReader, uid); } tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) { @@ -214,7 +143,7 @@ int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName) { int code = 0; SMetaReader mr = {0}; metaReaderInit(&mr, (SMeta *)meta, 0); - code = metaGetTableEntryByUid(&mr, uid); + code = metaReaderGetTableEntryByUid(&mr, uid); if (code < 0) { metaReaderClear(&mr); return -1; @@ -230,7 +159,7 @@ int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName) { int code = 0; SMetaReader mr = {0}; metaReaderInit(&mr, (SMeta *)meta, 0); - code = metaGetTableEntryByUid(&mr, uid); + code = metaReaderGetTableEntryByUid(&mr, uid); if (code < 0) { metaReaderClear(&mr); return -1; @@ -283,7 +212,7 @@ int metaReadNext(SMetaReader *pReader) { } #if 1 // =================================================== -SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { +SMTbCursor *metaOpenTbCursor(void *pVnode) { SMTbCursor *pTbCur = NULL; pTbCur = (SMTbCursor *)taosMemoryCalloc(1, sizeof(*pTbCur)); @@ -291,12 +220,12 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { return NULL; } - metaReaderInit(&pTbCur->mr, pMeta, 0); + SVnode* pVnodeObj = pVnode; + metaReaderInit(&pTbCur->mr, pVnodeObj->pMeta, 0); - tdbTbcOpen(pMeta->pUidIdx, (TBC **)&pTbCur->pDbc, NULL); + tdbTbcOpen(pVnodeObj->pMeta->pUidIdx, (TBC **)&pTbCur->pDbc, NULL); tdbTbcMoveToFirst((TBC *)pTbCur->pDbc); - return pTbCur; } @@ -876,7 +805,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) { STSma *pTSma = NULL; for (int i = 0; i < pSW->number; ++i) { smaId = *(tb_uid_t *)taosArrayGet(pSmaIds, i); - if (metaGetTableEntryByUid(&mr, smaId) < 0) { + if (metaReaderGetTableEntryByUid(&mr, smaId) < 0) { tDecoderClear(&mr.coder); metaWarn("vgId:%d, no entry for tbId:%" PRIi64 ", smaId:%" PRIi64, TD_VID(pMeta->pVnode), uid, smaId); continue; @@ -926,7 +855,7 @@ STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) { STSma *pTSma = NULL; SMetaReader mr = {0}; metaReaderInit(&mr, pMeta, 0); - if (metaGetTableEntryByUid(&mr, indexUid) < 0) { + if (metaReaderGetTableEntryByUid(&mr, indexUid) < 0) { metaWarn("vgId:%d, failed to get table entry for smaId:%" PRIi64, TD_VID(pMeta->pVnode), indexUid); metaReaderClear(&mr); return NULL; @@ -1027,7 +956,7 @@ SArray *metaGetSmaTbUids(SMeta *pMeta) { #endif -const void *metaGetTableTagVal(void *pTag, int16_t type, STagVal *val) { +const void *metaGetTableTagVal(const void *pTag, int16_t type, STagVal *val) { STag *tag = (STag *)pTag; if (type == TSDB_DATA_TYPE_JSON) { return tag; @@ -1565,30 +1494,35 @@ _exit: return code; } -int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo) { +int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t* numOfTables) { int32_t code = 0; + *numOfTables = 0; - metaRLock(pMeta); + SVnode* pVnodeObj = pVnode; + metaRLock(pVnodeObj->pMeta); // fast path: search cache - if (metaStatsCacheGet(pMeta, uid, pInfo) == TSDB_CODE_SUCCESS) { - metaULock(pMeta); + SMetaStbStats state = {0}; + if (metaStatsCacheGet(pVnodeObj->pMeta, uid, &state) == TSDB_CODE_SUCCESS) { + metaULock(pVnodeObj->pMeta); + *numOfTables = state.ctbNum; goto _exit; } // slow path: search TDB int64_t ctbNum = 0; - vnodeGetCtbNum(pMeta->pVnode, uid, &ctbNum); + vnodeGetCtbNum(pVnode, uid, &ctbNum); - metaULock(pMeta); + metaULock(pVnodeObj->pMeta); + *numOfTables = ctbNum; - pInfo->uid = uid; - pInfo->ctbNum = ctbNum; + state.uid = uid; + state.ctbNum = ctbNum; // upsert the cache - metaWLock(pMeta); - metaStatsCacheUpsert(pMeta, pInfo); - metaULock(pMeta); + metaWLock(pVnodeObj->pMeta); + metaStatsCacheUpsert(pVnodeObj->pMeta, &state); + metaULock(pVnodeObj->pMeta); _exit: return code; diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c index 8d5821e28b..3fb491848a 100644 --- a/source/dnode/vnode/src/meta/metaSma.c +++ b/source/dnode/vnode/src/meta/metaSma.c @@ -36,7 +36,7 @@ int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) { // validate req // save smaIndex metaReaderInit(&mr, pMeta, 0); - if (metaGetTableEntryByUidCache(&mr, pCfg->indexUid) == 0) { + if (metaReaderGetTableEntryByUidCache(&mr, pCfg->indexUid) == 0) { #if 1 terrno = TSDB_CODE_TSMA_ALREADY_EXIST; metaReaderClear(&mr); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 0164a82c69..df16304e28 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -690,7 +690,7 @@ _err: return -1; } -int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMetaRsp **pMetaRsp) { +int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRsp **pMetaRsp) { SMetaEntry me = {0}; SMetaReader mr = {0}; @@ -729,7 +729,7 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMe metaReaderClear(&mr); // build SMetaEntry - me.version = version; + me.version = ver; me.type = pReq->type; me.uid = pReq->uid; me.name = pReq->name; diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index f66f85cc88..558cfdac2b 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -894,7 +894,7 @@ static int32_t tdRSmaInfoClone(SSma *pSma, SRSmaInfo *pInfo) { metaReaderInit(&mr, SMA_META(pSma), 0); smaDebug("vgId:%d, rsma clone qTaskInfo for suid:%" PRIi64, SMA_VID(pSma), pInfo->suid); - if (metaGetTableEntryByUidCache(&mr, pInfo->suid) < 0) { + if (metaReaderGetTableEntryByUidCache(&mr, pInfo->suid) < 0) { code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } @@ -1121,7 +1121,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { for (int64_t i = 0; i < arrSize; ++i) { suid = *(tb_uid_t *)taosArrayGet(suidList, i); smaDebug("vgId:%d, rsma restore, suid is %" PRIi64, TD_VID(pVnode), suid); - if (metaGetTableEntryByUidCache(&mr, suid) < 0) { + if (metaReaderGetTableEntryByUidCache(&mr, suid) < 0) { code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6befccdab2..4adbf02302 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -69,12 +69,12 @@ static void destroyTqHandle(void* data) { if (pData->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { taosMemoryFreeClear(pData->execHandle.execCol.qmsg); } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__DB) { - tqCloseReader(pData->execHandle.pTqReader); + tqReaderClose(pData->execHandle.pTqReader); walCloseReader(pData->pWalReader); taosHashCleanup(pData->execHandle.execDb.pFilterOutTbUid); } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { walCloseReader(pData->pWalReader); - tqCloseReader(pData->execHandle.pTqReader); + tqReaderClose(pData->execHandle.pTqReader); } if(pData->msg != NULL) { rpcFreeCont(pData->msg->pCont); @@ -787,6 +787,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { } SReadHandle handle = {.vnode = pTq->pVnode, .initTqReader = 1, .pStateBackend = pTask->pState}; + initStorageAPI(&handle.api); pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId); if (pTask->exec.pExecutor == NULL) { diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 84ead71fb5..2521d956a3 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -273,7 +273,7 @@ STqReader* tqReaderOpen(SVnode* pVnode) { return pReader; } -void tqCloseReader(STqReader* pReader) { +void tqReaderClose(STqReader* pReader) { // close wal reader if (pReader->pWalReader) { walCloseReader(pReader->pWalReader); @@ -294,7 +294,7 @@ void tqCloseReader(STqReader* pReader) { taosMemoryFree(pReader); } -int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) { +int32_t tqReaderSeek(STqReader* pReader, int64_t ver, const char* id) { if (walReaderSeekVer(pReader->pWalReader, ver) < 0) { return -1; } @@ -605,6 +605,8 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk++); SSDataBlock* pBlock = pReader->pResBlock; + *pRes = pBlock; + blockDataCleanup(pBlock); int32_t sversion = pSubmitTbData->sver; @@ -1084,7 +1086,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { for (int32_t i = 0; i < taosArrayGetSize(tbUidList); ++i) { uint64_t* id = (uint64_t*)taosArrayGet(tbUidList, i); - int32_t code = metaGetTableEntryByUidCache(&mr, *id); + int32_t code = metaReaderGetTableEntryByUidCache(&mr, *id); if (code != TSDB_CODE_SUCCESS) { tqError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno)); continue; diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index e268199e16..98304f9c18 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -51,7 +51,7 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, in metaReaderInit(&mr, pTq->pVnode->pMeta, 0); // TODO add reference to gurantee success - if (metaGetTableEntryByUidCache(&mr, uid) < 0) { + if (metaReaderGetTableEntryByUidCache(&mr, uid) < 0) { metaReaderClear(&mr); return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index c0a8de5743..2d8b7f9ecf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -980,7 +980,7 @@ static tb_uid_t getTableSuidByUid(tb_uid_t uid, STsdb *pTsdb) { SMetaReader mr = {0}; metaReaderInit(&mr, pTsdb->pVnode->pMeta, 0); - if (metaGetTableEntryByUidCache(&mr, uid) < 0) { + if (metaReaderGetTableEntryByUidCache(&mr, uid) < 0) { metaReaderClear(&mr); // table not esist return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index cde2672541..414ccfc8b2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -5376,7 +5376,7 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) { SMetaReader mr = {0}; metaReaderInit(&mr, pVnode->pMeta, 0); - int32_t code = metaGetTableEntryByUidCache(&mr, uid); + int32_t code = metaReaderGetTableEntryByUidCache(&mr, uid); if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; metaReaderClear(&mr); @@ -5389,7 +5389,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 if (mr.me.type == TSDB_CHILD_TABLE) { tDecoderClear(&mr.coder); *suid = mr.me.ctbEntry.suid; - code = metaGetTableEntryByUidCache(&mr, *suid); + code = metaReaderGetTableEntryByUidCache(&mr, *suid); if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; metaReaderClear(&mr); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index b5e7c6875b..ebfecc4c15 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -14,6 +14,7 @@ */ #include "vnd.h" +#include "tstreamUpdate.h" int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { SVnodeInfo info = {0}; @@ -444,3 +445,177 @@ void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) { pSnapshot->lastApplyTerm = pVnode->state.commitTerm; pSnapshot->lastConfigIndex = -1; } + +static void initTsdbReaderAPI(TsdReader* pReader); +static void initMetadataAPI(SStoreMeta* pMeta); +static void initTqAPI(SStoreTqReader* pTq); +static void initStateStoreAPI(SStateStore* pStore); +static void initMetaReaderAPI(SStoreMetaReader* pMetaReader); + +void initStorageAPI(SStorageAPI* pAPI) { + initTsdbReaderAPI(&pAPI->tsdReader); + initMetadataAPI(&pAPI->metaFn); + initTqAPI(&pAPI->tqReaderFn); + initStateStoreAPI(&pAPI->stateStore); + initMetaReaderAPI(&pAPI->metaReaderFn); +} + +void initTsdbReaderAPI(TsdReader* pReader) { + pReader->tsdReaderOpen = (__store_reader_open_fn_t)tsdbReaderOpen; + pReader->tsdReaderClose = tsdbReaderClose; + + pReader->tsdNextDataBlock = tsdbNextDataBlock; + + pReader->tsdReaderRetrieveDataBlock = tsdbRetrieveDataBlock; + pReader->tsdReaderReleaseDataBlock = tsdbReleaseDataBlock; + + pReader->tsdReaderRetrieveBlockSMAInfo = tsdbRetrieveDatablockSMA; + + pReader->tsdReaderNotifyClosing = tsdbReaderSetCloseFlag; + pReader->tsdReaderResetStatus = tsdbReaderReset; + + pReader->tsdReaderGetDataBlockDistInfo = tsdbGetFileBlocksDistInfo; + pReader->tsdReaderGetNumOfInMemRows = tsdbGetNumOfRowsInMemTable; // todo this function should be moved away + + pReader->tsdSetQueryTableList = tsdbSetTableList; + pReader->tsdSetReaderTaskId = (void (*)(void *, const char *))tsdbReaderSetId; +} + +void initMetadataAPI(SStoreMeta* pMeta) { + pMeta->isTableExisted = metaIsTableExist; + + pMeta->openTableMetaCursor = metaOpenTbCursor; + pMeta->closeTableMetaCursor = metaCloseTbCursor; + pMeta->cursorNext = metaTbCursorNext; + pMeta->cursorPrev = metaTbCursorPrev; + + pMeta->getBasicInfo = vnodeGetInfo; + pMeta->getNumOfChildTables = metaGetStbStats; + + pMeta->getChildTableList = vnodeGetCtbIdList; + + pMeta->storeGetIndexInfo = vnodeGetIdx; + pMeta->getInvertIndex = vnodeGetIvtIdx; + + pMeta->extractTagVal = (const void *(*)(const void *, int16_t, STagVal *))metaGetTableTagVal; + +} + +void initTqAPI(SStoreTqReader* pTq) { + pTq->tqReaderOpen = tqReaderOpen; + pTq->tqReaderSetColIdList = tqReaderSetColIdList; + + pTq->tqReaderClose = tqReaderClose; + pTq->tqReaderSeek = tqReaderSeek; + pTq->tqRetrieveBlock = tqRetrieveDataBlock; + + pTq->tqReaderNextBlockInWal = tqNextBlockInWal; + + pTq->tqNextBlockImpl = tqNextBlockImpl;// todo remove it + + pTq->tqReaderAddTables = tqReaderAddTbUidList; + pTq->tqReaderSetQueryTableList = tqReaderSetTbUidList; + + pTq->tqReaderRemoveTables = tqReaderRemoveTbUidList; + + pTq->tqReaderIsQueriedTable = tqReaderIsQueriedTable; + pTq->tqReaderCurrentBlockConsumed = tqCurrentBlockConsumed; + + pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it + pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it + + pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it + pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; +} + +void initStateStoreAPI(SStateStore* pStore) { + pStore->streamFileStateInit = streamFileStateInit; + pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF; + + pStore->streamStateGetByPos = streamStateGetByPos; + + pStore->streamStatePutParName = streamStatePutParName; + pStore->streamStateGetParName = streamStateGetParName; + + pStore->streamStateAddIfNotExist = streamStateAddIfNotExist; + pStore->streamStateReleaseBuf = streamStateReleaseBuf; + pStore->streamStateFreeVal = streamStateFreeVal; + + pStore->streamStatePut = streamStatePut; + pStore->streamStateGet = streamStateGet; + pStore->streamStateCheck = streamStateCheck; + pStore->streamStateGetByPos = streamStateGetByPos; + pStore->streamStateDel = streamStateDel; + pStore->streamStateClear = streamStateClear; + pStore->streamStateSaveInfo = streamStateSaveInfo; + pStore->streamStateGetInfo = streamStateGetInfo; + pStore->streamStateSetNumber = streamStateSetNumber; + + pStore->streamStateFillPut = streamStateFillPut; + pStore->streamStateFillGet = streamStateFillGet; + pStore->streamStateFillDel = streamStateFillDel; + + pStore->streamStateCurNext = streamStateCurNext; + pStore->streamStateCurPrev = streamStateCurPrev; + + pStore->streamStateGetAndCheckCur = streamStateGetAndCheckCur; + pStore->streamStateSeekKeyNext = streamStateSeekKeyNext; + pStore->streamStateFillSeekKeyNext = streamStateFillSeekKeyNext; + pStore->streamStateFillSeekKeyPrev = streamStateFillSeekKeyPrev; + pStore->streamStateFreeCur = streamStateFreeCur; + + pStore->streamStateGetGroupKVByCur = streamStateGetGroupKVByCur; + pStore->streamStateGetKVByCur = streamStateGetKVByCur; + + pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist; + pStore->streamStateSessionPut = streamStateSessionPut; + pStore->streamStateSessionGet = streamStateSessionGet; + pStore->streamStateSessionDel = streamStateSessionDel; + pStore->streamStateSessionClear = streamStateSessionClear; + pStore->streamStateSessionGetKVByCur = streamStateSessionGetKVByCur; + pStore->streamStateStateAddIfNotExist = streamStateStateAddIfNotExist; + pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange; + + pStore->updateInfoInit = updateInfoInit; + pStore->updateInfoFillBlockData = updateInfoFillBlockData; + pStore->updateInfoIsUpdated = updateInfoIsUpdated; + pStore->updateInfoIsTableInserted = updateInfoIsTableInserted; + pStore->updateInfoDestroy = updateInfoDestroy; + + pStore->updateInfoInitP = updateInfoInitP; + pStore->updateInfoAddCloseWindowSBF = updateInfoAddCloseWindowSBF; + pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF; + pStore->updateInfoSerialize = updateInfoSerialize; + pStore->updateInfoDeserialize = updateInfoDeserialize; + + pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext; + pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev; + pStore->streamStateSessionSeekKeyCurrentNext = streamStateSessionSeekKeyCurrentNext; + + pStore->streamFileStateInit = streamFileStateInit; + + pStore->streamFileStateDestroy = streamFileStateDestroy; + pStore->streamFileStateClear = streamFileStateClear; + pStore->needClearDiskBuff = needClearDiskBuff; + + pStore->streamStateOpen = streamStateOpen; + pStore->streamStateClose = streamStateClose; + pStore->streamStateBegin = streamStateBegin; + pStore->streamStateCommit = streamStateCommit; + pStore->streamStateDestroy= streamStateDestroy; + pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint; +} + +void initMetaReaderAPI(SStoreMetaReader* pMetaReader) { + pMetaReader->initReader = _metaReaderInit; + pMetaReader->clearReader = metaReaderClear; + + pMetaReader->getTableEntryByUid = metaReaderGetTableEntryByUid; + pMetaReader->clearReader = metaReaderClear; + + pMetaReader->getEntryGetUidCache = metaReaderGetTableEntryByUidCache; + pMetaReader->getTableEntryByName = metaGetTableEntryByName; + + pMetaReader->readerReleaseLock = metaReaderReleaseLock; +} + diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 303d2a9ca4..ac3e632172 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -80,7 +80,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { metaRsp.suid = mer1.me.uid; } else if (mer1.me.type == TSDB_CHILD_TABLE) { metaReaderInit(&mer2, pVnode->pMeta, META_READER_NOLOCK); - if (metaGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; + if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; strcpy(metaRsp.stbName, mer2.me.name); metaRsp.suid = mer2.me.uid; @@ -189,7 +189,7 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { goto _exit; } else if (mer1.me.type == TSDB_CHILD_TABLE) { metaReaderInit(&mer2, pVnode->pMeta, 0); - if (metaGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; + if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; strcpy(cfgRsp.stbName, mer2.me.name); schema = mer2.me.stbEntry.schemaRow; @@ -410,13 +410,24 @@ void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { "nBatchInsertSuccess"); } -void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId) { +void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables) { + SVnode* pVnodeObj = pVnode; + SVnodeCfg* pConf = &pVnodeObj->config; + if (dbname) { - *dbname = pVnode->config.dbname; + *dbname = pConf->dbname; } if (vgId) { - *vgId = TD_VID(pVnode); + *vgId = TD_VID(pVnodeObj); + } + + if (numOfTables) { + *numOfTables = pConf->vndStats.numOfNTables + pConf->vndStats.numOfCTables; + } + + if (numOfNormalTables) { + *numOfNormalTables = pConf->vndStats.numOfNTables; } } @@ -440,8 +451,10 @@ int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) { int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg) { return 0; } -int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list) { - SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, suid, 1); + +int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list) { + SVnode *pVnodeObj = pVnode; + SMCtbCursor *pCur = metaOpenCtbCursor(pVnodeObj->pMeta, suid, 1); while (1) { tb_uid_t id = metaCtbCursorNext(pCur); @@ -529,10 +542,8 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) { for (int64_t i = 0; i < arrSize; ++i) { tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); - SMetaStbStats stats = {0}; - metaGetStbStats(pVnode->pMeta, suid, &stats); - int64_t ctbNum = stats.ctbNum; - // vnodeGetCtbNum(pVnode, id, &ctbNum); + int64_t ctbNum = 0; + metaGetStbStats(pVnode, suid, &ctbNum); int numOfCols = 0; vnodeGetStbColumnNum(pVnode, suid, &numOfCols); @@ -567,16 +578,17 @@ int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num) { return TSDB_CODE_SUCCESS; } -void *vnodeGetIdx(SVnode *pVnode) { +void *vnodeGetIdx(void *pVnode) { if (pVnode == NULL) { return NULL; } - return metaGetIdx(pVnode->pMeta); + + return metaGetIdx(((SVnode*)pVnode)->pMeta); } -void *vnodeGetIvtIdx(SVnode *pVnode) { +void *vnodeGetIvtIdx(void *pVnode) { if (pVnode == NULL) { return NULL; } - return metaGetIvtIdx(pVnode->pMeta); + return metaGetIvtIdx(((SVnode*)pVnode)->pMeta); } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 268d51cd16..d16102037d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -515,6 +515,8 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { } SReadHandle handle = {.config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; + initStorageAPI(&handle.api); + switch (pMsg->msgType) { case TDMT_SCH_QUERY: case TDMT_SCH_MERGE_QUERY: diff --git a/source/libs/executor/inc/querytask.h b/source/libs/executor/inc/querytask.h index 1c2ed6c076..6497bd90b4 100644 --- a/source/libs/executor/inc/querytask.h +++ b/source/libs/executor/inc/querytask.h @@ -95,7 +95,7 @@ struct SExecTaskInfo { }; void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst); -SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model); +SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, SStorageAPI* pAPI); void doDestroyTask(SExecTaskInfo* pTaskInfo); bool isTaskKilled(SExecTaskInfo* pTaskInfo); void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index b24fcade79..a52e73eb49 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -307,7 +307,7 @@ int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, SMetaReader mr = {0}; pAPI->metaReaderFn.initReader(&mr, metaHandle, 0); - code = pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, info->uid); + code = pAPI->metaReaderFn.getEntryGetUidCache(&mr, info->uid); if (TSDB_CODE_SUCCESS != code) { pAPI->metaReaderFn.clearReader(&mr); *pQualified = false; @@ -1090,12 +1090,11 @@ int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, S } if (!pTagCond) { // no tag filter condition exists, let's fetch all tables of this super table -// vnodeGetCtbIdList(); - pStorageAPI->metaFn.storeGetChildTableList(pVnode, pScanNode->suid, pUidList); + pStorageAPI->metaFn.getChildTableList(pVnode, pScanNode->suid, pUidList); } else { // failed to find the result in the cache, let try to calculate the results if (pTagIndexCond) { - void* pIndex = pStorageAPI->metaFn.storeGetInvertIndex(pVnode); + void* pIndex = pStorageAPI->metaFn.getInvertIndex(pVnode); SIndexMetaArg metaArg = { .metaEx = pVnode, .idx = pStorageAPI->metaFn.storeGetIndexInfo(pVnode), .ivtIdx = pIndex, .suid = pScanNode->uid}; @@ -1170,7 +1169,7 @@ int32_t getGroupIdFromTagsVal(void* pVnode, uint64_t uid, SNodeList* pGroupNode, SMetaReader mr = {0}; pAPI->metaReaderFn.initReader(&mr, pVnode, 0); - if (pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, uid) != 0) { // table not exist + if (pAPI->metaReaderFn.getEntryGetUidCache(&mr, uid) != 0) { // table not exist pAPI->metaReaderFn.clearReader(&mr); return TSDB_CODE_PAR_TABLE_NOT_EXIST; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index cad98348d0..78a015269a 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -252,7 +252,7 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, uint64_t id) { if (msg == NULL) { // create raw scan - SExecTaskInfo* pTaskInfo = doCreateTask(0, id, vgId, OPTR_EXEC_MODEL_QUEUE); + SExecTaskInfo* pTaskInfo = doCreateTask(0, id, vgId, OPTR_EXEC_MODEL_QUEUE, &pReaderHandle->api); if (NULL == pTaskInfo) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -264,6 +264,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 return NULL; } + pTaskInfo->storageAPI = pReaderHandle->api; qDebug("create raw scan task info completed, vgId:%d, %s", vgId, GET_TASKID(pTaskInfo)); return pTaskInfo; } @@ -1092,8 +1093,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT ASSERT(0); // walReaderVerifyOffset(pInfo->tqReader->pWalReader, pOffset); -// if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) { -// qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id); +// if (tqReaderSeek(pInfo->tqReader, pOffset->version + 1, id) < 0) { +// qError("tqReaderSeek failed ver:%" PRId64 ", %s", pOffset->version + 1, id); // return -1; // } } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c index a684a6bf94..f79966a021 100644 --- a/source/libs/executor/src/querytask.c +++ b/source/libs/executor/src/querytask.c @@ -35,7 +35,7 @@ #define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) -SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model) { +SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, SStorageAPI* pAPI) { SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); if (pTaskInfo == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -48,6 +48,7 @@ SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOP pTaskInfo->execModel = model; pTaskInfo->stopInfo.pStopInfo = taosArrayInit(4, sizeof(SExchangeOpStopInfo)); pTaskInfo->pResultBlockList = taosArrayInit(128, POINTER_BYTES); + pTaskInfo->storageAPI = *pAPI; taosInitRWLatch(&pTaskInfo->lock); @@ -55,7 +56,6 @@ SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOP pTaskInfo->id.queryId = queryId; pTaskInfo->id.str = taosMemoryMalloc(64); buildTaskId(taskId, queryId, pTaskInfo->id.str); - return pTaskInfo; } @@ -78,7 +78,7 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, int32_t vgId, char* sql, EOPTR_EXEC_MODEL model) { - *pTaskInfo = doCreateTask(pPlan->id.queryId, taskId, vgId, model); + *pTaskInfo = doCreateTask(pPlan->id.queryId, taskId, vgId, model, &pHandle->api); if (*pTaskInfo == NULL) { taosMemoryFree(sql); return terrno; @@ -123,7 +123,7 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo SStorageAPI* pAPI = &pTaskInfo->storageAPI; pAPI->metaReaderFn.initReader(&mr, pHandle->vnode, 0); - int32_t code = pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, pScanNode->uid); + int32_t code = pAPI->metaReaderFn.getEntryGetUidCache(&mr, pScanNode->uid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get the table meta, uid:0x%" PRIx64 ", suid:0x%" PRIx64 ", %s", pScanNode->uid, pScanNode->suid, GET_TASKID(pTaskInfo)); @@ -144,7 +144,7 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo tDecoderClear(&mr.coder); tb_uid_t suid = mr.me.ctbEntry.suid; - code = pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, suid); + code = pAPI->metaReaderFn.getEntryGetUidCache(&mr, suid); if (code != TSDB_CODE_SUCCESS) { pAPI->metaReaderFn.clearReader(&mr); return terrno; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 23e029e780..83e62faf90 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -532,7 +532,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int // 1. check if it is existed in meta cache if (pCache == NULL) { pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0); - code = pHandle->api.metaReaderFn.readerGetEntryGetUidCache(&mr, pBlock->info.id.uid); + code = pHandle->api.metaReaderFn.getEntryGetUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { // when encounter the TSDB_CODE_PAR_TABLE_NOT_EXIST error, we proceed. if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { @@ -561,7 +561,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int h = taosLRUCacheLookup(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid)); if (h == NULL) { pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0); - code = pHandle->api.metaReaderFn.readerGetEntryGetUidCache(&mr, pBlock->info.id.uid); + code = pHandle->api.metaReaderFn.getEntryGetUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", @@ -693,7 +693,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { int64_t st = taosGetTimestampUs(); while (true) { - code = pAPI->tsdReader.tsdReaderNextDataBlock(pTableScanInfo->base.dataReader, &hasNext); + code = pAPI->tsdReader.tsdNextDataBlock(pTableScanInfo->base.dataReader, &hasNext); if (code) { pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->base.dataReader); T_LONG_JMP(pTaskInfo->env, code); @@ -975,6 +975,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->sample.sampleRatio = pTableScanNode->ratio; pInfo->sample.seed = taosGetTimestampSec(); + pInfo->readerAPI = pTaskInfo->storageAPI.tsdReader; initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pResBlock = createDataBlockFromDescNode(pDescNode); // blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity); @@ -1103,7 +1104,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } bool hasNext = false; - code = pAPI->tsdReader.tsdReaderNextDataBlock(pReader, &hasNext); + code = pAPI->tsdReader.tsdNextDataBlock(pReader, &hasNext); if (code != TSDB_CODE_SUCCESS) { terrno = code; T_LONG_JMP(pTaskInfo->env, code); @@ -2162,7 +2163,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { bool hasNext = false; if (pInfo->dataReader) { - code = pAPI->tsdReader.tsdReaderNextDataBlock(pInfo->dataReader, &hasNext); + code = pAPI->tsdReader.tsdNextDataBlock(pInfo->dataReader, &hasNext); if (code) { pAPI->tsdReader.tsdReaderReleaseDataBlock(pInfo->dataReader); T_LONG_JMP(pTaskInfo->env, code); @@ -2416,7 +2417,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys // set the extract column id to streamHandle pAPI->tqReaderFn.tqReaderSetColIdList(pInfo->tqReader, pColIds); SArray* tableIdList = extractTableIdList(((STableScanInfo*)(pInfo->pTableScanOp->info))->base.pTableListInfo); - code = pAPI->tqReaderFn.tqReaderSetTargetTableList(pInfo->tqReader, tableIdList); + code = pAPI->tqReaderFn.tqReaderSetQueryTableList(pInfo->tqReader, tableIdList); if (code != 0) { taosArrayDestroy(tableIdList); goto _error; @@ -2457,6 +2458,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->igExpired = pTableScanNode->igExpired; pInfo->twAggSup.maxTs = INT64_MIN; pInfo->pState = NULL; + pInfo->stateStore = pTaskInfo->storageAPI.stateStore; + pInfo->readerFn = pTaskInfo->storageAPI.tqReaderFn; // for stream if (pTaskInfo->streamInfo.pState) { @@ -2674,7 +2677,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { qTrace("tsdb/read-table-data: %p, enter next reader", reader); while (true) { - code = pAPI->tsdReader.tsdReaderNextDataBlock(reader, &hasNext); + code = pAPI->tsdReader.tsdNextDataBlock(reader, &hasNext); if (code != 0) { pAPI->tsdReader.tsdReaderReleaseDataBlock(reader); pInfo->base.dataReader = NULL; @@ -3354,7 +3357,7 @@ static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCount SStorageAPI* pAPI = &pTaskInfo->storageAPI; // get dbname - pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL); SName sn = {0}; tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); tNameGetDbName(&sn, dbName); @@ -3396,7 +3399,7 @@ static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountSca pRes->info.id.groupId = groupId; int64_t dbTableCount = 0; - pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &dbTableCount); + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, &dbTableCount, NULL); fillTableCountScanDataBlock(pSupp, dbName, "", dbTableCount, pRes); setOperatorCompleted(pOperator); } @@ -3411,20 +3414,21 @@ static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanO if (strlen(pSupp->stbNameFilter) != 0) { tb_uid_t uid = 0; pAPI->metaFn.getTableUidByName(pInfo->readHandle.vnode, pSupp->stbNameFilter, &uid); - SMetaStbStats stats = {0}; - ASSERT(0); -// metaGetStbStats(pInfo->readHandle.vnode, uid, &stats); - int64_t ctbNum = stats.ctbNum; - fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, ctbNum, pRes); + + int64_t numOfChildTables = 0; + pAPI->metaFn.getNumOfChildTables(pInfo->readHandle.vnode, uid, &numOfChildTables); + + fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, numOfChildTables, pRes); } else { int64_t tbNumVnode = 0;//metaGetTbNum(pInfo->readHandle.vnode); fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } } else { - int64_t tbNumVnode = 0;//metaGetTbNum(pInfo->readHandle.vnode); - pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode); + int64_t tbNumVnode = 0; + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, &tbNumVnode, NULL); fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } + setOperatorCompleted(pOperator); } diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index a9450fb183..16f2b64c07 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -157,7 +157,7 @@ int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) { const char* db = NULL; ASSERT(0); -// pAPI->metaFn.storeGetBasicInfo(pVnode, &db, NULL); +// pAPI->metaFn.getBasicInfo(pVnode, &db, NULL); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -185,7 +185,7 @@ int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) { int64_t vgId = 0; ASSERT(0); -// pAPI->metaFn.storeGetBasicInfo(pVnode, NULL, (int32_t*)&vgId); +// pAPI->metaFn.getBasicInfo(pVnode, NULL, (int32_t*)&vgId); SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; @@ -451,7 +451,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { const char* db = NULL; int32_t vgId = 0; - pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -522,7 +522,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { int32_t ret = 0; if (pInfo->pCur == NULL) { - pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.vnode); + pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); } if (pInfo->pSchema == NULL) { @@ -614,7 +614,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { blockDataDestroy(dataBlock); if (ret != 0) { - pAPI->metaFn.closeMetaCursor(pInfo->pCur); + pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; setOperatorCompleted(pOperator); } @@ -642,7 +642,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { const char* db = NULL; int32_t vgId = 0; - pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -702,7 +702,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { int32_t ret = 0; if (pInfo->pCur == NULL) { - pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.vnode); + pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); } bool blockFull = false; @@ -722,7 +722,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); pAPI->metaReaderFn.clearReader(&smrSuperTable); - pAPI->metaFn.closeMetaCursor(pInfo->pCur); + pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; T_LONG_JMP(pTaskInfo->env, terrno); } @@ -756,7 +756,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { blockDataDestroy(dataBlock); if (ret != 0) { - pAPI->metaFn.closeMetaCursor(pInfo->pCur); + pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; setOperatorCompleted(pOperator); } @@ -1113,7 +1113,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { const char* db = NULL; int32_t vgId = 0; - pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -1292,7 +1292,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { SSysTableScanInfo* pInfo = pOperator->info; if (pInfo->pCur == NULL) { - pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.vnode); + pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); } blockDataCleanup(pInfo->pRes); @@ -1300,7 +1300,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { const char* db = NULL; int32_t vgId = 0; - pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -1346,7 +1346,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); pAPI->metaReaderFn.clearReader(&mr); - pAPI->metaFn.closeMetaCursor(pInfo->pCur); + pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; T_LONG_JMP(pTaskInfo->env, terrno); } @@ -1456,8 +1456,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found if (ret != 0) { - pAPI->metaFn.closeMetaCursor(pInfo->pCur); - pAPI->metaFn.closeMetaCursor(pInfo->pCur); + pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; setOperatorCompleted(pOperator); } @@ -1741,6 +1740,8 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan extractTbnameSlotId(pInfo, pScanNode); + pInfo->pAPI = &pTaskInfo->storageAPI; + pInfo->accountId = pScanPhyNode->accountId; pInfo->pUser = taosStrdup((void*)pUser); pInfo->sysInfo = pScanPhyNode->sysInfo; @@ -1813,7 +1814,7 @@ void destroySysScanOperator(void* param) { if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { - pInfo->pAPI->metaFn.closeMetaCursor(pInfo->pCur); + pInfo->pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; } if (pInfo->pIdx) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index b4fd571e23..2d366b6110 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2804,6 +2804,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->pState->pFileState = pAPI->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, compareTs, pInfo->pState, pInfo->twAggSup.deleteMark); pInfo->dataVersion = 0; + pInfo->statestore = pTaskInfo->storageAPI.stateStore; pOperator->operatorType = pPhyNode->type; pOperator->blocking = true; @@ -4938,7 +4939,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; initResultSizeInfo(&pOperator->resultInfo, 4096); - pInfo->pState = taosMemoryCalloc(1, sizeof(void)); + pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); *(pInfo->pState) = *(pTaskInfo->streamInfo.pState); pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1); @@ -4986,6 +4987,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL); + pInfo->statestore = pTaskInfo->storageAPI.stateStore; initIntervalDownStream(downstream, pPhyNode->type, pInfo); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 98b685d8b9..24339337b7 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -91,13 +91,14 @@ int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { return winKeyCmprImpl(&pWin1->key, &pWin2->key); } -SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages) { - qWarn("open stream state, %s", path); +SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages) { + qDebug("open stream state, %s", path); SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState)); if (pState == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + pState->pTdbState = taosMemoryCalloc(1, sizeof(STdbState)); if (pState->pTdbState == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -105,29 +106,33 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int return NULL; } + SStreamTask* pStreamTask = pTask; char statePath[1024]; if (!specPath) { - sprintf(statePath, "%s/%d", path, pTask->id.taskId); + sprintf(statePath, "%s/%d", path, pStreamTask->id.taskId); } else { memset(statePath, 0, 1024); tstrncpy(statePath, path, 1024); } - pState->taskId = pTask->id.taskId; - pState->streamId = pTask->id.streamId; + + pState->taskId = pStreamTask->id.taskId; + pState->streamId = pStreamTask->id.streamId; + #ifdef USE_ROCKSDB - // qWarn("open stream state1"); - taosAcquireRef(pTask->pMeta->streamBackendId, pTask->pMeta->streamBackendRid); - int code = streamStateOpenBackend(pTask->pMeta->streamBackend, pState); + SStreamMeta* pMeta = pStreamTask->pMeta; + taosAcquireRef(pMeta->streamBackendId, pMeta->streamBackendRid); + int code = streamStateOpenBackend(pMeta->streamBackend, pState); if (code == -1) { - taosReleaseRef(pTask->pMeta->streamBackendId, pTask->pMeta->streamBackendRid); + taosReleaseRef(pMeta->streamBackendId, pMeta->streamBackendRid); taosMemoryFree(pState); pState = NULL; } + pState->pTdbState->pOwner = pTask; pState->pFileState = NULL; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT); - pState->parNameMap = tSimpleHashInit(1024, hashFn); + pState->parNameMap = tSimpleHashInit(1024, hashFn); return pState; #else @@ -449,7 +454,7 @@ int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pV #ifdef USE_ROCKSDB taosMemoryFree(pVal); #else - streamFreeVal(pVal); + streamStateFreeVal(pVal); #endif return 0; } @@ -700,7 +705,7 @@ void streamStateFreeCur(SStreamStateCur* pCur) { taosMemoryFree(pCur); } -void streamFreeVal(void* val) { +void streamStateFreeVal(void* val) { #ifdef USE_ROCKSDB taosMemoryFree(val); #else From a2f29aa0f52decad5510024718184ffbe909fb2e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 May 2023 13:34:46 +0800 Subject: [PATCH 27/97] refactor: do some internal refactor. --- source/dnode/vnode/CMakeLists.txt | 1 + source/dnode/vnode/src/vnd/vnodeOpen.c | 175 ------------------------- 2 files changed, 1 insertion(+), 175 deletions(-) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index e8660cd6ad..04fad2d78e 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -15,6 +15,7 @@ target_sources( "src/vnd/vnodeSync.c" "src/vnd/vnodeSnapshot.c" "src/vnd/vnodeRetention.c" + "src/vnd/vnodeInitApi.c" # meta "src/meta/metaOpen.c" diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index ebfecc4c15..b5e7c6875b 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -14,7 +14,6 @@ */ #include "vnd.h" -#include "tstreamUpdate.h" int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { SVnodeInfo info = {0}; @@ -445,177 +444,3 @@ void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) { pSnapshot->lastApplyTerm = pVnode->state.commitTerm; pSnapshot->lastConfigIndex = -1; } - -static void initTsdbReaderAPI(TsdReader* pReader); -static void initMetadataAPI(SStoreMeta* pMeta); -static void initTqAPI(SStoreTqReader* pTq); -static void initStateStoreAPI(SStateStore* pStore); -static void initMetaReaderAPI(SStoreMetaReader* pMetaReader); - -void initStorageAPI(SStorageAPI* pAPI) { - initTsdbReaderAPI(&pAPI->tsdReader); - initMetadataAPI(&pAPI->metaFn); - initTqAPI(&pAPI->tqReaderFn); - initStateStoreAPI(&pAPI->stateStore); - initMetaReaderAPI(&pAPI->metaReaderFn); -} - -void initTsdbReaderAPI(TsdReader* pReader) { - pReader->tsdReaderOpen = (__store_reader_open_fn_t)tsdbReaderOpen; - pReader->tsdReaderClose = tsdbReaderClose; - - pReader->tsdNextDataBlock = tsdbNextDataBlock; - - pReader->tsdReaderRetrieveDataBlock = tsdbRetrieveDataBlock; - pReader->tsdReaderReleaseDataBlock = tsdbReleaseDataBlock; - - pReader->tsdReaderRetrieveBlockSMAInfo = tsdbRetrieveDatablockSMA; - - pReader->tsdReaderNotifyClosing = tsdbReaderSetCloseFlag; - pReader->tsdReaderResetStatus = tsdbReaderReset; - - pReader->tsdReaderGetDataBlockDistInfo = tsdbGetFileBlocksDistInfo; - pReader->tsdReaderGetNumOfInMemRows = tsdbGetNumOfRowsInMemTable; // todo this function should be moved away - - pReader->tsdSetQueryTableList = tsdbSetTableList; - pReader->tsdSetReaderTaskId = (void (*)(void *, const char *))tsdbReaderSetId; -} - -void initMetadataAPI(SStoreMeta* pMeta) { - pMeta->isTableExisted = metaIsTableExist; - - pMeta->openTableMetaCursor = metaOpenTbCursor; - pMeta->closeTableMetaCursor = metaCloseTbCursor; - pMeta->cursorNext = metaTbCursorNext; - pMeta->cursorPrev = metaTbCursorPrev; - - pMeta->getBasicInfo = vnodeGetInfo; - pMeta->getNumOfChildTables = metaGetStbStats; - - pMeta->getChildTableList = vnodeGetCtbIdList; - - pMeta->storeGetIndexInfo = vnodeGetIdx; - pMeta->getInvertIndex = vnodeGetIvtIdx; - - pMeta->extractTagVal = (const void *(*)(const void *, int16_t, STagVal *))metaGetTableTagVal; - -} - -void initTqAPI(SStoreTqReader* pTq) { - pTq->tqReaderOpen = tqReaderOpen; - pTq->tqReaderSetColIdList = tqReaderSetColIdList; - - pTq->tqReaderClose = tqReaderClose; - pTq->tqReaderSeek = tqReaderSeek; - pTq->tqRetrieveBlock = tqRetrieveDataBlock; - - pTq->tqReaderNextBlockInWal = tqNextBlockInWal; - - pTq->tqNextBlockImpl = tqNextBlockImpl;// todo remove it - - pTq->tqReaderAddTables = tqReaderAddTbUidList; - pTq->tqReaderSetQueryTableList = tqReaderSetTbUidList; - - pTq->tqReaderRemoveTables = tqReaderRemoveTbUidList; - - pTq->tqReaderIsQueriedTable = tqReaderIsQueriedTable; - pTq->tqReaderCurrentBlockConsumed = tqCurrentBlockConsumed; - - pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it - pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it - - pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it - pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; -} - -void initStateStoreAPI(SStateStore* pStore) { - pStore->streamFileStateInit = streamFileStateInit; - pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF; - - pStore->streamStateGetByPos = streamStateGetByPos; - - pStore->streamStatePutParName = streamStatePutParName; - pStore->streamStateGetParName = streamStateGetParName; - - pStore->streamStateAddIfNotExist = streamStateAddIfNotExist; - pStore->streamStateReleaseBuf = streamStateReleaseBuf; - pStore->streamStateFreeVal = streamStateFreeVal; - - pStore->streamStatePut = streamStatePut; - pStore->streamStateGet = streamStateGet; - pStore->streamStateCheck = streamStateCheck; - pStore->streamStateGetByPos = streamStateGetByPos; - pStore->streamStateDel = streamStateDel; - pStore->streamStateClear = streamStateClear; - pStore->streamStateSaveInfo = streamStateSaveInfo; - pStore->streamStateGetInfo = streamStateGetInfo; - pStore->streamStateSetNumber = streamStateSetNumber; - - pStore->streamStateFillPut = streamStateFillPut; - pStore->streamStateFillGet = streamStateFillGet; - pStore->streamStateFillDel = streamStateFillDel; - - pStore->streamStateCurNext = streamStateCurNext; - pStore->streamStateCurPrev = streamStateCurPrev; - - pStore->streamStateGetAndCheckCur = streamStateGetAndCheckCur; - pStore->streamStateSeekKeyNext = streamStateSeekKeyNext; - pStore->streamStateFillSeekKeyNext = streamStateFillSeekKeyNext; - pStore->streamStateFillSeekKeyPrev = streamStateFillSeekKeyPrev; - pStore->streamStateFreeCur = streamStateFreeCur; - - pStore->streamStateGetGroupKVByCur = streamStateGetGroupKVByCur; - pStore->streamStateGetKVByCur = streamStateGetKVByCur; - - pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist; - pStore->streamStateSessionPut = streamStateSessionPut; - pStore->streamStateSessionGet = streamStateSessionGet; - pStore->streamStateSessionDel = streamStateSessionDel; - pStore->streamStateSessionClear = streamStateSessionClear; - pStore->streamStateSessionGetKVByCur = streamStateSessionGetKVByCur; - pStore->streamStateStateAddIfNotExist = streamStateStateAddIfNotExist; - pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange; - - pStore->updateInfoInit = updateInfoInit; - pStore->updateInfoFillBlockData = updateInfoFillBlockData; - pStore->updateInfoIsUpdated = updateInfoIsUpdated; - pStore->updateInfoIsTableInserted = updateInfoIsTableInserted; - pStore->updateInfoDestroy = updateInfoDestroy; - - pStore->updateInfoInitP = updateInfoInitP; - pStore->updateInfoAddCloseWindowSBF = updateInfoAddCloseWindowSBF; - pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF; - pStore->updateInfoSerialize = updateInfoSerialize; - pStore->updateInfoDeserialize = updateInfoDeserialize; - - pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext; - pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev; - pStore->streamStateSessionSeekKeyCurrentNext = streamStateSessionSeekKeyCurrentNext; - - pStore->streamFileStateInit = streamFileStateInit; - - pStore->streamFileStateDestroy = streamFileStateDestroy; - pStore->streamFileStateClear = streamFileStateClear; - pStore->needClearDiskBuff = needClearDiskBuff; - - pStore->streamStateOpen = streamStateOpen; - pStore->streamStateClose = streamStateClose; - pStore->streamStateBegin = streamStateBegin; - pStore->streamStateCommit = streamStateCommit; - pStore->streamStateDestroy= streamStateDestroy; - pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint; -} - -void initMetaReaderAPI(SStoreMetaReader* pMetaReader) { - pMetaReader->initReader = _metaReaderInit; - pMetaReader->clearReader = metaReaderClear; - - pMetaReader->getTableEntryByUid = metaReaderGetTableEntryByUid; - pMetaReader->clearReader = metaReaderClear; - - pMetaReader->getEntryGetUidCache = metaReaderGetTableEntryByUidCache; - pMetaReader->getTableEntryByName = metaGetTableEntryByName; - - pMetaReader->readerReleaseLock = metaReaderReleaseLock; -} - From b2ce45b07bb4094acfe483e9f65d788d04fbe2f3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 May 2023 13:39:20 +0800 Subject: [PATCH 28/97] refactor: do some internal refactor. --- source/dnode/vnode/src/vnd/vnodeInitApi.c | 191 ++++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 source/dnode/vnode/src/vnd/vnodeInitApi.c diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c new file mode 100644 index 0000000000..3f427d9965 --- /dev/null +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "storageapi.h" +#include "vnodeInt.h" +#include "tstreamUpdate.h" + +static void initTsdbReaderAPI(TsdReader* pReader); +static void initMetadataAPI(SStoreMeta* pMeta); +static void initTqAPI(SStoreTqReader* pTq); +static void initStateStoreAPI(SStateStore* pStore); +static void initMetaReaderAPI(SStoreMetaReader* pMetaReader); + +void initStorageAPI(SStorageAPI* pAPI) { + initTsdbReaderAPI(&pAPI->tsdReader); + initMetadataAPI(&pAPI->metaFn); + initTqAPI(&pAPI->tqReaderFn); + initStateStoreAPI(&pAPI->stateStore); + initMetaReaderAPI(&pAPI->metaReaderFn); +} + +void initTsdbReaderAPI(TsdReader* pReader) { + pReader->tsdReaderOpen = (__store_reader_open_fn_t)tsdbReaderOpen; + pReader->tsdReaderClose = tsdbReaderClose; + + pReader->tsdNextDataBlock = tsdbNextDataBlock; + + pReader->tsdReaderRetrieveDataBlock = tsdbRetrieveDataBlock; + pReader->tsdReaderReleaseDataBlock = tsdbReleaseDataBlock; + + pReader->tsdReaderRetrieveBlockSMAInfo = tsdbRetrieveDatablockSMA; + + pReader->tsdReaderNotifyClosing = tsdbReaderSetCloseFlag; + pReader->tsdReaderResetStatus = tsdbReaderReset; + + pReader->tsdReaderGetDataBlockDistInfo = tsdbGetFileBlocksDistInfo; + pReader->tsdReaderGetNumOfInMemRows = tsdbGetNumOfRowsInMemTable; // todo this function should be moved away + + pReader->tsdSetQueryTableList = tsdbSetTableList; + pReader->tsdSetReaderTaskId = (void (*)(void *, const char *))tsdbReaderSetId; +} + +void initMetadataAPI(SStoreMeta* pMeta) { + pMeta->isTableExisted = metaIsTableExist; + + pMeta->openTableMetaCursor = metaOpenTbCursor; + pMeta->closeTableMetaCursor = metaCloseTbCursor; + pMeta->cursorNext = metaTbCursorNext; + pMeta->cursorPrev = metaTbCursorPrev; + + pMeta->getBasicInfo = vnodeGetInfo; + pMeta->getNumOfChildTables = metaGetStbStats; + + pMeta->getChildTableList = vnodeGetCtbIdList; + + pMeta->storeGetIndexInfo = vnodeGetIdx; + pMeta->getInvertIndex = vnodeGetIvtIdx; + + pMeta->extractTagVal = (const void *(*)(const void *, int16_t, STagVal *))metaGetTableTagVal; + +} + +void initTqAPI(SStoreTqReader* pTq) { + pTq->tqReaderOpen = tqReaderOpen; + pTq->tqReaderSetColIdList = tqReaderSetColIdList; + + pTq->tqReaderClose = tqReaderClose; + pTq->tqReaderSeek = tqReaderSeek; + pTq->tqRetrieveBlock = tqRetrieveDataBlock; + + pTq->tqReaderNextBlockInWal = tqNextBlockInWal; + + pTq->tqNextBlockImpl = tqNextBlockImpl;// todo remove it + + pTq->tqReaderAddTables = tqReaderAddTbUidList; + pTq->tqReaderSetQueryTableList = tqReaderSetTbUidList; + + pTq->tqReaderRemoveTables = tqReaderRemoveTbUidList; + + pTq->tqReaderIsQueriedTable = tqReaderIsQueriedTable; + pTq->tqReaderCurrentBlockConsumed = tqCurrentBlockConsumed; + + pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it + pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it + + pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it + pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; +} + +void initStateStoreAPI(SStateStore* pStore) { + pStore->streamFileStateInit = streamFileStateInit; + pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF; + + pStore->streamStateGetByPos = streamStateGetByPos; + + pStore->streamStatePutParName = streamStatePutParName; + pStore->streamStateGetParName = streamStateGetParName; + + pStore->streamStateAddIfNotExist = streamStateAddIfNotExist; + pStore->streamStateReleaseBuf = streamStateReleaseBuf; + pStore->streamStateFreeVal = streamStateFreeVal; + + pStore->streamStatePut = streamStatePut; + pStore->streamStateGet = streamStateGet; + pStore->streamStateCheck = streamStateCheck; + pStore->streamStateGetByPos = streamStateGetByPos; + pStore->streamStateDel = streamStateDel; + pStore->streamStateClear = streamStateClear; + pStore->streamStateSaveInfo = streamStateSaveInfo; + pStore->streamStateGetInfo = streamStateGetInfo; + pStore->streamStateSetNumber = streamStateSetNumber; + + pStore->streamStateFillPut = streamStateFillPut; + pStore->streamStateFillGet = streamStateFillGet; + pStore->streamStateFillDel = streamStateFillDel; + + pStore->streamStateCurNext = streamStateCurNext; + pStore->streamStateCurPrev = streamStateCurPrev; + + pStore->streamStateGetAndCheckCur = streamStateGetAndCheckCur; + pStore->streamStateSeekKeyNext = streamStateSeekKeyNext; + pStore->streamStateFillSeekKeyNext = streamStateFillSeekKeyNext; + pStore->streamStateFillSeekKeyPrev = streamStateFillSeekKeyPrev; + pStore->streamStateFreeCur = streamStateFreeCur; + + pStore->streamStateGetGroupKVByCur = streamStateGetGroupKVByCur; + pStore->streamStateGetKVByCur = streamStateGetKVByCur; + + pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist; + pStore->streamStateSessionPut = streamStateSessionPut; + pStore->streamStateSessionGet = streamStateSessionGet; + pStore->streamStateSessionDel = streamStateSessionDel; + pStore->streamStateSessionClear = streamStateSessionClear; + pStore->streamStateSessionGetKVByCur = streamStateSessionGetKVByCur; + pStore->streamStateStateAddIfNotExist = streamStateStateAddIfNotExist; + pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange; + + pStore->updateInfoInit = updateInfoInit; + pStore->updateInfoFillBlockData = updateInfoFillBlockData; + pStore->updateInfoIsUpdated = updateInfoIsUpdated; + pStore->updateInfoIsTableInserted = updateInfoIsTableInserted; + pStore->updateInfoDestroy = updateInfoDestroy; + + pStore->updateInfoInitP = updateInfoInitP; + pStore->updateInfoAddCloseWindowSBF = updateInfoAddCloseWindowSBF; + pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF; + pStore->updateInfoSerialize = updateInfoSerialize; + pStore->updateInfoDeserialize = updateInfoDeserialize; + + pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext; + pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev; + pStore->streamStateSessionSeekKeyCurrentNext = streamStateSessionSeekKeyCurrentNext; + + pStore->streamFileStateInit = streamFileStateInit; + + pStore->streamFileStateDestroy = streamFileStateDestroy; + pStore->streamFileStateClear = streamFileStateClear; + pStore->needClearDiskBuff = needClearDiskBuff; + + pStore->streamStateOpen = streamStateOpen; + pStore->streamStateClose = streamStateClose; + pStore->streamStateBegin = streamStateBegin; + pStore->streamStateCommit = streamStateCommit; + pStore->streamStateDestroy= streamStateDestroy; + pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint; +} + +void initMetaReaderAPI(SStoreMetaReader* pMetaReader) { + pMetaReader->initReader = _metaReaderInit; + pMetaReader->clearReader = metaReaderClear; + + pMetaReader->getTableEntryByUid = metaReaderGetTableEntryByUid; + pMetaReader->clearReader = metaReaderClear; + + pMetaReader->getEntryGetUidCache = metaReaderGetTableEntryByUidCache; + pMetaReader->getTableEntryByName = metaGetTableEntryByName; + + pMetaReader->readerReleaseLock = metaReaderReleaseLock; +} \ No newline at end of file From 5b1e0be7560ebc658aec864e6db2b7fabcce15bb Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 24 May 2023 13:41:21 +0800 Subject: [PATCH 29/97] enhance: add log --- source/libs/scalar/src/filter.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index ecf621a726..3c01edf857 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3446,10 +3446,10 @@ typedef enum { typedef struct { SFltSclDatumKind kind; union { - int64_t i; // for int64, uint64 and double and bool (1 true, 0 false) - uint64_t u; - double d; - uint8_t *pData; // for varchar, nchar + int64_t i; // for int and bool (1 true, 0 false) and ts + uint64_t u; // for uint + double d; // for double + uint8_t *pData; // for varchar, nchar, len prefixed }; SDataType type; // TODO: original data type, may not be used? } SFltSclDatum; @@ -3765,9 +3765,15 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t SColumnDataAgg *pAgg = pDataStatis[j]; SArray *points = taosArrayInit(2, sizeof(SFltSclPoint)); fltSclBuildRangeFromBlockSma(colRange, pAgg, numOfRows, points); + qDebug("column data agg: nulls %d, rows %d, max %" PRId64 " min " PRId64, pAgg->numOfNull, numOfRows, pAgg->max, + pAgg->min); + SArray *merged = taosArrayInit(8, sizeof(SFltSclPoint)); fltSclIntersect(points, colRange->points, merged); bool isIntersect = taosArrayGetSize(merged) != 0; + qDebug("filter range execute, scalar mode, column range found. colId: %d colName: %s has overlap: %d", + colRange->colNode->colId, colRange->colNode->colName, isIntersect); + taosArrayDestroy(merged); taosArrayDestroy(points); if (!isIntersect) { From f47a495d118d16275525160317ae586556b64bb6 Mon Sep 17 00:00:00 2001 From: Li Ya Qiang Date: Wed, 24 May 2023 13:57:59 +0800 Subject: [PATCH 30/97] implement TD-24378 --- packaging/tools/makepkg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 45ef20a22d..2f1e803689 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -429,7 +429,7 @@ if [ "$exitcode" != "0" ]; then exit $exitcode fi -if [ -n "${taostools_bin_files}" ]; then +if [ -n "${taostools_bin_files}" ] && [ "$verMode" != "cloud" ]; then wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${taostools_install_dir}/bin/TDinsight.sh && echo "TDinsight.sh downloaded!"|| echo "failed to download TDinsight.sh" if [ "$osType" != "Darwin" ]; then tar -zcv -f "$(basename ${taostools_pkg_name}).tar.gz" "$(basename ${taostools_install_dir})" --remove-files || : From fb9b0a3e86c47fa6abb7420184dc128392a19304 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 24 May 2023 14:34:52 +0800 Subject: [PATCH 31/97] feat:add privilege for schemaless --- source/client/inc/clientSml.h | 6 +- source/client/src/clientSml.c | 162 ++++++++++++++++++++-------------- utils/test/c/sml_test.c | 30 +++++++ 3 files changed, 131 insertions(+), 67 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index b20fc6f57a..f9cbffa748 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -232,9 +232,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t * int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset); //SArray *smlJsonParseTags(char *start, char *end); bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); -void* nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn); -int nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn); -int nodeListSize(NodeList* list); +//void* nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn); +//int nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn); +//int nodeListSize(NodeList* list); bool smlDoubleToInt64OverFlow(double num); int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 6170b0a056..e873c81c0f 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -24,72 +24,91 @@ int64_t smlToMilli[3] = {3600000LL, 60000LL, 1000LL}; int64_t smlFactorNS[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; int64_t smlFactorS[3] = {1000LL, 1000000LL, 1000000000LL}; -void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) { - NodeList *tmp = list; - while (tmp) { - if (fn == NULL) { - if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { - return tmp->data.value; - } - } else { - if (tmp->data.used && fn(tmp->data.key, key) == 0) { - return tmp->data.value; - } - } +//void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) { +// NodeList *tmp = list; +// while (tmp) { +// if (fn == NULL) { +// if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { +// return tmp->data.value; +// } +// } else { +// if (tmp->data.used && fn(tmp->data.key, key) == 0) { +// return tmp->data.value; +// } +// } +// +// tmp = tmp->next; +// } +// return NULL; +//} +// +//int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) { +// NodeList *tmp = *list; +// while (tmp) { +// if (!tmp->data.used) break; +// if (fn == NULL) { +// if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { +// return -1; +// } +// } else { +// if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) { +// return -1; +// } +// } +// +// tmp = tmp->next; +// } +// if (tmp) { +// tmp->data.key = key; +// tmp->data.keyLen = len; +// tmp->data.value = value; +// tmp->data.used = true; +// } else { +// NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList)); +// if (newNode == NULL) { +// return -1; +// } +// newNode->data.key = key; +// newNode->data.keyLen = len; +// newNode->data.value = value; +// newNode->data.used = true; +// newNode->next = *list; +// *list = newNode; +// } +// return 0; +//} +// +//int nodeListSize(NodeList *list) { +// int cnt = 0; +// while (list) { +// if (list->data.used) +// cnt++; +// else +// break; +// list = list->next; +// } +// return cnt; +//} - tmp = tmp->next; - } - return NULL; -} - -int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) { - NodeList *tmp = *list; - while (tmp) { - if (!tmp->data.used) break; - if (fn == NULL) { - if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { - return -1; - } - } else { - if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) { - return -1; - } - } - - tmp = tmp->next; - } - if (tmp) { - tmp->data.key = key; - tmp->data.keyLen = len; - tmp->data.value = value; - tmp->data.used = true; +static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo* conn, const char* pTabName, AUTH_TYPE type){ + SUserAuthInfo pAuth = {0}; + snprintf(pAuth.user, sizeof(pAuth.user), "%s", info->taos->user); + if (NULL == pTabName) { + tNameSetDbName(&pAuth.tbName, info->taos->acctId, info->pRequest->pDb, strlen(info->pRequest->pDb)); } else { - NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList)); - if (newNode == NULL) { - return -1; - } - newNode->data.key = key; - newNode->data.keyLen = len; - newNode->data.value = value; - newNode->data.used = true; - newNode->next = *list; - *list = newNode; + toName(info->taos->acctId, info->pRequest->pDb, pTabName, &pAuth.tbName); } - return 0; -} + pAuth.type = type; -int nodeListSize(NodeList *list) { - int cnt = 0; - while (list) { - if (list->data.used) - cnt++; - else - break; - list = list->next; - } - return cnt; -} + int32_t code = TSDB_CODE_SUCCESS; + SUserAuthRes authRes = {0}; + code = catalogChkAuth(info->pCatalog, conn, &pAuth, &authRes); + + + return code ? (authRes.pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code; + +} inline bool smlDoubleToInt64OverFlow(double num) { if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; return false; @@ -813,6 +832,10 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) { + code = smlCheckAuth(info, &conn, NULL, AUTH_TYPE_WRITE); + if(code != TSDB_CODE_SUCCESS){ + goto end; + } uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName.tname); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); @@ -857,6 +880,10 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { goto end; } if (action != SCHEMA_ACTION_NULL) { + code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE); + if(code != TSDB_CODE_SUCCESS){ + goto end; + } uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table tag, table:%s, action:%d", info->id, pName.tname, action); SArray *pColumns = @@ -927,6 +954,10 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { goto end; } if (action != SCHEMA_ACTION_NULL) { + code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE); + if(code != TSDB_CODE_SUCCESS){ + goto end; + } uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table col, table:%s, action:%d", info->id, pName.tname, action); SArray *pColumns = @@ -1367,6 +1398,11 @@ static int32_t smlInsertData(SSmlHandle *info) { conn.requestObjRefId = info->pRequest->self; conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE); + if(code != TSDB_CODE_SUCCESS){ + return code; + } + SVgroupInfo vg; code = catalogGetTableHashVgroup(info->pCatalog, &conn, &pName, &vg); if (code != TSDB_CODE_SUCCESS) { @@ -1586,9 +1622,7 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL do { code = smlModifyDBSchemas(info); - if (code == 0 || code == TSDB_CODE_SML_INVALID_DATA || code == TSDB_CODE_PAR_TOO_MANY_COLUMNS - || code == TSDB_CODE_PAR_INVALID_TAGS_NUM || code == TSDB_CODE_PAR_INVALID_TAGS_LENGTH - || code == TSDB_CODE_PAR_INVALID_ROW_LENGTH || code == TSDB_CODE_MND_FIELD_VALUE_OVERFLOW) { + if (code != TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER && code != TSDB_CODE_SDB_OBJ_CREATING && code != TSDB_CODE_MND_TRANS_CONFLICT) { break; } taosMsleep(100); diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index ac5aff4727..f82b820244 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1132,6 +1132,34 @@ int sml_td22900_Test() { return code; } +int sml_td24070_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "CREATE user test pass 'test'"); + taos_free_result(pRes); + + pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS td24070"); + taos_free_result(pRes); + + taos_close(taos); + + taos = taos_connect("localhost", "test", "test", NULL, 0); + const char* sql[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299372000"}; + + pRes = taos_query(taos, "use td24070"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + taos_close(taos); + + return code; +} + int sml_td23881_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -1379,6 +1407,8 @@ int main(int argc, char *argv[]) { } int ret = 0; + ret = sml_td24070_Test(); + ASSERT(!ret); ret = sml_td23881_Test(); ASSERT(ret); ret = sml_escape_Test(); From f203c7806de763444c10f8ea08438ba8ea57300c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 May 2023 14:52:50 +0800 Subject: [PATCH 32/97] fix(tools): fix build failure on enterprise edition. --- source/libs/qworker/CMakeLists.txt | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/source/libs/qworker/CMakeLists.txt b/source/libs/qworker/CMakeLists.txt index 92ccde3163..a947a2dd19 100644 --- a/source/libs/qworker/CMakeLists.txt +++ b/source/libs/qworker/CMakeLists.txt @@ -7,9 +7,15 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries(qworker - PRIVATE os util transport nodes planner qcom executor - ) +IF (TD_GRANT) + TARGET_LINK_LIBRARIES(qworker + PRIVATE os util transport nodes planner qcom executor grant + ) +ELSE () + TARGET_LINK_LIBRARIES(qworker + PRIVATE os util transport nodes planner qcom executor + ) +ENDIF() if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) From a56618077049d97bf369f10f9f492a259e9f8dde Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 May 2023 15:16:50 +0800 Subject: [PATCH 33/97] fix: fix syntax error. --- include/libs/executor/storageapi.h | 4 ++-- source/libs/stream/test/CMakeLists.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 8790c302ce..300507c132 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -86,7 +86,7 @@ typedef struct SMetaReader { } SMetaReader; typedef struct SMTbCursor { - struct TBC *pDbc; + void * pDbc; void * pKey; void * pVal; int32_t kLen; @@ -169,7 +169,7 @@ typedef struct SMetaTableInfo { typedef struct SSnapContext { SMeta * pMeta; // todo remove it int64_t snapVersion; - struct TBC *pCur; + void * pCur; int64_t suid; int8_t subType; SHashObj * idVersion; diff --git a/source/libs/stream/test/CMakeLists.txt b/source/libs/stream/test/CMakeLists.txt index a0c1717690..75acec7f01 100644 --- a/source/libs/stream/test/CMakeLists.txt +++ b/source/libs/stream/test/CMakeLists.txt @@ -10,7 +10,7 @@ ADD_EXECUTABLE(streamUpdateTest "tstreamUpdateTest.cpp") TARGET_LINK_LIBRARIES( streamUpdateTest - PUBLIC os util common gtest gtest_main stream + PUBLIC os util common gtest gtest_main stream executor ) TARGET_INCLUDE_DIRECTORIES( From e5f9d78b6d0df0c0fa9970a29ce9f1d2617ae52a Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 24 May 2023 16:39:38 +0800 Subject: [PATCH 34/97] fix: rocksdb so --- cmake/cmake.options | 5 -- contrib/CMakeLists.txt | 56 --------------------- contrib/test/CMakeLists.txt | 3 -- source/dnode/mgmt/mgmt_dnode/CMakeLists.txt | 2 +- source/dnode/vnode/CMakeLists.txt | 2 - source/libs/stream/CMakeLists.txt | 2 - 6 files changed, 1 insertion(+), 69 deletions(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index a7e62350df..064b1f7c6f 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -106,11 +106,6 @@ option( OFF ) -option( - BUILD_WITH_ROCKSDB - "If build with rocksdb" - ON -) option( BUILD_WITH_SQLITE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 536d4eae8e..ad3d8bc326 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -77,11 +77,6 @@ if(${BUILD_WITH_LEVELDB}) cat("${TD_SUPPORT_DIR}/leveldb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_WITH_LEVELDB}) -# rocksdb -if(${BUILD_WITH_ROCKSDB}) - cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) - add_definitions(-DUSE_ROCKSDB) -endif(${BUILD_WITH_ROCKSDB}) # canonical-raft if(${BUILD_WITH_CRAFT}) @@ -220,57 +215,6 @@ if(${BUILD_WITH_LEVELDB}) ) endif(${BUILD_WITH_LEVELDB}) -# rocksdb -# To support rocksdb build on ubuntu: sudo apt-get install libgflags-dev -if(${BUILD_WITH_ROCKSDB}) - if(${TD_LINUX}) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized -Wno-error=unused-but-set-variable -Wno-error=unused-variable -Wno-error=unused-function -Wno-errno=unused-private-field -Wno-error=unused-result") - endif(${TD_LINUX}) - - if(${TD_DARWIN}) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized") - endif(${TD_DARWIN}) - - if (${TD_WINDOWS}) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819") - endif(${TD_WINDOWS}) - - - if(${TD_DARWIN}) - option(HAVE_THREAD_LOCAL "" OFF) - option(WITH_IOSTATS_CONTEXT "" OFF) - option(WITH_PERF_CONTEXT "" OFF) - endif(${TD_DARWIN}) - - if(${TD_WINDOWS}) - option(WITH_JNI "" ON) - endif(${TD_WINDOWS}) - - if(${TD_WINDOWS}) - option(WITH_MD_LIBRARY "build with MD" OFF) - set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib) - endif(${TD_WINDOWS}) - - - option(WITH_FALLOCATE "" OFF) - option(WITH_JEMALLOC "" OFF) - option(WITH_GFLAGS "" OFF) - option(PORTABLE "" ON) - option(WITH_LIBURING "" OFF) - option(FAIL_ON_WARNINGS OFF) - - option(WITH_TESTS "" OFF) - option(WITH_BENCHMARK_TOOLS "" OFF) - option(WITH_TOOLS "" OFF) - option(WITH_LIBURING "" OFF) - - option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF) - add_subdirectory(rocksdb EXCLUDE_FROM_ALL) - target_include_directories( - rocksdb - PUBLIC $ - ) -endif(${BUILD_WITH_ROCKSDB}) # lucene # To support build on ubuntu: sudo apt-get install libboost-all-dev diff --git a/contrib/test/CMakeLists.txt b/contrib/test/CMakeLists.txt index f35cf0d13d..8d095518e2 100644 --- a/contrib/test/CMakeLists.txt +++ b/contrib/test/CMakeLists.txt @@ -1,7 +1,4 @@ # rocksdb -if(${BUILD_WITH_ROCKSDB}) - add_subdirectory(rocksdb) -endif(${BUILD_WITH_ROCKSDB}) if(${BUILD_WITH_LUCENE}) add_subdirectory(lucene) diff --git a/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt b/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt index fdd0830a58..7d0b6d617d 100644 --- a/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt +++ b/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt @@ -9,5 +9,5 @@ target_include_directories( PUBLIC "${GRANT_CFG_INCLUDE_DIR}" ) target_link_libraries( - mgmt_dnode node_util + mgmt_dnode node_util rocksdb ) \ No newline at end of file diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index e8660cd6ad..79a7d55198 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -117,9 +117,7 @@ if(${BUILD_WITH_INVERTEDINDEX}) add_definitions(-DUSE_INVERTED_INDEX) endif(${BUILD_WITH_INVERTEDINDEX}) -if(${BUILD_WITH_ROCKSDB}) add_definitions(-DUSE_ROCKSDB) -endif(${BUILD_WITH_ROCKSDB}) diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt index 2edbc44aae..5952250b3e 100644 --- a/source/libs/stream/CMakeLists.txt +++ b/source/libs/stream/CMakeLists.txt @@ -6,7 +6,6 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -if(${BUILD_WITH_ROCKSDB}) target_link_libraries( stream PUBLIC rocksdb tdb @@ -19,7 +18,6 @@ if(${BUILD_WITH_ROCKSDB}) ) add_definitions(-DUSE_ROCKSDB) -endif(${BUILD_WITH_ROCKSDB}) #target_link_libraries( From d7b1613d850eed2d65faf94c84f58a07320082b8 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 24 May 2023 16:42:50 +0800 Subject: [PATCH 35/97] Revert "fix: rocksdb so" This reverts commit e5f9d78b6d0df0c0fa9970a29ce9f1d2617ae52a. --- cmake/cmake.options | 5 ++ contrib/CMakeLists.txt | 56 +++++++++++++++++++++ contrib/test/CMakeLists.txt | 3 ++ source/dnode/mgmt/mgmt_dnode/CMakeLists.txt | 2 +- source/dnode/vnode/CMakeLists.txt | 2 + source/libs/stream/CMakeLists.txt | 2 + 6 files changed, 69 insertions(+), 1 deletion(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index 064b1f7c6f..a7e62350df 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -106,6 +106,11 @@ option( OFF ) +option( + BUILD_WITH_ROCKSDB + "If build with rocksdb" + ON +) option( BUILD_WITH_SQLITE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index ad3d8bc326..536d4eae8e 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -77,6 +77,11 @@ if(${BUILD_WITH_LEVELDB}) cat("${TD_SUPPORT_DIR}/leveldb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_WITH_LEVELDB}) +# rocksdb +if(${BUILD_WITH_ROCKSDB}) + cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + add_definitions(-DUSE_ROCKSDB) +endif(${BUILD_WITH_ROCKSDB}) # canonical-raft if(${BUILD_WITH_CRAFT}) @@ -215,6 +220,57 @@ if(${BUILD_WITH_LEVELDB}) ) endif(${BUILD_WITH_LEVELDB}) +# rocksdb +# To support rocksdb build on ubuntu: sudo apt-get install libgflags-dev +if(${BUILD_WITH_ROCKSDB}) + if(${TD_LINUX}) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized -Wno-error=unused-but-set-variable -Wno-error=unused-variable -Wno-error=unused-function -Wno-errno=unused-private-field -Wno-error=unused-result") + endif(${TD_LINUX}) + + if(${TD_DARWIN}) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized") + endif(${TD_DARWIN}) + + if (${TD_WINDOWS}) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819") + endif(${TD_WINDOWS}) + + + if(${TD_DARWIN}) + option(HAVE_THREAD_LOCAL "" OFF) + option(WITH_IOSTATS_CONTEXT "" OFF) + option(WITH_PERF_CONTEXT "" OFF) + endif(${TD_DARWIN}) + + if(${TD_WINDOWS}) + option(WITH_JNI "" ON) + endif(${TD_WINDOWS}) + + if(${TD_WINDOWS}) + option(WITH_MD_LIBRARY "build with MD" OFF) + set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib) + endif(${TD_WINDOWS}) + + + option(WITH_FALLOCATE "" OFF) + option(WITH_JEMALLOC "" OFF) + option(WITH_GFLAGS "" OFF) + option(PORTABLE "" ON) + option(WITH_LIBURING "" OFF) + option(FAIL_ON_WARNINGS OFF) + + option(WITH_TESTS "" OFF) + option(WITH_BENCHMARK_TOOLS "" OFF) + option(WITH_TOOLS "" OFF) + option(WITH_LIBURING "" OFF) + + option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF) + add_subdirectory(rocksdb EXCLUDE_FROM_ALL) + target_include_directories( + rocksdb + PUBLIC $ + ) +endif(${BUILD_WITH_ROCKSDB}) # lucene # To support build on ubuntu: sudo apt-get install libboost-all-dev diff --git a/contrib/test/CMakeLists.txt b/contrib/test/CMakeLists.txt index 8d095518e2..f35cf0d13d 100644 --- a/contrib/test/CMakeLists.txt +++ b/contrib/test/CMakeLists.txt @@ -1,4 +1,7 @@ # rocksdb +if(${BUILD_WITH_ROCKSDB}) + add_subdirectory(rocksdb) +endif(${BUILD_WITH_ROCKSDB}) if(${BUILD_WITH_LUCENE}) add_subdirectory(lucene) diff --git a/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt b/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt index 7d0b6d617d..fdd0830a58 100644 --- a/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt +++ b/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt @@ -9,5 +9,5 @@ target_include_directories( PUBLIC "${GRANT_CFG_INCLUDE_DIR}" ) target_link_libraries( - mgmt_dnode node_util rocksdb + mgmt_dnode node_util ) \ No newline at end of file diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 79a7d55198..e8660cd6ad 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -117,7 +117,9 @@ if(${BUILD_WITH_INVERTEDINDEX}) add_definitions(-DUSE_INVERTED_INDEX) endif(${BUILD_WITH_INVERTEDINDEX}) +if(${BUILD_WITH_ROCKSDB}) add_definitions(-DUSE_ROCKSDB) +endif(${BUILD_WITH_ROCKSDB}) diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt index 5952250b3e..2edbc44aae 100644 --- a/source/libs/stream/CMakeLists.txt +++ b/source/libs/stream/CMakeLists.txt @@ -6,6 +6,7 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) +if(${BUILD_WITH_ROCKSDB}) target_link_libraries( stream PUBLIC rocksdb tdb @@ -18,6 +19,7 @@ target_include_directories( ) add_definitions(-DUSE_ROCKSDB) +endif(${BUILD_WITH_ROCKSDB}) #target_link_libraries( From 768372fc7bca72b43b47a4dc29ebc3b96dab4c6d Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 24 May 2023 16:45:06 +0800 Subject: [PATCH 36/97] fix: fix compilation warning --- source/libs/scalar/src/filter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 3c01edf857..27cf9da1d5 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3765,7 +3765,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t SColumnDataAgg *pAgg = pDataStatis[j]; SArray *points = taosArrayInit(2, sizeof(SFltSclPoint)); fltSclBuildRangeFromBlockSma(colRange, pAgg, numOfRows, points); - qDebug("column data agg: nulls %d, rows %d, max %" PRId64 " min " PRId64, pAgg->numOfNull, numOfRows, pAgg->max, + qDebug("column data agg: nulls %d, rows %d, max %" PRId64 " min %" PRId64, pAgg->numOfNull, numOfRows, pAgg->max, pAgg->min); SArray *merged = taosArrayInit(8, sizeof(SFltSclPoint)); From fe09705855ecd5f695ce91809cdbf0c596b26e29 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 May 2023 17:13:20 +0800 Subject: [PATCH 37/97] fix: set the init function pointer. --- include/libs/executor/storageapi.h | 2 ++ include/libs/index/index.h | 18 +++++--------- source/dnode/qnode/CMakeLists.txt | 1 + source/dnode/snode/CMakeLists.txt | 1 + source/dnode/vnode/inc/vnode.h | 5 ---- source/dnode/vnode/src/inc/meta.h | 6 +++++ source/dnode/vnode/src/meta/metaQuery.c | 16 ++++++------- source/dnode/vnode/src/vnd/vnodeInitApi.c | 11 ++++++++- source/libs/executor/src/sysscanoperator.c | 28 +++++++++++----------- source/libs/index/src/indexFilter.c | 7 +++--- source/libs/qworker/CMakeLists.txt | 4 ++-- source/libs/scheduler/CMakeLists.txt | 2 +- source/libs/stream/CMakeLists.txt | 2 +- 13 files changed, 55 insertions(+), 48 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 300507c132..97900de7ba 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -21,6 +21,7 @@ #include "taosdef.h" #include "tmsg.h" #include "tcommon.h" +#include "index.h" #ifdef __cplusplus extern "C" { @@ -544,6 +545,7 @@ typedef struct SStorageAPI { SStoreSnapshotFn snapshotFn; SStoreTqReader tqReaderFn; SStateStore stateStore; + SMetaDataFilterAPI metaFilter; } SStorageAPI; #ifdef __cplusplus diff --git a/include/libs/index/index.h b/include/libs/index/index.h index fd71c531fa..162b99e553 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -239,7 +239,6 @@ void indexCleanup(); * @param results * @return */ - typedef struct SMetaFltParam { uint64_t suid; int16_t cid; @@ -250,17 +249,12 @@ typedef struct SMetaFltParam { int (*filterFunc)(void *a, void *b, int16_t type); } SMetaFltParam; -typedef struct SStoreAPI { - int32_t (*metaFilterTableIds)(); - int32_t (*metaFilterCreateTime)(); - int32_t (*metaFilterTableName)(); - int32_t (*metaFilterTtl)(); -} SStoreAPI; - -//int32_t metaFilterTableIds(void *pMeta, SMetaFltParam *param, SArray *results); -//int32_t metaFilterCreateTime(void *pMeta, SMetaFltParam *parm, SArray *pUids); -//int32_t metaFilterTableName(void *pMeta, SMetaFltParam *param, SArray *pUids); -//int32_t metaFilterTtl(void *pMeta, SMetaFltParam *param, SArray *pUids); +typedef struct SMetaDataFilterAPI { + int32_t (*metaFilterTableIds)(void *pVnode, SMetaFltParam *arg, SArray *pUids); + int32_t (*metaFilterCreateTime)(void *pVnode, SMetaFltParam *arg, SArray *pUids); + int32_t (*metaFilterTableName)(void *pVnode, SMetaFltParam *arg, SArray *pUids); + int32_t (*metaFilterTtl)(void *pVnode, SMetaFltParam *arg, SArray *pUids); +} SMetaDataFilterAPI; #ifdef __cplusplus } diff --git a/source/dnode/qnode/CMakeLists.txt b/source/dnode/qnode/CMakeLists.txt index 13429d0cc1..10bbbc1b26 100644 --- a/source/dnode/qnode/CMakeLists.txt +++ b/source/dnode/qnode/CMakeLists.txt @@ -16,4 +16,5 @@ target_link_libraries( PRIVATE executor PRIVATE tdb PRIVATE wal + PRIVATE index ) \ No newline at end of file diff --git a/source/dnode/snode/CMakeLists.txt b/source/dnode/snode/CMakeLists.txt index 1d8c80c422..ebfe80ecab 100644 --- a/source/dnode/snode/CMakeLists.txt +++ b/source/dnode/snode/CMakeLists.txt @@ -15,4 +15,5 @@ target_link_libraries( PRIVATE qcom PRIVATE stream PRIVATE wal + PRIVATE index ) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 95fe150443..84153f3867 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -151,11 +151,6 @@ int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables); // //} SMetaFltParam; -// TODO, refactor later -//int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *results); -//int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *parm, SArray *pUids); -//int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids); -//int32_t metaFilterTtl(SMeta *pMeta, SMetaFltParam *param, SArray *pUids); #if 1 // refact APIs below (TODO) typedef SVCreateTbReq STbCfg; diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 3999aa0b7f..b39008147b 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -162,6 +162,12 @@ typedef struct { int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void* pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, STagIdxKey** ppTagIdxKey, int32_t* nTagIdxKey); +// TODO, refactor later +int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *param, SArray *results); +int32_t metaFilterCreateTime(void *pVnode, SMetaFltParam *parm, SArray *pUids); +int32_t metaFilterTableName(void *pVnode, SMetaFltParam *param, SArray *pUids); +int32_t metaFilterTtl(void *pVnode, SMetaFltParam *param, SArray *pUids); + #ifndef META_REFACT // SMetaDB int metaOpenDB(SMeta* pMeta); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 40f4b6fa67..3836b37002 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1012,8 +1012,8 @@ typedef struct { int32_t vLen; } SIdxCursor; -int32_t metaFilterCreateTime(void *meta, void *arg, SArray *pUids) { - SMeta *pMeta = meta; +int32_t metaFilterCreateTime(void *pVnode, SMetaFltParam *arg, SArray *pUids) { + SMeta *pMeta = ((SVnode*)pVnode)->pMeta; SMetaFltParam *param = arg; int32_t ret = 0; @@ -1072,8 +1072,8 @@ END: return ret; } -int32_t metaFilterTableName(void *meta, void *arg, SArray *pUids) { - SMeta *pMeta = meta; +int32_t metaFilterTableName(void *pVnode, SMetaFltParam *arg, SArray *pUids) { + SMeta *pMeta = ((SVnode*)pVnode)->pMeta; SMetaFltParam *param = arg; int32_t ret = 0; char *buf = NULL; @@ -1139,8 +1139,8 @@ END: return ret; } -int32_t metaFilterTtl(void *meta, void *arg, SArray *pUids) { - SMeta *pMeta = meta; +int32_t metaFilterTtl(void *pVnode, SMetaFltParam *arg, SArray *pUids) { + SMeta *pMeta = ((SVnode*)pVnode)->pMeta; SMetaFltParam *param = arg; int32_t ret = 0; char *buf = NULL; @@ -1170,8 +1170,8 @@ END: // impl later return 0; } -int32_t metaFilterTableIds(void *meta, void *arg, SArray *pUids) { - SMeta *pMeta = meta; +int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *arg, SArray *pUids) { + SMeta *pMeta = ((SVnode*)pVnode)->pMeta; SMetaFltParam *param = arg; SMetaEntry oStbEntry = {0}; diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 3f427d9965..7a26d7b837 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -16,12 +16,14 @@ #include "storageapi.h" #include "vnodeInt.h" #include "tstreamUpdate.h" +#include "meta.h" static void initTsdbReaderAPI(TsdReader* pReader); static void initMetadataAPI(SStoreMeta* pMeta); static void initTqAPI(SStoreTqReader* pTq); static void initStateStoreAPI(SStateStore* pStore); static void initMetaReaderAPI(SStoreMetaReader* pMetaReader); +static void initMetaFilterAPI(SMetaDataFilterAPI* pFilter); void initStorageAPI(SStorageAPI* pAPI) { initTsdbReaderAPI(&pAPI->tsdReader); @@ -29,6 +31,7 @@ void initStorageAPI(SStorageAPI* pAPI) { initTqAPI(&pAPI->tqReaderFn); initStateStoreAPI(&pAPI->stateStore); initMetaReaderAPI(&pAPI->metaReaderFn); + initMetaFilterAPI(&pAPI->metaFilter); } void initTsdbReaderAPI(TsdReader* pReader) { @@ -69,7 +72,6 @@ void initMetadataAPI(SStoreMeta* pMeta) { pMeta->getInvertIndex = vnodeGetIvtIdx; pMeta->extractTagVal = (const void *(*)(const void *, int16_t, STagVal *))metaGetTableTagVal; - } void initTqAPI(SStoreTqReader* pTq) { @@ -188,4 +190,11 @@ void initMetaReaderAPI(SStoreMetaReader* pMetaReader) { pMetaReader->getTableEntryByName = metaGetTableEntryByName; pMetaReader->readerReleaseLock = metaReaderReleaseLock; +} + +void initMetaFilterAPI(SMetaDataFilterAPI* pFilter) { + pFilter->metaFilterCreateTime = metaFilterCreateTime; + pFilter->metaFilterTableIds = metaFilterTableIds; + pFilter->metaFilterTableName = metaFilterTableName; + pFilter->metaFilterTtl = metaFilterTtl; } \ No newline at end of file diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 16f2b64c07..0d1926e21d 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -39,8 +39,9 @@ typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result); typedef int32_t (*__sys_check)(SNode* cond); typedef struct SSTabFltArg { - void* pMeta; - void* pVnode; + void* pMeta; + void* pVnode; + SStorageAPI* pAPI; } SSTabFltArg; typedef struct SSysTableIndex { @@ -153,11 +154,11 @@ static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t SFilterInfo* pFilterInfo); int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) { - void* pVnode = ((SSTabFltArg*)arg)->pVnode; + SSTabFltArg* pArg = arg; + void* pVnode = pArg->pVnode; const char* db = NULL; - ASSERT(0); -// pAPI->metaFn.getBasicInfo(pVnode, &db, NULL); + pArg->pAPI->metaFn.getBasicInfo(pVnode, &db, NULL, NULL, NULL); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -181,11 +182,11 @@ int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) { } int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) { - void* pVnode = ((SSTabFltArg*)arg)->pVnode; + SSTabFltArg* pArg = arg; + void* pVnode = ((SSTabFltArg*)arg)->pVnode; int64_t vgId = 0; - ASSERT(0); -// pAPI->metaFn.getBasicInfo(pVnode, NULL, (int32_t*)&vgId); + pArg->pAPI->metaFn.getBasicInfo(pVnode, NULL, (int32_t*)&vgId, NULL, NULL); SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; @@ -202,7 +203,7 @@ int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) { } int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; + SSTabFltArg* pArg = arg; SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; @@ -222,7 +223,8 @@ int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) { } int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; + SSTabFltArg* pArg = arg; + SStorageAPI* pAPI = pArg->pAPI; SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; @@ -239,10 +241,8 @@ int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) { .equal = equal, .filterFunc = func}; - ASSERT(0); - return 0; -// int32_t ret = metaFilterCreateTime(pMeta, ¶m, result); -// return ret; + int32_t ret = pAPI->metaFilter.metaFilterCreateTime(pArg->pVnode, ¶m, result); + return ret; } int32_t sysFilte__Ncolumn(void* arg, SNode* pNode, SArray* result) { diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index b90959579d..6be8b67264 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -67,9 +67,8 @@ typedef union { typedef struct SIFParam { SHashObj *pFilter; - - SArray *result; - char *condValue; + SArray *result; + char *condValue; SIdxFltStatus status; uint8_t colValType; @@ -79,7 +78,7 @@ typedef struct SIFParam { char colName[TSDB_COL_NAME_LEN * 2 + 4]; SIndexMetaArg arg; - SStoreAPI api; + SMetaDataFilterAPI api; } SIFParam; typedef struct SIFCtx { diff --git a/source/libs/qworker/CMakeLists.txt b/source/libs/qworker/CMakeLists.txt index a947a2dd19..8ba8b79ab8 100644 --- a/source/libs/qworker/CMakeLists.txt +++ b/source/libs/qworker/CMakeLists.txt @@ -9,11 +9,11 @@ target_include_directories( IF (TD_GRANT) TARGET_LINK_LIBRARIES(qworker - PRIVATE os util transport nodes planner qcom executor grant + PRIVATE os util transport nodes planner qcom executor index grant ) ELSE () TARGET_LINK_LIBRARIES(qworker - PRIVATE os util transport nodes planner qcom executor + PRIVATE os util transport nodes planner qcom executor index ) ENDIF() diff --git a/source/libs/scheduler/CMakeLists.txt b/source/libs/scheduler/CMakeLists.txt index 3288120b67..fafc2a27e0 100644 --- a/source/libs/scheduler/CMakeLists.txt +++ b/source/libs/scheduler/CMakeLists.txt @@ -9,7 +9,7 @@ target_include_directories( target_link_libraries( scheduler - PUBLIC os util nodes planner qcom common catalog transport command qworker executor + PUBLIC os util nodes planner qcom common catalog transport command qworker executor index ) if(${BUILD_TEST}) diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt index 2edbc44aae..89b82ef9ba 100644 --- a/source/libs/stream/CMakeLists.txt +++ b/source/libs/stream/CMakeLists.txt @@ -10,7 +10,7 @@ if(${BUILD_WITH_ROCKSDB}) target_link_libraries( stream PUBLIC rocksdb tdb - PRIVATE os util transport qcom executor wal + PRIVATE os util transport qcom executor wal index ) target_include_directories( From 61df1a1b211be03ad119c7f14861dc4ea15fc294 Mon Sep 17 00:00:00 2001 From: huolibo Date: Wed, 24 May 2023 17:23:31 +0800 Subject: [PATCH 38/97] enh: stmt insert demo --- .../com/taos/example/StmtInsertExample.java | 104 +++++++++++------- 1 file changed, 65 insertions(+), 39 deletions(-) diff --git a/docs/examples/java/src/main/java/com/taos/example/StmtInsertExample.java b/docs/examples/java/src/main/java/com/taos/example/StmtInsertExample.java index bbcc92b22f..72d4ecc725 100644 --- a/docs/examples/java/src/main/java/com/taos/example/StmtInsertExample.java +++ b/docs/examples/java/src/main/java/com/taos/example/StmtInsertExample.java @@ -6,39 +6,32 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import java.text.SimpleDateFormat; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Comparator; import java.util.List; +import java.util.Random; +import java.util.stream.Collectors; public class StmtInsertExample { - private static ArrayList tsToLongArray(String ts) { - ArrayList result = new ArrayList<>(); - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"); - LocalDateTime localDateTime = LocalDateTime.parse(ts, formatter); - result.add(localDateTime.toInstant(ZoneOffset.of("+8")).toEpochMilli()); - return result; - } + private static String datePattern = "yyyy-MM-dd HH:mm:ss.SSS"; + private static DateTimeFormatter formatter = DateTimeFormatter.ofPattern(datePattern); - private static ArrayList toArray(T v) { - ArrayList result = new ArrayList<>(); - result.add(v); - return result; - } - - private static List getRawData() { - return Arrays.asList( - "d1001,2018-10-03 14:38:05.000,10.30000,219,0.31000,California.SanFrancisco,2", - "d1001,2018-10-03 14:38:15.000,12.60000,218,0.33000,California.SanFrancisco,2", - "d1001,2018-10-03 14:38:16.800,12.30000,221,0.31000,California.SanFrancisco,2", - "d1002,2018-10-03 14:38:16.650,10.30000,218,0.25000,California.SanFrancisco,3", - "d1003,2018-10-03 14:38:05.500,11.80000,221,0.28000,California.LosAngeles,2", - "d1003,2018-10-03 14:38:16.600,13.40000,223,0.29000,California.LosAngeles,2", - "d1004,2018-10-03 14:38:05.000,10.80000,223,0.29000,California.LosAngeles,3", - "d1004,2018-10-03 14:38:06.500,11.50000,221,0.35000,California.LosAngeles,3" - ); + private static List getRawData(int size) { + SimpleDateFormat format = new SimpleDateFormat(datePattern); + List result = new ArrayList<>(); + long current = System.currentTimeMillis(); + Random random = new Random(); + for (int i = 0; i < size; i++) { + String time = format.format(current + i); + int id = random.nextInt(10); + result.add("d" + id + "," + time + ",10.30000,219,0.31000,California.SanFrancisco,2"); + } + return result.stream() + .sorted(Comparator.comparing(s -> s.split(",")[0])).collect(Collectors.toList()); } private static Connection getConnection() throws SQLException { @@ -48,9 +41,9 @@ public class StmtInsertExample { private static void createTable(Connection conn) throws SQLException { try (Statement stmt = conn.createStatement()) { - stmt.execute("CREATE DATABASE power KEEP 3650"); - stmt.executeUpdate("USE power"); - stmt.execute("CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " + + stmt.execute("CREATE DATABASE if not exists power KEEP 3650"); + stmt.executeUpdate("use power"); + stmt.execute("CREATE STABLE if not exists meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " + "TAGS (location BINARY(64), groupId INT)"); } } @@ -58,21 +51,54 @@ public class StmtInsertExample { private static void insertData() throws SQLException { try (Connection conn = getConnection()) { createTable(conn); - String psql = "INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)"; + String psql = "INSERT INTO ? USING power.meters TAGS(?, ?) VALUES(?, ?, ?, ?)"; try (TSDBPreparedStatement pst = (TSDBPreparedStatement) conn.prepareStatement(psql)) { - for (String line : getRawData()) { + String tableName = null; + ArrayList ts = new ArrayList<>(); + ArrayList current = new ArrayList<>(); + ArrayList voltage = new ArrayList<>(); + ArrayList phase = new ArrayList<>(); + for (String line : getRawData(100000)) { String[] ps = line.split(","); - // bind table name and tags - pst.setTableName(ps[0]); - pst.setTagString(0, ps[5]); - pst.setTagInt(1, Integer.valueOf(ps[6])); + if (tableName == null) { + // bind table name and tags + tableName = "power." + ps[0]; + pst.setTableName(ps[0]); + pst.setTagString(0, ps[5]); + pst.setTagInt(1, Integer.valueOf(ps[6])); + } else { + if (!tableName.equals(ps[0])) { + pst.setTimestamp(0, ts); + pst.setFloat(1, current); + pst.setInt(2, voltage); + pst.setFloat(3, phase); + pst.columnDataAddBatch(); + pst.columnDataExecuteBatch(); + + // bind table name and tags + tableName = ps[0]; + pst.setTableName(ps[0]); + pst.setTagString(0, ps[5]); + pst.setTagInt(1, Integer.valueOf(ps[6])); + ts.clear(); + current.clear(); + voltage.clear(); + phase.clear(); + } + } // bind values - pst.setTimestamp(0, tsToLongArray(ps[1])); //ps[1] looks like: 2018-10-03 14:38:05.000 - pst.setFloat(1, toArray(Float.valueOf(ps[2]))); - pst.setInt(2, toArray(Integer.valueOf(ps[3]))); - pst.setFloat(3, toArray(Float.valueOf(ps[4]))); - pst.columnDataAddBatch(); + // ps[1] looks like: 2018-10-03 14:38:05.000 + LocalDateTime localDateTime = LocalDateTime.parse(ps[1], formatter); + ts.add(localDateTime.toInstant(ZoneOffset.of("+8")).toEpochMilli()); + current.add(Float.valueOf(ps[2])); + voltage.add(Integer.valueOf(ps[3])); + phase.add(Float.valueOf(ps[4])); } + pst.setTimestamp(0, ts); + pst.setFloat(1, current); + pst.setInt(2, voltage); + pst.setFloat(3, phase); + pst.columnDataAddBatch(); pst.columnDataExecuteBatch(); } } From fc5d06681e3cc9c493036ddc696290cf02fd1c0e Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 24 May 2023 18:55:51 +0800 Subject: [PATCH 39/97] cache/last_row: treat none as null with last_row --- source/dnode/vnode/src/tsdb/tsdbCache.c | 88 ++++++++++++------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 71c74997bc..d33f04c145 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -404,16 +404,47 @@ int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow for (int i = 0; i < num_keys; ++i) { SColVal *pColVal = (SColVal *)taosArrayGet(aColVal, i); - if (!COL_VAL_IS_NONE(pColVal)) { - SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i + num_keys]); + // if (!COL_VAL_IS_NONE(pColVal)) { + SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i + num_keys]); + + if (NULL == pLastCol || pLastCol->ts <= keyTs) { + char *value = NULL; + size_t vlen = 0; + tsdbCacheSerialize(&(SLastCol){.ts = keyTs, .colVal = *pColVal}, &value, &vlen); + SLastKey key = (SLastKey){.ltype = 0, .uid = uid, .cid = pColVal->cid}; + size_t klen = ROCKS_KEY_LEN; + rocksdb_writebatch_put(wb, (char *)&key, klen, value, vlen); + + pLastCol = (SLastCol *)value; + SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol)); + *pTmpLastCol = *pLastCol; + pLastCol = pTmpLastCol; + + reallocVarData(&pLastCol->colVal); + size_t charge = sizeof(*pLastCol); + if (IS_VAR_DATA_TYPE(pLastCol->colVal.type)) { + charge += pLastCol->colVal.value.nData; + } + + LRUStatus status = taosLRUCacheInsert(pTsdb->lruCache, &key, ROCKS_KEY_LEN, pLastCol, charge, tsdbCacheDeleter, + NULL, TAOS_LRU_PRIORITY_LOW); + if (status != TAOS_LRU_STATUS_OK) { + code = -1; + } + + taosMemoryFree(value); + } + + if (COL_VAL_IS_VALUE(pColVal)) { + SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i]); if (NULL == pLastCol || pLastCol->ts <= keyTs) { char *value = NULL; size_t vlen = 0; tsdbCacheSerialize(&(SLastCol){.ts = keyTs, .colVal = *pColVal}, &value, &vlen); - SLastKey key = (SLastKey){.ltype = 0, .uid = uid, .cid = pColVal->cid}; - size_t klen = ROCKS_KEY_LEN; - rocksdb_writebatch_put(wb, (char *)&key, klen, value, vlen); + SLastKey key = (SLastKey){.ltype = 1, .uid = uid, .cid = pColVal->cid}; + + rocksdb_writebatch_put(wb, (char *)&key, ROCKS_KEY_LEN, value, vlen); pLastCol = (SLastCol *)value; SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol)); @@ -434,39 +465,8 @@ int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow taosMemoryFree(value); } - - if (COL_VAL_IS_VALUE(pColVal)) { - SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i]); - - if (NULL == pLastCol || pLastCol->ts <= keyTs) { - char *value = NULL; - size_t vlen = 0; - tsdbCacheSerialize(&(SLastCol){.ts = keyTs, .colVal = *pColVal}, &value, &vlen); - SLastKey key = (SLastKey){.ltype = 1, .uid = uid, .cid = pColVal->cid}; - - rocksdb_writebatch_put(wb, (char *)&key, ROCKS_KEY_LEN, value, vlen); - - pLastCol = (SLastCol *)value; - SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol)); - *pTmpLastCol = *pLastCol; - pLastCol = pTmpLastCol; - - reallocVarData(&pLastCol->colVal); - size_t charge = sizeof(*pLastCol); - if (IS_VAR_DATA_TYPE(pLastCol->colVal.type)) { - charge += pLastCol->colVal.value.nData; - } - - LRUStatus status = taosLRUCacheInsert(pTsdb->lruCache, &key, ROCKS_KEY_LEN, pLastCol, charge, - tsdbCacheDeleter, NULL, TAOS_LRU_PRIORITY_LOW); - if (status != TAOS_LRU_STATUS_OK) { - code = -1; - } - - taosMemoryFree(value); - } - } } + //} rocksdb_free(values_list[i]); rocksdb_free(values_list[i + num_keys]); @@ -474,7 +474,7 @@ int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow taosMemoryFree(values_list); taosMemoryFree(values_list_sizes); - rocksMayWrite(pTsdb, false, false, false); + rocksMayWrite(pTsdb, true, false, false); taosThreadMutexUnlock(&pTsdb->rCache.rMutex); _exit: @@ -3010,17 +3010,17 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData); } - if (COL_VAL_IS_NONE(pColVal)) { + /*if (COL_VAL_IS_NONE(pColVal)) { if (!setNoneCol) { noneCol = iCol; setNoneCol = true; } - } else { - int32_t aColIndex = taosArraySearchIdx(aColArray, &pColVal->cid, compareInt16Val, TD_EQ); - if (aColIndex >= 0) { - taosArrayRemove(aColArray, aColIndex); - } + } else {*/ + int32_t aColIndex = taosArraySearchIdx(aColArray, &pColVal->cid, compareInt16Val, TD_EQ); + if (aColIndex >= 0) { + taosArrayRemove(aColArray, aColIndex); } + //} } if (!setNoneCol) { // done, goto return pColArray From ab907e3344f74fa2bc66d0525b7defa1f7ceec21 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 May 2023 21:53:40 +0800 Subject: [PATCH 40/97] fix(query): fix syntax error. --- include/libs/executor/storageapi.h | 6 ++-- include/libs/index/index.h | 34 +++++++++++----------- source/dnode/vnode/inc/vnode.h | 16 ++-------- source/dnode/vnode/src/meta/metaQuery.c | 7 +++-- source/dnode/vnode/src/vnd/vnodeInitApi.c | 6 ++++ source/libs/executor/src/executil.c | 2 +- source/libs/executor/src/sysscanoperator.c | 2 +- source/libs/index/src/indexFilter.c | 21 ++++++++----- source/libs/qworker/test/CMakeLists.txt | 2 +- 9 files changed, 48 insertions(+), 48 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 97900de7ba..77576cf1af 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -354,13 +354,13 @@ typedef struct SStoreMetaReader { } SStoreMetaReader; typedef struct SStoreMeta { - SMTbCursor *(*openTableMetaCursor)(); // metaOpenTbCursor - void (*closeTableMetaCursor)(); // metaCloseTbCursor + SMTbCursor *(*openTableMetaCursor)(void *pVnode); // metaOpenTbCursor + void (*closeTableMetaCursor)(SMTbCursor *pTbCur); // metaCloseTbCursor int32_t (*cursorNext)(); // metaTbCursorNext int32_t (*cursorPrev)(); // metaTbCursorPrev int32_t (*getTableTags)(void *pVnode, uint64_t suid, SArray *uidList); - int32_t (*getTableTagsByUid)(); + int32_t (*getTableTagsByUid)(void *pVnode, int64_t suid, SArray *uidList); const void *(*extractTagVal)(const void *tag, int16_t type, STagVal *tagVal); // todo remove it int32_t (*getTableUidByName)(void *pVnode, char *tbName, uint64_t *uid); diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 162b99e553..cfcc9993cf 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -215,23 +215,6 @@ typedef struct SIndexMetaArg { int (*metaFilterFunc)(void* metaEx, void* param, SArray* result); } SIndexMetaArg; -typedef enum { SFLT_NOT_INDEX, SFLT_COARSE_INDEX, SFLT_ACCURATE_INDEX } SIdxFltStatus; - -SIdxFltStatus idxGetFltStatus(SNode* pFilterNode); - -int32_t doFilterTag(SNode* pFilterNode, SIndexMetaArg* metaArg, SArray* result, SIdxFltStatus* status); - -/* - * init index env - * - */ -void indexInit(int32_t threads); -/* - * destory index env - * - */ -void indexCleanup(); - /** * the underlying storage module must implement this API to employ the index functions. * @param pMeta @@ -256,6 +239,23 @@ typedef struct SMetaDataFilterAPI { int32_t (*metaFilterTtl)(void *pVnode, SMetaFltParam *arg, SArray *pUids); } SMetaDataFilterAPI; +typedef enum { SFLT_NOT_INDEX, SFLT_COARSE_INDEX, SFLT_ACCURATE_INDEX } SIdxFltStatus; + +SIdxFltStatus idxGetFltStatus(SNode* pFilterNode, SMetaDataFilterAPI* pAPI); + +int32_t doFilterTag(SNode* pFilterNode, SIndexMetaArg* metaArg, SArray* result, SIdxFltStatus* status, SMetaDataFilterAPI* pAPI); + +/* + * init index env + * + */ +void indexInit(int32_t threads); +/* + * destory index env + * + */ +void indexCleanup(); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 84153f3867..4e3e8a6a8d 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -111,8 +111,8 @@ void metaReaderClear(SMetaReader *pReader); int32_t metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); int32_t metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); int metaGetTableEntryByName(SMetaReader *pReader, const char *name); -int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList); -int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList); +int32_t metaGetTableTags(void *pVnode, uint64_t suid, SArray *uidList); +int32_t metaGetTableTagsByUids(void* pVnode, int64_t suid, SArray *uidList); int32_t metaReadNext(SMetaReader *pReader); const void *metaGetTableTagVal(const void *tag, int16_t type, STagVal *tagVal); int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); @@ -140,18 +140,6 @@ int64_t metaGetNtbNum(SMeta *pMeta); //} SMetaStbStats; int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables); -//typedef struct SMetaFltParam { -// tb_uid_t suid; -// int16_t cid; -// int16_t type; -// void *val; -// bool reverse; -// bool equal; -// int (*filterFunc)(void *a, void *b, int16_t type); -// -//} SMetaFltParam; - - #if 1 // refact APIs below (TODO) typedef SVCreateTbReq STbCfg; typedef SVCreateTSmaReq SSmaCfg; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 3836b37002..617a3a2c08 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1356,7 +1356,8 @@ static int32_t metaGetTableTagByUid(SMeta *pMeta, int64_t suid, int64_t uid, voi return ret; } -int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList) { +int32_t metaGetTableTagsByUids(void *pVnode, int64_t suid, SArray *uidList) { + SMeta* pMeta = ((SVnode*) pVnode)->pMeta; const int32_t LIMIT = 128; int32_t isLock = false; @@ -1388,8 +1389,8 @@ int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList) { return 0; } -int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *pUidTagInfo) { - SMCtbCursor *pCur = metaOpenCtbCursor(pMeta, suid, 1); +int32_t metaGetTableTags(void* pVnode, uint64_t suid, SArray *pUidTagInfo) { + SMCtbCursor *pCur = metaOpenCtbCursor(((SVnode*)pVnode)->pMeta, suid, 1); // If len > 0 means there already have uids, and we only want the // tags of the specified tables, of which uid in the uid list. Otherwise, all table tags are retrieved and kept diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 7a26d7b837..34c6da1398 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -72,6 +72,12 @@ void initMetadataAPI(SStoreMeta* pMeta) { pMeta->getInvertIndex = vnodeGetIvtIdx; pMeta->extractTagVal = (const void *(*)(const void *, int16_t, STagVal *))metaGetTableTagVal; + pMeta->getTableTags = metaGetTableTags; + pMeta->getTableTagsByUid = metaGetTableTagsByUids; + + pMeta->getTableUidByName = metaGetTableUidByName; + pMeta->getTableTypeByName = metaGetTableTypeByName; + pMeta->getTableNameByUid = metaGetTableNameByUid; } void initTqAPI(SStoreTqReader* pTq) { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index a52e73eb49..47b08b3323 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1100,7 +1100,7 @@ int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, S .metaEx = pVnode, .idx = pStorageAPI->metaFn.storeGetIndexInfo(pVnode), .ivtIdx = pIndex, .suid = pScanNode->uid}; status = SFLT_NOT_INDEX; - code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status); + code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status, &pStorageAPI->metaFilter); if (code != 0 || status == SFLT_NOT_INDEX) { // temporarily disable it for performance sake qWarn("failed to get tableIds from index, suid:%" PRIu64, pScanNode->uid); code = TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 0d1926e21d..8f705a7eeb 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1485,7 +1485,7 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { } else { if (pInfo->showRewrite == false) { if (pCondition != NULL && pInfo->pIdx == NULL) { - SSTabFltArg arg = {.pMeta = pInfo->readHandle.vnode, .pVnode = pInfo->readHandle.vnode}; + SSTabFltArg arg = {.pMeta = pInfo->readHandle.vnode, .pVnode = pInfo->readHandle.vnode, .pAPI = &pTaskInfo->storageAPI}; SSysTableIndex* idx = taosMemoryMalloc(sizeof(SSysTableIndex)); idx->init = 0; diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 6be8b67264..2c12c84081 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -86,6 +86,7 @@ typedef struct SIFCtx { SHashObj *pRes; /* element is SIFParam */ bool noExec; // true: just iterate condition tree, and add hint to executor plan SIndexMetaArg arg; + SMetaDataFilterAPI *pAPI; } SIFCtx; static FORCE_INLINE int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { @@ -288,6 +289,8 @@ static int32_t sifInitParamValByCol(SNode *r, SNode *l, SIFParam *param, SIFCtx } static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { param->status = SFLT_COARSE_INDEX; + param->api = *ctx->pAPI; + switch (nodeType(node)) { case QUERY_NODE_VALUE: { SValueNode *vn = (SValueNode *)node; @@ -364,6 +367,7 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } } + SIFParam *paramList = taosMemoryCalloc(nParam, sizeof(SIFParam)); if (NULL == paramList) { @@ -972,8 +976,9 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { if (pNode == NULL || pDst == NULL) { return TSDB_CODE_QRY_INVALID_INPUT; } + int32_t code = 0; - SIFCtx ctx = {.code = 0, .noExec = false, .arg = pDst->arg}; + SIFCtx ctx = {.code = 0, .noExec = false, .arg = pDst->arg, .pAPI = &pDst->api}; ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { @@ -1006,13 +1011,13 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { return code; } -static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) { +static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status, SMetaDataFilterAPI* pAPI) { int32_t code = TSDB_CODE_SUCCESS; if (pNode == NULL) { return TSDB_CODE_QRY_INVALID_INPUT; } - SIFCtx ctx = {.code = 0, .noExec = true}; + SIFCtx ctx = {.code = 0, .noExec = true, .pAPI = pAPI}; ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { indexError("index-filter failed to taosHashInit"); @@ -1044,8 +1049,8 @@ static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) { return code; } -int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result, SIdxFltStatus *status) { - SIdxFltStatus st = idxGetFltStatus(pFilterNode); +int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result, SIdxFltStatus *status, SMetaDataFilterAPI* pAPI) { + SIdxFltStatus st = idxGetFltStatus(pFilterNode, pAPI); if (st == SFLT_NOT_INDEX) { *status = st; return 0; @@ -1054,7 +1059,7 @@ int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result, SFilterInfo *filter = NULL; SArray *output = taosArrayInit(8, sizeof(uint64_t)); - SIFParam param = {.arg = *metaArg, .result = output, .status = SFLT_NOT_INDEX}; + SIFParam param = {.arg = *metaArg, .result = output, .status = SFLT_NOT_INDEX, .api = *pAPI}; int32_t code = sifCalculate((SNode *)pFilterNode, ¶m); if (code != 0) { sifFreeParam(¶m); @@ -1071,13 +1076,13 @@ int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result, return TSDB_CODE_SUCCESS; } -SIdxFltStatus idxGetFltStatus(SNode *pFilterNode) { +SIdxFltStatus idxGetFltStatus(SNode *pFilterNode, SMetaDataFilterAPI* pAPI) { SIdxFltStatus st = SFLT_NOT_INDEX; if (pFilterNode == NULL) { return SFLT_NOT_INDEX; } - if (sifGetFltHint((SNode *)pFilterNode, &st) != TSDB_CODE_SUCCESS) { + if (sifGetFltHint((SNode *)pFilterNode, &st, pAPI) != TSDB_CODE_SUCCESS) { st = SFLT_NOT_INDEX; } return st; diff --git a/source/libs/qworker/test/CMakeLists.txt b/source/libs/qworker/test/CMakeLists.txt index 780f5ae84b..22870ea94d 100644 --- a/source/libs/qworker/test/CMakeLists.txt +++ b/source/libs/qworker/test/CMakeLists.txt @@ -8,7 +8,7 @@ IF(NOT TD_DARWIN) ADD_EXECUTABLE(qworkerTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( qworkerTest - PUBLIC os util common transport gtest qcom nodes planner qworker executor + PUBLIC os util common transport gtest qcom nodes planner qworker executor index ) TARGET_INCLUDE_DIRECTORIES( From 01f6f74fb929429565bbadca51d49d2ffa9c3f50 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 May 2023 22:27:45 +0800 Subject: [PATCH 41/97] fix: update the cmake file. --- source/libs/stream/test/CMakeLists.txt | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/source/libs/stream/test/CMakeLists.txt b/source/libs/stream/test/CMakeLists.txt index 75acec7f01..160cddbd4c 100644 --- a/source/libs/stream/test/CMakeLists.txt +++ b/source/libs/stream/test/CMakeLists.txt @@ -8,10 +8,20 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) # bloomFilterTest ADD_EXECUTABLE(streamUpdateTest "tstreamUpdateTest.cpp") -TARGET_LINK_LIBRARIES( - streamUpdateTest - PUBLIC os util common gtest gtest_main stream executor -) +#TARGET_LINK_LIBRARIES( +# streamUpdateTest +# PUBLIC os util common gtest gtest_main stream executor +#) + +IF (TD_GRANT) + TARGET_LINK_LIBRARIES(streamUpdateTest + PUBLIC os util common gtest gtest_main stream executor grant + ) +ELSE () + TARGET_LINK_LIBRARIES(streamUpdateTest + PUBLIC os util common gtest gtest_main stream executor + ) +ENDIF() TARGET_INCLUDE_DIRECTORIES( streamUpdateTest From a9b18d203daa2299aaa546723d86606f9f117ccc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 May 2023 23:49:26 +0800 Subject: [PATCH 42/97] fix:fix error. --- include/libs/executor/dataSinkMgt.h | 2 +- source/dnode/vnode/inc/vnode.h | 4 ++-- source/dnode/vnode/src/meta/metaQuery.c | 12 ++++++------ source/dnode/vnode/src/tsdb/tsdbRead.c | 6 +++--- source/dnode/vnode/src/vnd/vnodeInitApi.c | 2 ++ source/libs/executor/inc/dataSinkInt.h | 4 ++-- source/libs/executor/inc/executorInt.h | 2 +- source/libs/executor/src/dataInserter.c | 2 +- source/libs/executor/src/dataSinkMgt.c | 3 ++- source/libs/executor/src/executor.c | 2 +- source/libs/executor/src/scanoperator.c | 9 +++++---- 11 files changed, 26 insertions(+), 22 deletions(-) diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index ce7d038d42..0a9037d21c 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -59,7 +59,7 @@ typedef struct SDataSinkMgtCfg { uint32_t maxDataBlockNumPerQuery; } SDataSinkMgtCfg; -int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg); +int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg, SStorageAPI* pAPI); typedef struct SInputData { const struct SSDataBlock* pData; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 4e3e8a6a8d..2723e869ff 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -118,7 +118,7 @@ const void *metaGetTableTagVal(const void *tag, int16_t type, STagVal *tagVal); int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName); -int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid); +int metaGetTableUidByName(void *pVnode, char *tbName, uint64_t *uid); int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); bool metaIsTableExist(void* pVnode, tb_uid_t uid); int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList, @@ -192,7 +192,7 @@ int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, SArray *pTableUids); void *tsdbCacherowsReaderClose(void *pReader); -int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); +int32_t tsdbGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); size_t tsdbCacheGetCapacity(SVnode *pVnode); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 617a3a2c08..255fa10f57 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -139,10 +139,10 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) { return uid; } -int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName) { +int metaGetTableNameByUid(void *pVnode, uint64_t uid, char *tbName) { int code = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, (SMeta *)meta, 0); + metaReaderInit(&mr, ((SVnode*)pVnode)->pMeta, 0); code = metaReaderGetTableEntryByUid(&mr, uid); if (code < 0) { metaReaderClear(&mr); @@ -170,10 +170,10 @@ int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName) { return 0; } -int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) { +int metaGetTableUidByName(void *pVnode, char *tbName, uint64_t *uid) { int code = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, (SMeta *)meta, 0); + metaReaderInit(&mr, ((SVnode *)pVnode)->pMeta, 0); SMetaReader *pReader = &mr; @@ -191,10 +191,10 @@ int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) { return 0; } -int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType) { +int metaGetTableTypeByName(void *pVnode, char *tbName, ETableType *tbType) { int code = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, (SMeta *)meta, 0); + metaReaderInit(&mr, ((SVnode*)pVnode)->pMeta, 0); code = metaGetTableEntryByName(&mr, tbName); if (code == 0) *tbType = mr.me.type; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 414ccfc8b2..62b121e4d7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -5373,9 +5373,9 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { return rows; } -int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) { +int32_t tsdbGetTableSchema(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) { SMetaReader mr = {0}; - metaReaderInit(&mr, pVnode->pMeta, 0); + metaReaderInit(&mr, ((SVnode*)pVnode)->pMeta, 0); int32_t code = metaReaderGetTableEntryByUidCache(&mr, uid); if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; @@ -5405,7 +5405,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 metaReaderClear(&mr); // get the newest table schema version - code = metaGetTbTSchemaEx(pVnode->pMeta, *suid, uid, -1, pSchema); + code = metaGetTbTSchemaEx(((SVnode*)pVnode)->pMeta, *suid, uid, -1, pSchema); return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 34c6da1398..70cff5e68a 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -78,6 +78,8 @@ void initMetadataAPI(SStoreMeta* pMeta) { pMeta->getTableUidByName = metaGetTableUidByName; pMeta->getTableTypeByName = metaGetTableTypeByName; pMeta->getTableNameByUid = metaGetTableNameByUid; + + pMeta->getTableSchema = tsdbGetTableSchema; // todo refactor } void initTqAPI(SStoreTqReader* pTq) { diff --git a/source/libs/executor/inc/dataSinkInt.h b/source/libs/executor/inc/dataSinkInt.h index 3255831476..9893b4eb76 100644 --- a/source/libs/executor/inc/dataSinkInt.h +++ b/source/libs/executor/inc/dataSinkInt.h @@ -29,8 +29,8 @@ struct SDataSink; struct SDataSinkHandle; typedef struct SDataSinkManager { - SDataSinkMgtCfg cfg; - SStorageAPI storeFn; + SDataSinkMgtCfg cfg; + SStorageAPI* pAPI; } SDataSinkManager; typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue); diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index eb585e7046..8188ecd305 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -224,7 +224,7 @@ typedef struct STableScanInfo { int8_t assignBlockUid; bool hasGroupByTag; bool countOnly; - TsdReader readerAPI; +// TsdReader readerAPI; } STableScanInfo; typedef struct STableMergeScanInfo { diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index aadbf1f1a9..646964ebf4 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -429,7 +429,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat inserter->explain = pInserterNode->explain; int64_t suid = 0; - int32_t code = pManager->storeFn.metaFn.getTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid); + int32_t code = pManager->pAPI->metaFn.getTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid); if (code) { destroyDataSinker((SDataSinkHandle*)inserter); taosMemoryFree(inserter); diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index b3cb57325b..3a972c1c20 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -21,8 +21,9 @@ static SDataSinkManager gDataSinkManager = {0}; SDataSinkStat gDataSinkStat = {0}; -int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg) { +int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg, SStorageAPI* pAPI) { gDataSinkManager.cfg = *cfg; + gDataSinkManager.pAPI = pAPI; return 0; // to avoid compiler eror } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 78a015269a..53fad6503e 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -507,7 +507,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, } SDataSinkMgtCfg cfg = {.maxDataBlockNum = 500, .maxDataBlockNumPerQuery = 50}; - code = dsDataSinkMgtInit(&cfg); + code = dsDataSinkMgtInit(&cfg, &(*pTask)->storageAPI); if (code != TSDB_CODE_SUCCESS) { qError("failed to dsDataSinkMgtInit, code:%s, %s", tstrerror(code), (*pTask)->id.str); goto _error; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 83e62faf90..4bd8faf8e9 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -929,7 +929,7 @@ static void destroyTableScanOperatorInfo(void* param) { STableScanInfo* pTableScanInfo = (STableScanInfo*)param; blockDataDestroy(pTableScanInfo->pResBlock); taosHashCleanup(pTableScanInfo->pIgnoreTables); - destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->readerAPI); + destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); taosMemoryFreeClear(param); } @@ -975,7 +975,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->sample.sampleRatio = pTableScanNode->ratio; pInfo->sample.seed = taosGetTimestampSec(); - pInfo->readerAPI = pTaskInfo->storageAPI.tsdReader; + pInfo->base.readerAPI = pTaskInfo->storageAPI.tsdReader; initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pResBlock = createDataBlockFromDescNode(pDescNode); // blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity); @@ -1075,7 +1075,7 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint6 pTableScanInfo->base.cond.endVersion = version; pTableScanInfo->scanTimes = 0; pTableScanInfo->currentGroupId = -1; - pTableScanInfo->readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); + pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; } @@ -1250,7 +1250,7 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 *pRowIndex = 0; pInfo->updateWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - pTableScanInfo->readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); + pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; return NULL; } @@ -3045,6 +3045,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN goto _error; } + pInfo->base.readerAPI = pTaskInfo->storageAPI.tsdReader; pInfo->base.dataBlockLoadFlag = FUNC_DATA_REQUIRED_DATA_LOAD; pInfo->base.scanFlag = MAIN_SCAN; pInfo->base.readHandle = *readHandle; From 4a42f6b983aeb26883ab863f0b8b19e53b2bcbd9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 25 May 2023 00:50:03 +0800 Subject: [PATCH 43/97] fix: fix error in api call. --- include/libs/executor/storageapi.h | 2 +- source/dnode/mnode/impl/src/mndQuery.c | 1 + source/dnode/vnode/src/sma/smaRollup.c | 4 ++++ source/dnode/vnode/src/tq/tq.c | 2 ++ source/dnode/vnode/src/vnd/vnodeInitApi.c | 1 + source/dnode/vnode/src/vnd/vnodeSvr.c | 2 ++ source/libs/executor/src/sysscanoperator.c | 8 ++++++-- source/libs/executor/src/timewindowoperator.c | 8 +++++--- 8 files changed, 22 insertions(+), 6 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 77576cf1af..0fd8a74127 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -396,7 +396,7 @@ int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, in int32_t (*getNumOfChildTables)(void* pVnode, int64_t uid, int64_t* numOfTables); // int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); void (*getBasicInfo)(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables);// vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) & metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta); - int64_t (*getNumOfRowsInMem)(); + int64_t (*getNumOfRowsInMem)(void* pVnode); /** int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list); int32_t vnodeGetCtbIdListByFilter(void *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg); diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 5278fc7761..8e95fa3d6d 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -33,6 +33,7 @@ void mndPostProcessQueryMsg(SRpcMsg *pMsg) { int32_t mndProcessQueryMsg(SRpcMsg *pMsg) { int32_t code = -1; SMnode *pMnode = pMsg->info.node; + SReadHandle handle = {.mnd = pMnode, .pMsgCb = &pMnode->msgCb}; mTrace("msg:%p, in query queue is processing", pMsg); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 558cfdac2b..39aa5c3043 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -277,6 +277,8 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat } SReadHandle handle = { .vnode = pVnode, .initTqReader = 1, .pStateBackend = pStreamState }; + initStorageAPI(&handle.api); + pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle, TD_VID(pVnode)); if (!pRSmaInfo->taskInfo[idx]) { terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE; @@ -849,6 +851,8 @@ static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t TSDB_CHECK_CODE(code, lino, _exit); SReadHandle handle = { .vnode = pVnode, .initTqReader = 1 }; + initStorageAPI(&handle.api); + if (ASSERTS(!dstTaskInfo, "dstTaskInfo:%p is not NULL", dstTaskInfo)) { code = TSDB_CODE_APP_ERROR; TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 4adbf02302..362b8204fc 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -671,6 +671,8 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg pHandle->pRef = pRef; SReadHandle handle = {.vnode = pVnode, .initTableReader = true, .initTqReader = true, .version = ver}; + initStorageAPI(&handle.api); + pHandle->snapshotVer = ver; if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 70cff5e68a..d9bb4270d8 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -65,6 +65,7 @@ void initMetadataAPI(SStoreMeta* pMeta) { pMeta->getBasicInfo = vnodeGetInfo; pMeta->getNumOfChildTables = metaGetStbStats; +// pMeta->getNumOfRowsInMem = tsdbGetNumOfRowsInMemTable; pMeta->getChildTableList = vnodeGetCtbIdList; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index d16102037d..b950437f23 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -250,7 +250,9 @@ static int32_t vnodePreProcessDeleteMsg(SVnode *pVnode, SRpcMsg *pMsg) { uint8_t *pCont; SEncoder *pCoder = &(SEncoder){0}; SDeleteRes res = {0}; + SReadHandle handle = {.config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; + initStorageAPI(&handle.api); code = qWorkerProcessDeleteMsg(&handle, pVnode->pQuery, pMsg, &res); if (code) goto _exit; diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 8f705a7eeb..d15b6d5754 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1814,9 +1814,13 @@ void destroySysScanOperator(void* param) { if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { - pInfo->pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); + if (pInfo->pAPI->metaFn.closeTableMetaCursor != NULL) { + pInfo->pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); + } + pInfo->pCur = NULL; } + if (pInfo->pIdx) { taosArrayDestroy(pInfo->pIdx->uids); taosMemoryFree(pInfo->pIdx); @@ -2200,7 +2204,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { } pAPI->tsdReader.tsdReaderGetDataBlockDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); - blockDistInfo.numOfInmemRows = (int32_t) pAPI->metaFn.getNumOfRowsInMem(pBlockScanInfo->pHandle); + blockDistInfo.numOfInmemRows = (int32_t) pAPI->tsdReader.tsdReaderGetNumOfInMemRows(pBlockScanInfo->pHandle); SSDataBlock* pBlock = pBlockScanInfo->pResBlock; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 2d366b6110..cf58892920 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2906,7 +2906,7 @@ void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uin } int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, int64_t gap, - SStreamState* pState, int32_t keySize, int16_t keyType) { + SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore) { pSup->resultRowSize = keySize + getResultRowSize(pCtx, numOfOutput); pSup->pScanBlock = createSpecialDataBlock(STREAM_CLEAR); pSup->gap = gap; @@ -2939,6 +2939,8 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, qError("Init stream agg supporter failed since %s, tempDir:%s", terrstr(), tsTempDir); return terrno; } + + pSup->stateStore = *pStore; int32_t code = createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, "function", tsTempDir); for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].saveHandle.pBuf = pSup->pResultBuf; @@ -3600,7 +3602,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh } code = initStreamAggSupporter(&pInfo->streamAggSup, pSup->pCtx, numOfCols, pSessionNode->gap, - pTaskInfo->streamInfo.pState, 0, 0); + pTaskInfo->streamInfo.pState, 0, 0, &pTaskInfo->storageAPI.stateStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4149,7 +4151,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys int32_t keySize = sizeof(SStateKeys) + pColNode->node.resType.bytes; int16_t type = pColNode->node.resType.type; code = initStreamAggSupporter(&pInfo->streamAggSup, pSup->pCtx, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize, - type); + type, &pTaskInfo->storageAPI.stateStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } From 1f816423033ef97746e75ae0d55f5c78994052fa Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Thu, 25 May 2023 09:02:41 +0800 Subject: [PATCH 44/97] fix sma sliding window --- source/common/src/tdatablock.c | 4 ++-- source/libs/executor/src/timewindowoperator.c | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 4a872482b4..311c79381c 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1970,7 +1970,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) if (len >= size - 1) return dumpBuf; for (int32_t j = 0; j < rows; j++) { - len += snprintf(dumpBuf + len, size - len, "%s %d|", flag, j); + len += snprintf(dumpBuf + len, size - len, "%s|", flag); if (len >= size - 1) return dumpBuf; for (int32_t k = 0; k < colNum; k++) { @@ -2053,7 +2053,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) } break; } } - len += snprintf(dumpBuf + len, size - len, "\n"); + len += snprintf(dumpBuf + len, size - len, "%d\n", j); if (len >= size - 1) return dumpBuf; } len += snprintf(dumpBuf + len, size - len, "%s |end\n", flag); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 903975a618..9d882a90ce 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -448,15 +448,17 @@ static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SEx return true; } -bool inCalSlidingWindow(SInterval* pInterval, STimeWindow* pWin, TSKEY calStart, TSKEY calEnd) { - if (pInterval->interval != pInterval->sliding && (pWin->ekey < calStart || pWin->skey > calEnd)) { +bool inCalSlidingWindow(SInterval* pInterval, STimeWindow* pWin, TSKEY calStart, TSKEY calEnd, EStreamType blockType) { + if (pInterval->interval != pInterval->sliding && + ((pWin->ekey < calStart || pWin->skey > calEnd) || (blockType == STREAM_PULL_DATA && pWin->skey < calStart) )) { return false; } + return true; } bool inSlidingWindow(SInterval* pInterval, STimeWindow* pWin, SDataBlockInfo* pBlockInfo) { - return inCalSlidingWindow(pInterval, pWin, pBlockInfo->calWin.skey, pBlockInfo->calWin.ekey); + return inCalSlidingWindow(pInterval, pWin, pBlockInfo->calWin.skey, pBlockInfo->calWin.ekey, pBlockInfo->type); } static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, @@ -1365,7 +1367,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa } do { - if (!inCalSlidingWindow(pInterval, &win, calStTsCols[i], calEnTsCols[i])) { + if (!inCalSlidingWindow(pInterval, &win, calStTsCols[i], calEnTsCols[i], pBlock->info.type)) { getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); continue; } From da9f33a483ce658098bbc142f223d46166d1a272 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 25 May 2023 09:56:34 +0800 Subject: [PATCH 45/97] fix: fix error in set function ptr. --- source/dnode/snode/src/snode.c | 4 +- source/dnode/vnode/src/tq/tq.c | 5 ++- source/libs/executor/src/filloperator.c | 7 ++- source/libs/executor/src/scanoperator.c | 43 +++++++++---------- source/libs/executor/src/timewindowoperator.c | 7 +-- 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index e9225f3d6e..24e2b13f46 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -87,9 +87,9 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { } int32_t numOfChildEp = taosArrayGetSize(pTask->childEpInfo); - SReadHandle mgHandle = { .vnode = NULL, .numOfVgroups = numOfChildEp, .pStateBackend = pTask->pState }; + SReadHandle handle = { .vnode = NULL, .numOfVgroups = numOfChildEp, .pStateBackend = pTask->pState }; - pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, 0); + pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, 0); ASSERT(pTask->exec.pExecutor); streamSetupTrigger(pTask); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 362b8204fc..d98fbab192 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -804,9 +804,10 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { } int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo); - SReadHandle mgHandle = {.vnode = NULL, .numOfVgroups = numOfVgroups, .pStateBackend = pTask->pState}; + SReadHandle handle = {.vnode = NULL, .numOfVgroups = numOfVgroups, .pStateBackend = pTask->pState}; + initStorageAPI(&handle.api); - pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, vgId); + pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId); if (pTask->exec.pExecutor == NULL) { return -1; } diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 0b89dbc80b..b448e8f2b0 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -443,7 +443,6 @@ void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols); tSimpleHashCleanup(pFillSup->pResMap); pFillSup->pResMap = NULL; - ASSERT(0); releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal, &pFillSup->pAPI->stateStore); //????? pFillSup->cur.pRowVal = NULL; cleanupExprSupp(&pFillSup->notFillExprSup); @@ -493,7 +492,6 @@ static void resetFillWindow(SResultRowData* pRowData) { void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, void* pState, SStorageAPI* pAPI) { resetFillWindow(&pFillSup->prev); - ASSERT(0); releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal, &pAPI->stateStore); //??? resetFillWindow(&pFillSup->cur); resetFillWindow(&pFillSup->next); @@ -1335,7 +1333,7 @@ static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { } static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNode, SInterval* pInterval, - SExprInfo* pFillExprInfo, int32_t numOfFillCols) { + SExprInfo* pFillExprInfo, int32_t numOfFillCols, SStorageAPI* pAPI) { SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter)); if (!pFillSup) { return NULL; @@ -1348,6 +1346,7 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod pFillSup->type = convertFillType(pPhyFillNode->mode); pFillSup->numOfAllCols = pFillSup->numOfFillCols + numOfNotFillCols; pFillSup->interval = *pInterval; + pFillSup->pAPI = pAPI; int32_t code = initResultBuf(pFillSup); if (code != TSDB_CODE_SUCCESS) { @@ -1427,7 +1426,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval; int32_t numOfFillCols = 0; SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols); - pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols); + pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI); if (!pInfo->pFillSup) { goto _error; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 4bd8faf8e9..39d619172b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1070,9 +1070,9 @@ static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t grou pInfo->groupId = groupCol[rowIndex]; } -void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint64_t version) { +void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint64_t ver) { pTableScanInfo->base.cond.twindows = *pWin; - pTableScanInfo->base.cond.endVersion = version; + pTableScanInfo->base.cond.endVersion = ver; pTableScanInfo->scanTimes = 0; pTableScanInfo->currentGroupId = -1; pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); @@ -1182,17 +1182,19 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ win.ekey = TMAX(win.ekey, endData[*pRowIndex]); continue; } + if (win.skey == endData[*pRowIndex] && groupId == gpData[*pRowIndex]) { win.skey = TMIN(win.skey, startData[*pRowIndex]); continue; } + ASSERT(!(win.skey > startData[*pRowIndex] && win.ekey < endData[*pRowIndex]) || !(isInTimeWindow(&win, startData[*pRowIndex], 0) || isInTimeWindow(&win, endData[*pRowIndex], 0))); break; } - ASSERT(0); -// resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion); + STableScanInfo* pTScanInfo = pInfo->pTableScanOp->info; + resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion); pInfo->pTableScanOp->status = OP_OPENED; return true; } @@ -1650,13 +1652,13 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); if (pResult && pResult->info.rows > 0) { - ASSERT(0); // qDebug("queue scan tsdb return %" PRId64 " rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, // pResult->info.rows, pResult->info.window.skey, pResult->info.window.ekey, // pInfo->tqReader->pWalReader->curVersion); tqOffsetResetToData(&pTaskInfo->streamInfo.currentOffset, pResult->info.id.uid, pResult->info.window.ekey); return pResult; } + STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader); @@ -1673,7 +1675,6 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { while (1) { bool hasResult = pAPI->tqReaderFn.tqReaderNextBlockInWal(pInfo->tqReader, id); - ASSERT(0); SSDataBlock* pRes = NULL; struct SWalReader* pWalReader = pAPI->tqReaderFn.tqReaderGetWalReader(pInfo->tqReader); @@ -1757,8 +1758,7 @@ static void setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBlock) { if (!pInfo->igCheckUpdate && pInfo->pUpdateInfo) { - ASSERT(0); -// pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); + pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); checkUpdateData(pInfo, true, pBlock, true); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, endKey); if (pInfo->pUpdateDataRes->info.rows > 0) { @@ -1934,8 +1934,7 @@ FETCH_NEXT_BLOCK: pBlock->info.calWin.ekey = INT64_MAX; pBlock->info.dataLoad = 1; if (pInfo->pUpdateInfo) { - ASSERT(0); -// pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); + pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); } blockDataUpdateTsWindow(pBlock, 0); switch (pBlock->info.type) { @@ -3097,9 +3096,9 @@ _error: static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator); static void destoryTableCountScanOperator(void* param); static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, - SSDataBlock* pRes, char* dbName, tb_uid_t stbUid); + SSDataBlock* pRes, char* dbName, tb_uid_t stbUid, SStorageAPI* pAPI); static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, - SSDataBlock* pRes, char* dbName); + SSDataBlock* pRes, char* dbName, SStorageAPI* pAPI); static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName); static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, @@ -3385,11 +3384,11 @@ static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountSca } if (pInfo->currGrpIdx < taosArrayGetSize(pInfo->stbUidList)) { tb_uid_t stbUid = *(tb_uid_t*)taosArrayGet(pInfo->stbUidList, pInfo->currGrpIdx); - buildVnodeGroupedStbTableCount(pInfo, pSupp, pRes, dbName, stbUid); + buildVnodeGroupedStbTableCount(pInfo, pSupp, pRes, dbName, stbUid, pAPI); pInfo->currGrpIdx++; } else if (pInfo->currGrpIdx == taosArrayGetSize(pInfo->stbUidList)) { - buildVnodeGroupedNtbTableCount(pInfo, pSupp, pRes, dbName); + buildVnodeGroupedNtbTableCount(pInfo, pSupp, pRes, dbName, pAPI); pInfo->currGrpIdx++; } else { @@ -3434,7 +3433,7 @@ static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanO } static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, - SSDataBlock* pRes, char* dbName) { + SSDataBlock* pRes, char* dbName, SStorageAPI* pAPI) { char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; if (pSupp->groupByDbName) { snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, ""); @@ -3442,18 +3441,18 @@ static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, S uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); pRes->info.id.groupId = groupId; - int64_t ntbNum = 0;//metaGetNtbNum(pInfo->readHandle.vnode); - ASSERT(0); - if (ntbNum != 0) { - fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes); + + int64_t numOfTables = 0;//metaGetNtbNum(pInfo->readHandle.vnode); + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, NULL, &numOfTables); + if (numOfTables != 0) { + fillTableCountScanDataBlock(pSupp, dbName, "", numOfTables, pRes); } } static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, - SSDataBlock* pRes, char* dbName, tb_uid_t stbUid) { + SSDataBlock* pRes, char* dbName, tb_uid_t stbUid, SStorageAPI* pAPI) { char stbName[TSDB_TABLE_NAME_LEN] = {0}; - ASSERT(0); -// metaGetTableSzNameByUid(pInfo->readHandle.vnode, stbUid, stbName); + pAPI->metaFn.getTableNameByUid(pInfo->readHandle.vnode, stbUid, stbName); char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; if (pSupp->groupByDbName) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index cf58892920..290bc43cbe 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2755,7 +2755,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); initBasicInfo(&pInfo->binfo, pResBlock); - pInfo->pState = taosMemoryCalloc(1, sizeof(void)); + pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); *(pInfo->pState) = *(pTaskInfo->streamInfo.pState); pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1); int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, @@ -2917,8 +2917,10 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, return TSDB_CODE_OUT_OF_MEMORY; } + pSup->stateStore = *pStore; + initDummyFunction(pSup->pDummyCtx, pCtx, numOfOutput); - pSup->pState = taosMemoryCalloc(1, sizeof(void)); + pSup->pState = taosMemoryCalloc(1, sizeof(SStreamState)); *(pSup->pState) = *pState; pSup->stateStore.streamStateSetNumber(pSup->pState, -1); @@ -2940,7 +2942,6 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, return terrno; } - pSup->stateStore = *pStore; int32_t code = createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, "function", tsTempDir); for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].saveHandle.pBuf = pSup->pResultBuf; From 90f542857d6e7aa919384c1742259b4365c093d9 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 25 May 2023 11:13:05 +0800 Subject: [PATCH 46/97] enhance: improve document --- docs/en/07-develop/09-udf.md | 13 ++++++++++++- docs/zh/07-develop/09-udf.md | 13 ++++++++++++- tests/script/sh/pycumsum.py | 29 +++++++++++++++++++++++++++++ tests/script/tsim/query/udfpy.sim | 6 ++++++ 4 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 tests/script/sh/pycumsum.py diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md index f107512e9c..b0124b9c3f 100644 --- a/docs/en/07-develop/09-udf.md +++ b/docs/en/07-develop/09-udf.md @@ -377,7 +377,7 @@ The `pybitand` function implements bitwise addition for multiple columns. If the #### Aggregate Function [pyl2norm](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pyl2norm.py) -The `pyl2norm` function finds the second-order norm for all data in the input column. This squares the values, takes a cumulative sum, and finds the square root. +The `pyl2norm` function finds the second-order norm for all data in the input columns. This squares the values, takes a cumulative sum, and finds the square root.
pyl2norm.py @@ -387,5 +387,16 @@ The `pyl2norm` function finds the second-order norm for all data in the input co
+#### Aggregate Function [pycumsum](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pycumsum.py) + +The `pycumsum` function finds the cumulative sum for all data in the input columns. +
+pycumsum.py + +```c +{{#include tests/script/sh/pycumsum.py}} +``` + +
## Manage and Use UDF You need to add UDF to TDengine before using it in SQL queries. For more information about how to manage UDF and how to invoke UDF, please see [Manage and Use UDF](../12-taos-sql/26-udf.md). diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index 99ecd903b4..92f5d2a857 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -335,7 +335,7 @@ def init() def destroy() ``` -其中 init 完成初始化工作。 destroy 完成清理工作。如果没有初始化工作，无需定义 init 函数。如果没有清理工作，无需定义 destroy 函数。 +其中 init 完成初始化工作。 destroy 完成清理工作。 ### Python 和 TDengine之间的数据类型映射 @@ -386,6 +386,17 @@ pyl2norm 实现了输入列的所有数据的二阶范数，即对每个数据 +### 聚合函数示例 [pycumsum](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pycumsum.py) + +pycumsum 使用 numpy 计算输入列所有数据的累积和。 +
+pycumsum.py + +```c +{{#include tests/script/sh/pycumsum.py}} +``` + +
## 管理和使用 UDF 在使用 UDF 之前需要先将其加入到 TDengine 系统中。关于如何管理和使用 UDF，请参考[管理和使用 UDF](../12-taos-sql/26-udf.md) diff --git a/tests/script/sh/pycumsum.py b/tests/script/sh/pycumsum.py new file mode 100644 index 0000000000..27d575aec4 --- /dev/null +++ b/tests/script/sh/pycumsum.py @@ -0,0 +1,29 @@ +import pickle +import numpy as np + +def init(): + pass + +def destroy(): + pass + +def start(): + return pickle.dumps(0.0) + +def finish(buf): + return pickle.loads(buf) + +def reduce(datablock, buf): + (rows, cols) = datablock.shape() + state = pickle.loads(buf) + row = [] + for i in range(rows): + for j in range(cols): + cell = datablock.data(i, j) + if cell is not None: + row.append(datablock.data(i, j)) + if len(row) > 1: + new_state = np.cumsum(row)[-1] + else: + new_state = state + return pickle.dumps(new_state) diff --git a/tests/script/tsim/query/udfpy.sim b/tests/script/tsim/query/udfpy.sim index 9e0492ffd9..bf89ff96a7 100644 --- a/tests/script/tsim/query/udfpy.sim +++ b/tests/script/tsim/query/udfpy.sim @@ -15,6 +15,7 @@ system sh/prepare_pyudf.sh system mkdir -p /tmp/pyudf system cp sh/pybitand.py /tmp/pyudf/ system cp sh/pyl2norm.py /tmp/pyudf/ +system cp sh/pycumsum.py /tmp/pyudf/ system ls /tmp/pyudf sql create database udf vgroups 3; @@ -37,6 +38,7 @@ else endi sql create function pybitand as '/tmp/pyudf/pybitand.py' outputtype int language 'python'; sql create aggregate function pyl2norm as '/tmp/pyudf/pyl2norm.py' outputtype double bufSize 128 language 'python'; +sql create aggregate function pycumsum as '/tmp/pyudf/pycumsum.py' outputtype double bufSize 128 language 'python'; sql show functions; if $rows != 4 then @@ -280,6 +282,10 @@ if $data20 != 8.000000000 then return -1 endi +sql select pycumsum(f2) from udf.t2 +print ======= pycumsum +print $rows $data00 + sql create or replace function bit_and as '/tmp/udf/libbitand.so' outputtype int sql select func_version from information_schema.ins_functions where name='bit_and' if $data00 != 1 then From 0e41f82bb730ed59286ec11c63f40bb2595c737a Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 25 May 2023 11:26:11 +0800 Subject: [PATCH 47/97] test case parameter --- tests/pytest/util/dnodes.py | 12 ++++++++++++ tests/system-test/forcedrop | 1 - 2 files changed, 12 insertions(+), 1 deletion(-) delete mode 120000 tests/system-test/forcedrop diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 5948cde81d..543433b4ea 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -130,6 +130,18 @@ class TDDnode: "locale": "en_US.UTF-8", "charset": "UTF-8", "asyncLog": "0", + "mDebugFlag": "143", + "dDebugFlag": "143", + "vDebugFlag": "143", + "tqDebugFlag": "143", + "cDebugFlag": "143", + "jniDebugFlag": "143", + "qDebugFlag": "143", + "rpcDebugFlag": "143", + "tmrDebugFlag": "131", + "uDebugFlag": "143", + "sDebugFlag": "143", + "wDebugFlag": "143", "numOfLogLines": "100000000", "statusInterval": "1", "enableQueryHb": "1", diff --git a/tests/system-test/forcedrop b/tests/system-test/forcedrop deleted file mode 120000 index d2a16bf505..0000000000 --- a/tests/system-test/forcedrop +++ /dev/null @@ -1 +0,0 @@ -/home/ubuntu/Documents/github/cadem/emptydebug/forcedrop/ \ No newline at end of file From bbb93c3720ea1e59fa1d9c428e17c9fc49cab3f6 Mon Sep 17 00:00:00 2001 From: huolibo Date: Thu, 25 May 2023 11:27:49 +0800 Subject: [PATCH 48/97] enh: tmq example add all properties --- docs/en/14-reference/03-connector/04-java.mdx | 22 +++++++++++++++- .../java/com/taos/example/SubscribeDemo.java | 14 ++++++++--- .../taos/example/WebsocketSubscribeDemo.java | 13 ++++++++-- docs/zh/08-connector/14-java.mdx | 25 ++++++++++++++++--- 4 files changed, 65 insertions(+), 9 deletions(-) diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx index 260b38b24f..114026eca0 100644 --- a/docs/en/14-reference/03-connector/04-java.mdx +++ b/docs/en/14-reference/03-connector/04-java.mdx @@ -959,6 +959,7 @@ The preceding example uses the SQL statement `select ts, speed from speed_table` ```java Properties config = new Properties(); +config.setProperty("bootstrap.servers", "localhost:6030"); config.setProperty("enable.auto.commit", "true"); config.setProperty("group.id", "group1"); config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ResultDeserializer"); @@ -966,12 +967,14 @@ config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.Res TaosConsumer consumer = new TaosConsumer<>(config); ``` +- bootstrap.servers: `ip:port` where the TDengine server is located, or `ip:port` where the taosAdapter is located if WebSocket connection is used. - enable.auto.commit: Specifies whether to commit automatically. - group.id: consumer: Specifies the group that the consumer is in. - value.deserializer: To deserialize the results, you can inherit `com.taosdata.jdbc.tmq.ReferenceDeserializer` and specify the result set bean. You can also inherit `com.taosdata.jdbc.tmq.Deserializer` and perform custom deserialization based on the SQL result set. - td.connect.type: Specifies the type connect with TDengine, `jni` or `WebSocket`. default is `jni` - httpConnectTimeout: WebSocket connection timeout in milliseconds, the default value is 5000 ms. It only takes effect when using WebSocket type. - messageWaitTimeout: socket timeout in milliseconds, the default value is 10000 ms. It only takes effect when using WebSocket type. +- httpPoolSize: Maximum number of concurrent requests on the a connection。It only takes effect when using WebSocket type. - For more information, see [Consumer Parameters](../../../develop/tmq). #### Subscribe to consume data @@ -1015,10 +1018,20 @@ public abstract class ConsumerLoop { public ConsumerLoop() throws SQLException { Properties config = new Properties(); + config.setProperty("td.connect.type", "jni"); + config.setProperty("bootstrap.servers", "localhost:6030"); + config.setProperty("td.connect.user", "root"); + config.setProperty("td.connect.pass", "taosdata"); + config.setProperty("auto.offset.reset", "earliest"); config.setProperty("msg.with.table.name", "true"); config.setProperty("enable.auto.commit", "true"); + config.setProperty("auto.commit.interval.ms", "1000"); config.setProperty("group.id", "group1"); + config.setProperty("client.id", "1"); config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer"); + config.setProperty("value.deserializer.encoding", "UTF-8"); + config.setProperty("experimental.snapshot.enable", "true"); + this.consumer = new TaosConsumer<>(config); this.topics = Collections.singletonList("topic_speed"); @@ -1090,12 +1103,19 @@ public abstract class ConsumerLoop { public ConsumerLoop() throws SQLException { Properties config = new Properties(); - config.setProperty("bootstrap.servers", "localhost:6041"); config.setProperty("td.connect.type", "ws"); + config.setProperty("bootstrap.servers", "localhost:6041"); + config.setProperty("td.connect.user", "root"); + config.setProperty("td.connect.pass", "taosdata"); + config.setProperty("auto.offset.reset", "earliest"); config.setProperty("msg.with.table.name", "true"); config.setProperty("enable.auto.commit", "true"); + config.setProperty("auto.commit.interval.ms", "1000"); config.setProperty("group.id", "group2"); + config.setProperty("client.id", "1"); config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer"); + config.setProperty("value.deserializer.encoding", "UTF-8"); + config.setProperty("experimental.snapshot.enable", "true"); this.consumer = new TaosConsumer<>(config); this.topics = Collections.singletonList("topic_speed"); diff --git a/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java b/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java index b5cdedc34f..3c5d2867e2 100644 --- a/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java +++ b/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java @@ -53,20 +53,28 @@ public class SubscribeDemo { // create consumer Properties properties = new Properties(); + properties.getProperty(TMQConstants.CONNECT_TYPE, "jni"); properties.setProperty(TMQConstants.BOOTSTRAP_SERVERS, "127.0.0.1:6030"); + properties.setProperty(TMQConstants.CONNECT_USER, "root"); + properties.setProperty(TMQConstants.CONNECT_PASS, "taosdata"); properties.setProperty(TMQConstants.MSG_WITH_TABLE_NAME, "true"); properties.setProperty(TMQConstants.ENABLE_AUTO_COMMIT, "true"); - properties.setProperty(TMQConstants.GROUP_ID, "test"); + properties.setProperty(TMQConstants.AUTO_COMMIT_INTERVAL, "1000"); + properties.setProperty(TMQConstants.GROUP_ID, "test1"); + properties.setProperty(TMQConstants.CLIENT_ID, "1"); + properties.setProperty(TMQConstants.AUTO_OFFSET_RESET, "earliest"); properties.setProperty(TMQConstants.VALUE_DESERIALIZER, "com.taos.example.MetersDeserializer"); + properties.setProperty(TMQConstants.VALUE_DESERIALIZER_ENCODING, "UTF-8"); + properties.setProperty(TMQConstants.EXPERIMENTAL_SNAPSHOT_ENABLE, "true"); // poll data try (TaosConsumer consumer = new TaosConsumer<>(properties)) { consumer.subscribe(Collections.singletonList(TOPIC)); while (!shutdown.get()) { ConsumerRecords meters = consumer.poll(Duration.ofMillis(100)); - for (ConsumerRecord recode : meters) { - Meters meter = recode.value(); + for (ConsumerRecord r : meters) { + Meters meter = r.value(); System.out.println(meter); } } diff --git a/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java b/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java index 83cb04f552..03f7e3a11e 100644 --- a/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java +++ b/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java @@ -1,5 +1,6 @@ package com.taos.example; +import com.taosdata.jdbc.tmq.ConsumerRecord; import com.taosdata.jdbc.tmq.ConsumerRecords; import com.taosdata.jdbc.tmq.TMQConstants; import com.taosdata.jdbc.tmq.TaosConsumer; @@ -54,18 +55,26 @@ public class WebsocketSubscribeDemo { Properties properties = new Properties(); properties.setProperty(TMQConstants.BOOTSTRAP_SERVERS, "127.0.0.1:6041"); properties.setProperty(TMQConstants.CONNECT_TYPE, "ws"); + properties.setProperty(TMQConstants.CONNECT_USER, "root"); + properties.setProperty(TMQConstants.CONNECT_PASS, "taosdata"); + properties.setProperty(TMQConstants.AUTO_OFFSET_RESET, "earliest"); properties.setProperty(TMQConstants.MSG_WITH_TABLE_NAME, "true"); properties.setProperty(TMQConstants.ENABLE_AUTO_COMMIT, "true"); - properties.setProperty(TMQConstants.GROUP_ID, "test"); + properties.setProperty(TMQConstants.AUTO_COMMIT_INTERVAL, "1000"); + properties.setProperty(TMQConstants.GROUP_ID, "test2"); + properties.setProperty(TMQConstants.CLIENT_ID, "1"); properties.setProperty(TMQConstants.VALUE_DESERIALIZER, "com.taos.example.MetersDeserializer"); + properties.setProperty(TMQConstants.VALUE_DESERIALIZER_ENCODING, "UTF-8"); + properties.setProperty(TMQConstants.EXPERIMENTAL_SNAPSHOT_ENABLE, "true"); // poll data try (TaosConsumer consumer = new TaosConsumer<>(properties)) { consumer.subscribe(Collections.singletonList(TOPIC)); while (!shutdown.get()) { ConsumerRecords meters = consumer.poll(Duration.ofMillis(100)); - for (Meters meter : meters) { + for (ConsumerRecord r : meters) { + Meters meter = (Meters) r.value(); System.out.println(meter); } } diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx index 35332a9602..e4cf4a83e7 100644 --- a/docs/zh/08-connector/14-java.mdx +++ b/docs/zh/08-connector/14-java.mdx @@ -962,6 +962,7 @@ statement.executeUpdate("create topic if not exists topic_speed as select ts, sp ```java Properties config = new Properties(); +config.setProperty("bootstrap.servers", "localhost:6030"); config.setProperty("enable.auto.commit", "true"); config.setProperty("group.id", "group1"); config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ResultDeserializer"); @@ -969,12 +970,14 @@ config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.Res TaosConsumer consumer = new TaosConsumer<>(config); ``` +- bootstrap.servers: TDengine 服务端所在的`ip:port`，如果使用 WebSocket 连接，则为 taosAdapter 所在的`ip:port`。 - enable.auto.commit: 是否允许自动提交。 - group.id: consumer: 所在的 group。 - value.deserializer: 结果集反序列化方法，可以继承 `com.taosdata.jdbc.tmq.ReferenceDeserializer`，并指定结果集 bean，实现反序列化。也可以继承 `com.taosdata.jdbc.tmq.Deserializer`，根据 SQL 的 resultSet 自定义反序列化方式。 - td.connect.type: 连接方式。jni：表示使用动态库连接的方式，ws/WebSocket：表示使用 WebSocket 进行数据通信。默认为 jni 方式。 -- httpConnectTimeout：创建连接超时参数，单位 ms，默认为 5000 ms。仅在 WebSocket 连接下有效。 -- messageWaitTimeout：数据传输超时参数，单位 ms，默认为 10000 ms。仅在 WebSocket 连接下有效。 +- httpConnectTimeout: 创建连接超时参数，单位 ms，默认为 5000 ms。仅在 WebSocket 连接下有效。 +- messageWaitTimeout: 数据传输超时参数，单位 ms，默认为 10000 ms。仅在 WebSocket 连接下有效。 +- httpPoolSize: 同一个连接下最大并行请求数。仅在 WebSocket 连接下有效。 其他参数请参考：[Consumer 参数列表](../../../develop/tmq#创建-consumer-以及consumer-group) #### 订阅消费数据 @@ -1016,10 +1019,19 @@ public abstract class ConsumerLoop { public ConsumerLoop() throws SQLException { Properties config = new Properties(); + config.setProperty("td.connect.type", "jni"); + config.setProperty("bootstrap.servers", "localhost:6030"); + config.setProperty("td.connect.user", "root"); + config.setProperty("td.connect.pass", "taosdata"); + config.setProperty("auto.offset.reset", "earliest"); config.setProperty("msg.with.table.name", "true"); config.setProperty("enable.auto.commit", "true"); + config.setProperty("auto.commit.interval.ms", "1000"); config.setProperty("group.id", "group1"); + config.setProperty("client.id", "1"); config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer"); + config.setProperty("value.deserializer.encoding", "UTF-8"); + config.setProperty("experimental.snapshot.enable", "true"); this.consumer = new TaosConsumer<>(config); this.topics = Collections.singletonList("topic_speed"); @@ -1093,12 +1105,19 @@ public abstract class ConsumerLoop { public ConsumerLoop() throws SQLException { Properties config = new Properties(); - config.setProperty("bootstrap.servers", "localhost:6041"); config.setProperty("td.connect.type", "ws"); + config.setProperty("bootstrap.servers", "localhost:6041"); + config.setProperty("td.connect.user", "root"); + config.setProperty("td.connect.pass", "taosdata"); + config.setProperty("auto.offset.reset", "earliest"); config.setProperty("msg.with.table.name", "true"); config.setProperty("enable.auto.commit", "true"); + config.setProperty("auto.commit.interval.ms", "1000"); config.setProperty("group.id", "group2"); + config.setProperty("client.id", "1"); config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer"); + config.setProperty("value.deserializer.encoding", "UTF-8"); + config.setProperty("experimental.snapshot.enable", "true"); this.consumer = new TaosConsumer<>(config); this.topics = Collections.singletonList("topic_speed"); From 09662138472bd86beeb1e9f38b1aa92d0f0e8ddc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 25 May 2023 14:21:40 +0800 Subject: [PATCH 49/97] fix(arch): set the correct module api dependency. --- include/libs/executor/storageapi.h | 36 +----------- include/libs/function/function.h | 57 ++++++++++++++++--- include/libs/stream/streamState.h | 10 ---- source/dnode/vnode/src/vnd/vnodeInitApi.c | 7 +++ source/libs/executor/inc/executil.h | 2 +- source/libs/executor/inc/executorInt.h | 5 +- source/libs/executor/src/aggregateoperator.c | 8 +-- source/libs/executor/src/cachescanoperator.c | 2 +- .../libs/executor/src/eventwindowoperator.c | 4 +- source/libs/executor/src/executil.c | 3 +- source/libs/executor/src/executorInt.c | 4 +- source/libs/executor/src/filloperator.c | 13 +++-- source/libs/executor/src/groupoperator.c | 12 ++-- source/libs/executor/src/projectoperator.c | 6 +- source/libs/executor/src/scanoperator.c | 10 ++-- source/libs/executor/src/sortoperator.c | 4 +- source/libs/executor/src/sysscanoperator.c | 2 +- source/libs/executor/src/timesliceoperator.c | 4 +- source/libs/executor/src/timewindowoperator.c | 38 +++++++------ source/libs/function/src/builtinsimpl.c | 32 +++++------ source/libs/stream/src/streamState.c | 1 - 21 files changed, 135 insertions(+), 125 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 0fd8a74127..1240f536ae 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -22,6 +22,7 @@ #include "tmsg.h" #include "tcommon.h" #include "index.h" +#include "function.h" #ifdef __cplusplus extern "C" { @@ -404,41 +405,9 @@ int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list); */ } SStoreMeta; -typedef struct STdbState { - void* rocksdb; - void** pHandle; - void* writeOpts; - void* readOpts; - void** cfOpts; - void* dbOpt; - struct SStreamTask* pOwner; - void* param; - void* env; - SListNode* pComparNode; - void* pBackendHandle; - char idstr[64]; - void* compactFactory; - void* db; - void* pStateDb; - void* pFuncStateDb; - void* pFillStateDb; // todo refactor - void* pSessionStateDb; - void* pParNameDb; - void* pParTagDb; - void* txn; -} STdbState; -// incremental state storage -typedef struct { - STdbState* pTdbState; - struct SStreamFileState* pFileState; - int32_t number; - SSHashObj* parNameMap; - int64_t checkPointId; - int32_t taskId; - int64_t streamId; -} SStreamState; + typedef struct SUpdateInfo { SArray *pTsBuckets; @@ -546,6 +515,7 @@ typedef struct SStorageAPI { SStoreTqReader tqReaderFn; SStateStore stateStore; SMetaDataFilterAPI metaFilter; + SFunctionStateStore functionStore; } SStorageAPI; #ifdef __cplusplus diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 85f7cf7e2c..de2494ab3c 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -22,6 +22,7 @@ extern "C" { #include "tcommon.h" #include "tvariant.h" +#include "tsimplehash.h" struct SqlFunctionCtx; struct SResultRowEntryInfo; @@ -127,18 +128,59 @@ typedef struct SSerializeDataHandle { void *pState; } SSerializeDataHandle; +// incremental state storage +typedef struct STdbState { + void* rocksdb; + void** pHandle; + void* writeOpts; + void* readOpts; + void** cfOpts; + void* dbOpt; + struct SStreamTask* pOwner; + void* param; + void* env; + SListNode* pComparNode; + void* pBackendHandle; + char idstr[64]; + void* compactFactory; + + void* db; + void* pStateDb; + void* pFuncStateDb; + void* pFillStateDb; // todo refactor + void* pSessionStateDb; + void* pParNameDb; + void* pParTagDb; + void* txn; +} STdbState; + +typedef struct { + STdbState* pTdbState; + struct SStreamFileState* pFileState; + int32_t number; + SSHashObj* parNameMap; + int64_t checkPointId; + int32_t taskId; + int64_t streamId; +} SStreamState; + +typedef struct SFunctionStateStore { + int32_t (*streamStateFuncPut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); + int32_t (*streamStateFuncGet)(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen); +} SFunctionStateStore; + // sql function runtime context typedef struct SqlFunctionCtx { SInputColumnInfoData input; SResultDataInfo resDataInfo; - uint32_t order; // data block scanner order: asc|desc - uint8_t isPseudoFunc;// denote current function is pseudo function or not [added for perf reason] - uint8_t isNotNullFunc;// not return null value. - uint8_t scanFlag; // record current running step, default: 0 - int16_t functionId; // function id - char *pOutput; // final result output buffer, point to sdata->data + uint32_t order; // data block scanner order: asc|desc + uint8_t isPseudoFunc; // denote current function is pseudo function or not [added for perf reason] + uint8_t isNotNullFunc; // not return null value. + uint8_t scanFlag; // record current running step, default: 0 + int16_t functionId; // function id + char *pOutput; // final result output buffer, point to sdata->data // input parameter, e.g., top(k, 20), the number of results of top query is kept in param - SFunctParam *param; + SFunctParam *param; // corresponding output buffer for timestamp of each result, e.g., diff/csum SColumnInfoData *pTsOutput; int32_t numOfParams; @@ -155,6 +197,7 @@ typedef struct SqlFunctionCtx { SSerializeDataHandle saveHandle; int32_t exprIdx; char *udfName; + SFunctionStateStore *pStore; } SqlFunctionCtx; typedef struct tExprNode { diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 08357b8251..ca85622d8a 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -66,16 +66,6 @@ int32_t streamStateCommit(SStreamState* pState); void streamStateDestroy(SStreamState* pState, bool remove); int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark); -//typedef struct { -// rocksdb_iterator_t* iter; -// rocksdb_snapshot_t* snapshot; -// rocksdb_readoptions_t* readOpt; -// rocksdb_t* db; -// -// TBC* pCur; -// int64_t number; -//} SStreamStateCur; - int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen); diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index d9bb4270d8..923e25f73b 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -24,6 +24,7 @@ static void initTqAPI(SStoreTqReader* pTq); static void initStateStoreAPI(SStateStore* pStore); static void initMetaReaderAPI(SStoreMetaReader* pMetaReader); static void initMetaFilterAPI(SMetaDataFilterAPI* pFilter); +static void initFunctionStateStore(SFunctionStateStore* pStore); void initStorageAPI(SStorageAPI* pAPI) { initTsdbReaderAPI(&pAPI->tsdReader); @@ -32,6 +33,7 @@ void initStorageAPI(SStorageAPI* pAPI) { initStateStoreAPI(&pAPI->stateStore); initMetaReaderAPI(&pAPI->metaReaderFn); initMetaFilterAPI(&pAPI->metaFilter); + initFunctionStateStore(&pAPI->functionStore); } void initTsdbReaderAPI(TsdReader* pReader) { @@ -206,4 +208,9 @@ void initMetaFilterAPI(SMetaDataFilterAPI* pFilter) { pFilter->metaFilterTableIds = metaFilterTableIds; pFilter->metaFilterTableName = metaFilterTableName; pFilter->metaFilterTtl = metaFilterTtl; +} + +void initFunctionStateStore(SFunctionStateStore* pStore) { + pStore->streamStateFuncPut = streamStateFuncPut; + pStore->streamStateFuncGet = streamStateFuncGet; } \ No newline at end of file diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 6dc359c502..966c5fa2e4 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -166,7 +166,7 @@ void createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId); void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode); SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs); -SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset); +SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset, SFunctionStateStore* pStore); void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols, bool outputEveryColumn); void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow); diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 8188ecd305..79391bc1c5 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -579,12 +579,11 @@ void cleanupQueriedTableScanInfo(SSchemaInfo* pSchemaInfo); void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock); void cleanupBasicInfo(SOptrBasicInfo* pInfo); -int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr); +int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr, SFunctionStateStore* pStore); void cleanupExprSupp(SExprSupp* pSup); - int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, - const char* pkey, void* pState); + const char* pkey, void* pState, SFunctionStateStore* pStore); void cleanupAggSup(SAggSupporter* pAggSup); void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows); diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index 00aa52b593..b34c6f4b82 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -84,7 +84,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -95,7 +95,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN pScalarExprInfo = createExprInfo(pAggNode->pExprs, NULL, &numOfScalarExpr); } - code = initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr); + code = initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -485,8 +485,8 @@ void cleanupAggSup(SAggSupporter* pAggSup) { } int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, - const char* pkey, void* pState) { - int32_t code = initExprSupp(pSup, pExprInfo, numOfCols); + const char* pkey, void* pState, SFunctionStateStore* pStore) { + int32_t code = initExprSupp(pSup, pExprInfo, numOfCols, pStore); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 1b6c06b6c4..48003b277b 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -126,7 +126,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe if (pScanNode->scan.pScanPseudoCols != NULL) { SExprSupp* p = &pInfo->pseudoExprSup; p->pExprInfo = createExprInfo(pScanNode->scan.pScanPseudoCols, NULL, &p->numOfExprs); - p->pCtx = createSqlFunctionCtx(p->pExprInfo, p->numOfExprs, &p->rowEntryInfoOffset); + p->pCtx = createSqlFunctionCtx(p->pExprInfo, p->numOfExprs, &p->rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore); } setOperatorInfo(pOperator, "CachedRowScanOperator", QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, false, OP_NOT_OPENED, diff --git a/source/libs/executor/src/eventwindowoperator.c b/source/libs/executor/src/eventwindowoperator.c index 956d5b714d..b5dea6d94d 100644 --- a/source/libs/executor/src/eventwindowoperator.c +++ b/source/libs/executor/src/eventwindowoperator.c @@ -92,7 +92,7 @@ SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo if (pEventWindowNode->window.pExprs != NULL) { int32_t numOfScalarExpr = 0; SExprInfo* pScalarExprInfo = createExprInfo(pEventWindowNode->window.pExprs, NULL, &numOfScalarExpr); - code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr); + code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -110,7 +110,7 @@ SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo initResultSizeInfo(&pOperator->resultInfo, 4096); code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 47b08b3323..ebe3d69dbb 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1544,7 +1544,7 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu return TSDB_CODE_SUCCESS; } -SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset) { +SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset, SFunctionStateStore* pStore) { SqlFunctionCtx* pFuncCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx)); if (pFuncCtx == NULL) { return NULL; @@ -1607,6 +1607,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, pCtx->numOfParams = pExpr->base.numOfParams; pCtx->param = pFunct->pParam; pCtx->saveHandle.currentPage = -1; + pCtx->pStore = pStore; } for (int32_t i = 1; i < numOfOutput; ++i) { diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 45478cf837..5e0e7b0d6a 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -979,11 +979,11 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { return NULL; } -int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr) { +int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr, SFunctionStateStore* pStore) { pSup->pExprInfo = pExprInfo; pSup->numOfExprs = numOfExpr; if (pSup->pExprInfo != NULL) { - pSup->pCtx = createSqlFunctionCtx(pExprInfo, numOfExpr, &pSup->rowEntryInfoOffset); + pSup->pCtx = createSqlFunctionCtx(pExprInfo, numOfExpr, &pSup->rowEntryInfoOffset, pStore); if (pSup->pCtx == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index b448e8f2b0..90b7f4a77e 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -338,7 +338,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* goto _error; } - code = initExprSupp(pNoFillSupp, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs); + code = initExprSupp(pNoFillSupp, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -355,7 +355,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* initResultSizeInfo(&pOperator->resultInfo, 4096); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - code = initExprSupp(&pOperator->exprSupp, pExprInfo, pInfo->numOfExpr); + code = initExprSupp(&pOperator->exprSupp, pExprInfo, pInfo->numOfExpr, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -525,7 +525,7 @@ void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, pFillSup->cur.key = key.ts; pFillSup->cur.pRowVal = curVal; - void* pCur = pAPI->stateStore.streamStateFillSeekKeyPrev(pState, &key); + SStreamStateCur* pCur = pAPI->stateStore.streamStateFillSeekKeyPrev(pState, &key); SWinKey preKey = {.ts = INT64_MIN, .groupId = groupId}; void* preVal = NULL; int32_t preVLen = 0; @@ -1139,7 +1139,8 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { TSKEY endTs = ts; uint64_t groupId = groupIds[pInfo->srcDelRowIndex]; SWinKey key = {.ts = ts, .groupId = groupId}; - void* pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key); + SStreamStateCur* pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key); + if (!pCur) { pInfo->srcDelRowIndex++; continue; @@ -1355,7 +1356,7 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod } SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); - code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols); + code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols, &pAPI->functionStore); if (code != TSDB_CODE_SUCCESS) { destroyStreamFillSupporter(pFillSup); return NULL; @@ -1491,7 +1492,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi goto _error; } - code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols); + code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 77d251bab9..859fad3803 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -451,7 +451,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* } pInfo->pGroupCols = extractColumnInfo(pAggNode->pGroupKeys); - code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr); + code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -467,7 +467,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, pInfo->groupKeyLen, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -849,7 +849,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition if (pPartNode->pExprs != NULL) { int32_t num = 0; SExprInfo* pExprInfo1 = createExprInfo(pPartNode->pExprs, NULL, &num); - int32_t code = initExprSupp(&pInfo->scalarSup, pExprInfo1, num); + int32_t code = initExprSupp(&pInfo->scalarSup, pExprInfo1, num, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { terrno = code; pTaskInfo->code = terrno; @@ -1242,7 +1242,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr if (pPartNode->part.pExprs != NULL) { int32_t num = 0; SExprInfo* pCalExprInfo = createExprInfo(pPartNode->part.pExprs, NULL, &num); - code = initExprSupp(&pInfo->scalarSup, pCalExprInfo, num); + code = initExprSupp(&pInfo->scalarSup, pCalExprInfo, num, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1257,7 +1257,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr } pInfo->tbnameCalSup.pExprInfo = pSubTableExpr; createExprFromOneNode(pSubTableExpr, pPartNode->pSubtable, 0); - code = initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1); + code = initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1271,7 +1271,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr code = TSDB_CODE_OUT_OF_MEMORY; goto _error; } - if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) { + if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags, &pTaskInfo->storageAPI.functionStore) != 0) { code = TSDB_CODE_OUT_OF_MEMORY; goto _error; } diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index ae72627575..dde6f7c0e8 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -115,7 +115,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys initResultSizeInfo(&pOperator->resultInfo, numOfRows); code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -370,7 +370,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy if (pPhyNode->pExprs != NULL) { int32_t num = 0; SExprInfo* pSExpr = createExprInfo(pPhyNode->pExprs, NULL, &num); - int32_t code = initExprSupp(&pInfo->scalarSup, pSExpr, num); + int32_t code = initExprSupp(&pInfo->scalarSup, pSExpr, num, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -392,7 +392,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy blockDataEnsureCapacity(pResBlock, numOfRows); int32_t code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfExpr, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 39d619172b..66e200c81b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -962,7 +962,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, if (pScanNode->pScanPseudoCols != NULL) { SExprSupp* pSup = &pInfo->base.pseudoSup; pSup->pExprInfo = createExprInfo(pScanNode->pScanPseudoCols, NULL, &pSup->numOfExprs); - pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset); + pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore); } pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; @@ -2349,7 +2349,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->tbnameCalSup.pExprInfo = pSubTableExpr; createExprFromOneNode(pSubTableExpr, pTableScanNode->pSubtable, 0); - if (initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1) != 0) { + if (initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1, &pTaskInfo->storageAPI.functionStore) != 0) { tableListDestroy(pTableListInfo); goto _error; } @@ -2363,7 +2363,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys tableListDestroy(pTableListInfo); goto _error; } - if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) { + if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags, &pTaskInfo->storageAPI.functionStore) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; tableListDestroy(pTableListInfo); goto _error; @@ -2612,7 +2612,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi int32_t numOfExprs = 0; SExprInfo* pExprInfo = createExprInfo(pPhyNode->pScanPseudoCols, NULL, &numOfExprs); - int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs); + int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -3033,7 +3033,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN if (pTableScanNode->scan.pScanPseudoCols != NULL) { SExprSupp* pSup = &pInfo->base.pseudoSup; pSup->pExprInfo = createExprInfo(pTableScanNode->scan.pScanPseudoCols, NULL, &pSup->numOfExprs); - pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset); + pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore); } pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 718bb596c5..0357828732 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -60,7 +60,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* } pOperator->exprSupp.pCtx = - createSqlFunctionCtx(pOperator->exprSupp.pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset); + createSqlFunctionCtx(pOperator->exprSupp.pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore); initResultSizeInfo(&pOperator->resultInfo, 1024); code = filterInitFromNode((SNode*)pSortNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); if (code != TSDB_CODE_SUCCESS) { @@ -500,7 +500,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort pSup->numOfExprs = numOfCols; initResultSizeInfo(&pOperator->resultInfo, 1024); - pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset); + pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore); pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index d15b6d5754..40b45c25cd 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -2301,7 +2301,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols); - int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols); + int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 209e5cb8fc..6ed247a76d 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -818,7 +818,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode int32_t numOfExprs = 0; SExprInfo* pExprInfo = createExprInfo(pInterpPhyNode->pFuncs, NULL, &numOfExprs); - int32_t code = initExprSupp(pSup, pExprInfo, numOfExprs); + int32_t code = initExprSupp(pSup, pExprInfo, numOfExprs, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -826,7 +826,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode if (pInterpPhyNode->pExprs != NULL) { int32_t num = 0; SExprInfo* pScalarExprInfo = createExprInfo(pInterpPhyNode->pExprs, NULL, &num); - code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num); + code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 290bc43cbe..1adc171c5b 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1656,7 +1656,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num); int32_t code = - initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, pTaskInfo->streamInfo.pState); + initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1684,7 +1684,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh if (pPhyNode->window.pExprs != NULL) { int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar); - code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1884,7 +1884,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi if (pStateNode->window.pExprs != NULL) { int32_t numOfScalarExpr = 0; SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalarExpr); - int32_t code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr); + int32_t code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1910,7 +1910,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi initResultSizeInfo(&pOperator->resultInfo, 4096); code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1979,7 +1979,7 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW initBasicInfo(&pInfo->binfo, pResBlock); int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -2744,7 +2744,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, if (pIntervalPhyNode->window.pExprs != NULL) { int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); - int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -2757,9 +2757,10 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); *(pInfo->pState) = *(pTaskInfo->streamInfo.pState); + pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1); int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, - pInfo->pState); + pInfo->pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -2861,9 +2862,9 @@ void destroyStreamSessionAggOperatorInfo(void* param) { } int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResultBlock) { + SSDataBlock* pResultBlock, SFunctionStateStore* pStore) { initBasicInfo(pBasicInfo, pResultBlock); - int32_t code = initExprSupp(pSup, pExprInfo, numOfCols); + int32_t code = initExprSupp(pSup, pExprInfo, numOfCols, pStore); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2936,6 +2937,7 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, if (bufSize <= pageSize) { bufSize = pageSize * 4; } + if (!osTempSpaceAvailable()) { terrno = TSDB_CODE_NO_DISKSPACE; qError("Init stream agg supporter failed since %s, tempDir:%s", terrstr(), tsTempDir); @@ -3588,7 +3590,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh if (pSessionNode->window.pExprs != NULL) { int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar); - code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -3597,7 +3599,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); - code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock); + code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4126,7 +4128,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys if (pStateNode->window.pExprs != NULL) { int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar); - code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4145,7 +4147,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); - code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock); + code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4433,7 +4435,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num); code = initAggSup(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4719,7 +4721,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge initResultSizeInfo(&pOperator->resultInfo, 4096); int32_t code = initAggSup(pExprSupp, &pIntervalInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4764,7 +4766,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - SExprSupp* pSup = &pOperator->exprSupp; + SExprSupp* pSup = &pOperator->exprSupp; if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -4948,7 +4950,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, - pInfo->pState); + pInfo->pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4956,7 +4958,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys if (pIntervalPhyNode->window.pExprs != NULL) { int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); - code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index f83873848e..5c47daca5b 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -18,7 +18,6 @@ #include "function.h" #include "query.h" #include "querynodes.h" -//#include "streamState.h" #include "tcompare.h" #include "tdatablock.h" #include "tdigest.h" @@ -1697,7 +1696,7 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { GET_TYPED_DATA(v, double, pVal->nType, &pVal->i); - int32_t code = getPercentile(pMemBucket, v, &ppInfo->result); + code = getPercentile(pMemBucket, v, &ppInfo->result); if (code != TSDB_CODE_SUCCESS) { goto _fin_error; } @@ -3120,7 +3119,7 @@ void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsid } static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, SWinKey* key, - STuplePos* pPos) { + STuplePos* pPos, SFunctionStateStore* pStore) { STuplePos p = {0}; if (pHandle->pBuf != NULL) { SFilePage* pPage = NULL; @@ -3153,11 +3152,10 @@ static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, pPage->num += length; setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); - } else { - // other tuple save policy -// if (streamStateFuncPut(pHandle->pState, key, pBuf, length) >= 0) { -// p.streamTupleKey = *key; -// } + } else { // other tuple save policy + if (pStore->streamStateFuncPut(pHandle->pState, key, pBuf, length) >= 0) { + p.streamTupleKey = *key; + } } *pPos = p; @@ -3179,10 +3177,10 @@ int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* } char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf); - return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, &key, pPos); + return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, &key, pPos, pCtx->pStore); } -static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STuplePos* pPos) { +static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STuplePos* pPos, SFunctionStateStore* pStore) { if (pHandle->pBuf != NULL) { SFilePage* pPage = getBufPage(pHandle->pBuf, pPos->pageId); if (pPage == NULL) { @@ -3192,7 +3190,7 @@ static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); } else { -// streamStateFuncPut(pHandle->pState, &pPos->streamTupleKey, pBuf, length); + pStore->streamStateFuncPut(pHandle->pState, &pPos->streamTupleKey, pBuf, length); } return TSDB_CODE_SUCCESS; @@ -3202,10 +3200,10 @@ int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBloc prepareBuf(pCtx); char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf); - return doUpdateTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pPos); + return doUpdateTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pPos, pCtx->pStore); } -static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPos) { +static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPos, SFunctionStateStore* pStore) { if (pHandle->pBuf != NULL) { SFilePage* pPage = getBufPage(pHandle->pBuf, pPos->pageId); if (pPage == NULL) { @@ -3217,13 +3215,13 @@ static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPo } else { void* value = NULL; int32_t vLen; -// streamStateFuncGet(pHandle->pState, &pPos->streamTupleKey, &value, &vLen); + pStore->streamStateFuncGet(pHandle->pState, &pPos->streamTupleKey, &value, &vLen); return (char*)value; } } const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos) { - return doLoadTupleData(&pCtx->saveHandle, pPos); + return doLoadTupleData(&pCtx->saveHandle, pPos, pCtx->pStore); } int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { @@ -4991,7 +4989,7 @@ static int32_t saveModeTupleData(SqlFunctionCtx* pCtx, char* data, SModeInfo *pI memcpy(pInfo->buf, data, pInfo->colBytes); } - return doSaveTupleData(&pCtx->saveHandle, pInfo->buf, pInfo->colBytes, NULL, pPos); + return doSaveTupleData(&pCtx->saveHandle, pInfo->buf, pInfo->colBytes, NULL, pPos, pCtx->pStore); } static int32_t doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCtx, char* data) { @@ -5020,7 +5018,7 @@ static int32_t doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCt } else { pHashItem->count += 1; if (pCtx->subsidiaries.num > 0) { - int32_t code = updateTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &pHashItem->tuplePos); + code = updateTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &pHashItem->tuplePos); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 24339337b7..7af92cfb81 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -23,7 +23,6 @@ #include "tcommon.h" #include "tcompare.h" #include "tref.h" -#include "ttimer.h" #define MAX_TABLE_NAME_NUM 2000000 From fcba7e326cc7de4ee7e81751fea84bf096536e68 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 25 May 2023 14:24:25 +0800 Subject: [PATCH 50/97] feat:add privilege for schemaless --- source/client/src/clientSml.c | 2 +- utils/test/c/sml_test.c | 129 ++++++++++++++++++++++++++++++++-- 2 files changed, 126 insertions(+), 5 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index e873c81c0f..92e4d0f8da 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -106,7 +106,7 @@ static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo* conn, const cha code = catalogChkAuth(info->pCatalog, conn, &pAuth, &authRes); - return code ? (authRes.pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code; + return (code == TSDB_CODE_SUCCESS) ? (authRes.pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code; } inline bool smlDoubleToInt64OverFlow(double num) { diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index f82b820244..3ccff755f5 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1135,18 +1135,34 @@ int sml_td22900_Test() { int sml_td24070_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - TAOS_RES *pRes = taos_query(taos, "CREATE user test pass 'test'"); + TAOS_RES *pRes = taos_query(taos, "CREATE user test_db pass 'test'"); + ASSERT(taos_errno(pRes) == 0); taos_free_result(pRes); - pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS td24070"); + pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS td24070_read"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "grant read on td24070_read to test_db"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS td24070_write"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "grant write on td24070_write to test_db"); + ASSERT(taos_errno(pRes) == 0); taos_free_result(pRes); taos_close(taos); - taos = taos_connect("localhost", "test", "test", NULL, 0); + + // test db privilege + taos = taos_connect("localhost", "test_db", "test", NULL, 0); const char* sql[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299372000"}; - pRes = taos_query(taos, "use td24070"); + pRes = taos_query(taos, "use td24070_read"); taos_free_result(pRes); pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, @@ -1154,8 +1170,113 @@ int sml_td24070_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); + ASSERT(code != 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "use td24070_write"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + ASSERT(code == 0); taos_free_result(pRes); taos_close(taos); + // test db privilege end + + + // test stable privilege + taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + pRes = taos_query(taos, "CREATE user test_stb_read pass 'test'"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "CREATE user test_stb_write pass 'test'"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "grant read on td24070_write.stb2 to test_stb_read"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "grant write on td24070_write.stb2 to test_stb_write"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + taos_close(taos); + + taos = taos_connect("localhost", "test_stb_read", "test", "td24070_write", 0); + const char* sql1[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299373000"}; + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + ASSERT(code != 0); + taos_free_result(pRes); + taos_close(taos); + + taos = taos_connect("localhost", "test_stb_write", "test", "td24070_write", 0); + const char* sql2[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299373000"}; + + pRes = taos_schemaless_insert(taos, (char **)sql2, sizeof(sql2) / sizeof(sql2[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + ASSERT(code == 0); + taos_free_result(pRes); + taos_close(taos); + // test stable privilege + + // test table privilege + taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + pRes = taos_query(taos, "CREATE user test_tb_read pass 'test'"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "CREATE user test_tb_write pass 'test'"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "grant read on td24070_write.t_035c845c1a3df238c54d9b8f1a8844bd to test_tb_read"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "grant write on td24070_write.t_035c845c1a3df238c54d9b8f1a8844bd to test_tb_write"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + taos_close(taos); + + taos = taos_connect("localhost", "test_tb_read", "test", "td24070_write", 0); + const char* sql3[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299374000"}; + + + pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + ASSERT(code != 0); + taos_free_result(pRes); + taos_close(taos); + + taos = taos_connect("localhost", "test_tb_write", "test", "td24070_write", 0); + const char* sql4[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299374000"}; + + pRes = taos_schemaless_insert(taos, (char **)sql4, sizeof(sql4) / sizeof(sql4[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + ASSERT(code == 0); + taos_free_result(pRes); + taos_close(taos); + // test table privilege return code; } From de1848cb08726936958a51f6667ca33e49eb6995 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 25 May 2023 14:51:53 +0800 Subject: [PATCH 51/97] enhance: use scalar mode to run ci test --- source/libs/function/src/udfd.c | 2 +- source/libs/scalar/src/filter.c | 2 +- source/libs/scalar/src/sclvector.c | 17 ++++++++++++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index b6ab7bd6f1..3b827a2f99 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -600,9 +600,9 @@ SUdf *udfdGetOrCreateUdf(const char *udfName) { return udf; } else { (*pUdfHash)->expired = true; - taosHashRemove(global.udfsHash, udfName, strlen(udfName)); fnInfo("udfd expired, check for new version. existing udf %s udf version %d, udf created time %" PRIx64, (*pUdfHash)->name, (*pUdfHash)->version, (*pUdfHash)->createdTime); + taosHashRemove(global.udfsHash, udfName, strlen(udfName)); } } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 27cf9da1d5..464d388db4 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4556,7 +4556,7 @@ int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pInfo, uint32_t options) FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat)); - info->scalarMode = stat.scalarMode; + info->scalarMode = true; fltDebug("scalar mode: %d", info->scalarMode); if (!info->scalarMode) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 4d803cb638..6983f6313b 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1784,6 +1784,9 @@ void vectorNotMatch(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOu void vectorIsNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) { for (int32_t i = 0; i < pLeft->numOfRows; ++i) { int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 1 : 0; + if (v) { + ++pOut->numOfQualified; + } colDataSetInt8(pOut->columnData, i, &v); } pOut->numOfRows = pLeft->numOfRows; @@ -1792,6 +1795,9 @@ void vectorIsNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, void vectorNotNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) { for (int32_t i = 0; i < pLeft->numOfRows; ++i) { int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 0 : 1; + if (v) { + ++pOut->numOfQualified; + } colDataSetInt8(pOut->columnData, i, &v); } pOut->numOfRows = pLeft->numOfRows; @@ -1805,6 +1811,13 @@ void vectorIsTrue(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, colDataSetInt8(pOut->columnData, i, &v); colDataClearNull_f(pOut->columnData->nullbitmap, i); } + { + bool v = false; + GET_TYPED_DATA(v, bool, pOut->columnData->info.type, colDataGetData(pOut->columnData, i)); + if (v) { + ++pOut->numOfQualified; + } + } } pOut->columnData->hasNull = false; } @@ -1844,7 +1857,9 @@ void vectorJsonContains(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam char *pLeftData = colDataGetVarData(pLeft->columnData, i); getJsonValue(pLeftData, jsonKey, &isExist); } - + if (isExist) { + ++pOut->numOfQualified; + } colDataSetVal(pOutputCol, i, (const char *)(&isExist), false); } taosMemoryFree(jsonKey); From f6fef3e9d66b3733e8cff2b15cb43c9a8c090097 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 25 May 2023 14:54:21 +0800 Subject: [PATCH 52/97] feat:add privilege for schemaless --- utils/test/c/sml_test.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 3ccff755f5..94619339e9 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1243,11 +1243,11 @@ int sml_td24070_Test() { ASSERT(taos_errno(pRes) == 0); taos_free_result(pRes); - pRes = taos_query(taos, "grant read on td24070_write.t_035c845c1a3df238c54d9b8f1a8844bd to test_tb_read"); + pRes = taos_query(taos, "grant read on td24070_write.stb2 with t1=1 to test_tb_read"); ASSERT(taos_errno(pRes) == 0); taos_free_result(pRes); - pRes = taos_query(taos, "grant write on td24070_write.t_035c845c1a3df238c54d9b8f1a8844bd to test_tb_write"); + pRes = taos_query(taos, "grant write on td24070_write.stb2 with t1=1 to test_tb_write"); ASSERT(taos_errno(pRes) == 0); taos_free_result(pRes); taos_close(taos); From 61b063529384019a954f05fdcb247dc8623537ed Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 25 May 2023 16:27:48 +0800 Subject: [PATCH 53/97] fix: get time range using new scalar mode range --- source/libs/scalar/src/filter.c | 46 +++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 464d388db4..ae4712b9ba 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3765,8 +3765,8 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t SColumnDataAgg *pAgg = pDataStatis[j]; SArray *points = taosArrayInit(2, sizeof(SFltSclPoint)); fltSclBuildRangeFromBlockSma(colRange, pAgg, numOfRows, points); - qDebug("column data agg: nulls %d, rows %d, max %" PRId64 " min %" PRId64, pAgg->numOfNull, numOfRows, pAgg->max, - pAgg->min); + qDebug("column data agg: nulls %d, rows %d, max %" PRId64 " min %" PRId64, pAgg->numOfNull, numOfRows, + pAgg->max, pAgg->min); SArray *merged = taosArrayInit(8, sizeof(SFltSclPoint)); fltSclIntersect(points, colRange->points, merged); @@ -3960,6 +3960,31 @@ _return: return code; } +static int32_t fltSclGetDatumValueFromPoint(SFltSclPoint *point, SFltSclDatum *d) { + *d = point->val; + if (point->val.kind == FLT_SCL_DATUM_KIND_NULL) { + return TSDB_CODE_SUCCESS; + } + if (point->val.kind == FLT_SCL_DATUM_KIND_MAX) { + getDataMax(d->type.type, &(d->i)); + } else if (point->val.kind == FLT_SCL_DATUM_KIND_MIN) { + getDataMin(d->type.type, &(d->i)); + } + + if ((point->val.kind == FLT_SCL_DATUM_KIND_INT64) || (point->val.kind = FLT_SCL_DATUM_KIND_UINT64)) { + if (point->excl) { + if (point->start) { + ++d->i; + } else { + --d->i; + } + } + } else { + qError("not supported kind %d when get datum from point", point->val.kind); + } + return TSDB_CODE_SUCCESS; +} + int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) { SFilterInfo *info = NULL; int32_t code = 0; @@ -3969,6 +3994,23 @@ int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) { FLT_ERR_RET(filterInitFromNode(pNode, &info, FLT_OPTION_NO_REWRITE | FLT_OPTION_TIMESTAMP)); if (info->scalarMode) { + SArray *colRanges = info->sclCtx.fltSclRange; + if (taosArrayGetSize(colRanges) == 1) { + SFltSclColumnRange *colRange = taosArrayGet(colRanges, 0); + SArray *points = colRange->points; + if (taosArrayGetSize(points) == 2) { + SFltSclPoint *startPt = taosArrayGet(points, 0); + SFltSclPoint *endPt = taosArrayGet(points, 1); + SFltSclDatum start; + SFltSclDatum end; + fltSclGetDatumValueFromPoint(startPt, &start); + fltSclGetDatumValueFromPoint(endPt, &end); + win->skey = start.i; + win->ekey = end.i; + *isStrict = true; + goto _return; + } + } *win = TSWINDOW_INITIALIZER; *isStrict = false; goto _return; From cd2fdf49f3edeab61ab0b65b227757df24e07507 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 25 May 2023 16:48:47 +0800 Subject: [PATCH 54/97] docs: update readme (#21473) * docs: update readme with libgflags * docs: update readme with geos * docs: remove geometry docs * docs: update readme --- README-CN.md | 18 +++++++++--------- README.md | 16 ++++++++-------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/README-CN.md b/README-CN.md index 6c49ce0727..81d64941af 100644 --- a/README-CN.md +++ b/README-CN.md @@ -52,7 +52,7 @@ TDengine 还提供一组辅助工具软件 taosTools，目前它包含 taosBench ### Ubuntu 18.04 及以上版本 & Debian： ```bash -sudo apt-get install -y gcc cmake build-essential git libssl-dev libgflags2.2 libgflags-dev +sudo apt-get install -y gcc cmake build-essential git libssl-dev libgflags2.2 libgflags-dev libgeos-dev ``` #### 为 taos-tools 安装编译需要的软件 @@ -60,7 +60,7 @@ sudo apt-get install -y gcc cmake build-essential git libssl-dev libgflags2.2 li 为了在 Ubuntu/Debian 系统上编译 [taos-tools](https://github.com/taosdata/taos-tools) 需要安装如下软件： ```bash -sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config libgeos-dev +sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config ``` ### CentOS 7.9 @@ -68,14 +68,14 @@ sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-d ```bash sudo yum install epel-release sudo yum update -sudo yum install -y gcc gcc-c++ make cmake3 git openssl-devel +sudo yum install -y gcc gcc-c++ make cmake3 git openssl-devel geos geos-devel sudo ln -sf /usr/bin/cmake3 /usr/bin/cmake ``` -### CentOS 8 & Fedora +### CentOS 8/Fedora/Rocky Linux ```bash -sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel +sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel geos geos-devel ``` #### 在 CentOS 上构建 taosTools 安装依赖软件 @@ -85,23 +85,23 @@ sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel ``` -sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel geos geos-devel +sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel ``` -#### CentOS 8/Rocky Linux +#### CentOS 8/Fedora/Rocky Linux ``` sudo yum install -y epel-release sudo yum install -y dnf-plugins-core sudo yum config-manager --set-enabled powertools -sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel geos geos-devel +sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel ``` 注意：由于 snappy 缺乏 pkg-config 支持（参考 [链接](https://github.com/google/snappy/pull/86)），会导致 cmake 提示无法发现 libsnappy，实际上工作正常。 若 powertools 安装失败，可以尝试改用： ``` -sudo yum config-manager --set-enabled Powertools +sudo yum config-manager --set-enabled powertools ``` #### CentOS + devtoolset diff --git a/README.md b/README.md index 2f1650c3c9..c81e2d309d 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ To build TDengine, use [CMake](https://cmake.org/) 3.0.2 or higher versions in t ### Ubuntu 18.04 and above or Debian ```bash -sudo apt-get install -y gcc cmake build-essential git libssl-dev libgflags2.2 libgflags-dev +sudo apt-get install -y gcc cmake build-essential git libssl-dev libgflags2.2 libgflags-dev libgeos-dev ``` #### Install build dependencies for taosTools @@ -68,7 +68,7 @@ sudo apt-get install -y gcc cmake build-essential git libssl-dev libgflags2.2 li To build the [taosTools](https://github.com/taosdata/taos-tools) on Ubuntu/Debian, the following packages need to be installed. ```bash -sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config libgeos-dev +sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config ``` ### CentOS 7.9 @@ -76,14 +76,14 @@ sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-d ```bash sudo yum install epel-release sudo yum update -sudo yum install -y gcc gcc-c++ make cmake3 git openssl-devel +sudo yum install -y gcc gcc-c++ make cmake3 git openssl-devel geos geos-devel sudo ln -sf /usr/bin/cmake3 /usr/bin/cmake ``` -### CentOS 8 & Fedora +### CentOS 8/Fedora/Rocky Linux ```bash -sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel +sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel geos geos-devel ``` #### Install build dependencies for taosTools on CentOS @@ -91,16 +91,16 @@ sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel #### CentOS 7.9 ``` -sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel geos geos-devel +sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel ``` -#### CentOS 8/Rocky Linux +#### CentOS 8/Fedora/Rocky Linux ``` sudo yum install -y epel-release sudo yum install -y dnf-plugins-core sudo yum config-manager --set-enabled powertools -sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel geos geos-devel +sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel ``` Note: Since snappy lacks pkg-config support (refer to [link](https://github.com/google/snappy/pull/86)), it leads a cmake prompt libsnappy not found. But snappy still works well. From d9042f5ed38bdd9529709d685f555d49b43b558e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 25 May 2023 17:24:11 +0800 Subject: [PATCH 55/97] fix:memory leak --- source/client/inc/clientSml.h | 2 +- source/client/src/clientSml.c | 69 ++++++++++++++++------------- source/client/src/clientSmlJson.c | 15 +++++-- source/client/src/clientSmlLine.c | 8 +++- source/client/src/clientSmlTelnet.c | 5 ++- tests/parallel_test/cases.task | 4 +- 6 files changed, 64 insertions(+), 39 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index f9cbffa748..c9eb951014 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -251,7 +251,7 @@ int32_t smlClearForRerun(SSmlHandle *info); int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg); uint8_t smlGetTimestampLen(int64_t num); void clearColValArray(SArray* pCols); -void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag); +void smlDestroyTableInfo(void *para); void freeSSmlKv(void* data); int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 92e4d0f8da..5748c06e8b 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -230,6 +230,16 @@ void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tin } } +static void smlDestroySTableMeta(void *para) { + SSmlSTableMeta *meta = *(SSmlSTableMeta**)para; + taosHashCleanup(meta->tagHash); + taosHashCleanup(meta->colHash); + taosArrayDestroy(meta->tags); + taosArrayDestroy(meta->cols); + taosMemoryFree(meta->tableMeta); + taosMemoryFree(meta); +} + SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); if (!meta) { @@ -264,7 +274,7 @@ SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { return meta; cleanup: - taosMemoryFree(meta); + smlDestroySTableMeta(meta); return NULL; } @@ -1093,15 +1103,6 @@ static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) { } } -static void smlDestroySTableMeta(SSmlSTableMeta *meta) { - taosHashCleanup(meta->tagHash); - taosHashCleanup(meta->colHash); - taosArrayDestroy(meta->tags); - taosArrayDestroy(meta->cols); - taosMemoryFree(meta->tableMeta); - taosMemoryFree(meta); -} - static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg) { for (int i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); @@ -1141,7 +1142,8 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols return TSDB_CODE_SUCCESS; } -void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) { +void smlDestroyTableInfo(void *para) { + SSmlTableInfo *tag = *(SSmlTableInfo**)para; for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i); taosHashCleanup(kvHash); @@ -1178,18 +1180,18 @@ void smlDestroyInfo(SSmlHandle *info) { qDestroyQuery(info->pQuery); // destroy info->childTables - SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); - while (oneTable) { - smlDestroyTableInfo(info, *oneTable); - oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); - } +// SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); +// while (oneTable) { +// smlDestroyTableInfo(oneTable); +// oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); +// } // destroy info->superTables - SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); - while (oneSTable) { - smlDestroySTableMeta(*oneSTable); - oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); - } +// SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); +// while (oneSTable) { +// smlDestroySTableMeta(*oneSTable); +// oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); +// } // destroy info->pVgHash taosHashCleanup(info->pVgHash); @@ -1248,6 +1250,8 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) { info->childTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); info->tableUids = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); info->superTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + taosHashSetFreeFp(info->superTables, smlDestroySTableMeta); + taosHashSetFreeFp(info->childTables, smlDestroyTableInfo); info->id = smlGenId(); info->pQuery = smlInitHandle(); @@ -1354,6 +1358,9 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); + if(meta == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); terrno = 0; smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); @@ -1473,18 +1480,18 @@ static void smlPrintStatisticInfo(SSmlHandle *info) { int32_t smlClearForRerun(SSmlHandle *info) { info->reRun = false; // clear info->childTables - SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); - while (oneTable) { - smlDestroyTableInfo(info, *oneTable); - oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); - } +// SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); +// while (oneTable) { +// smlDestroyTableInfo(info, *oneTable); +// oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); +// } // clear info->superTables - SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); - while (oneSTable) { - smlDestroySTableMeta(*oneSTable); - oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); - } +// SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); +// while (oneSTable) { +// smlDestroySTableMeta(*oneSTable); +// oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); +// } taosHashClear(info->childTables); taosHashClear(info->superTables); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 7ccf930964..ad5b199c4e 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -695,6 +695,9 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo return TSDB_CODE_SUCCESS; } sMeta = smlBuildSTableMeta(info->dataFormat); + if(sMeta == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } sMeta->tableMeta = pTableMeta; taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES); for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){ @@ -784,7 +787,7 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); if (tinfo->tableDataCtx == NULL) { smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); - smlDestroyTableInfo(info, tinfo); + smlDestroyTableInfo(&tinfo); return TSDB_CODE_SML_INVALID_DATA; } } @@ -1048,12 +1051,18 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { return TSDB_CODE_TSC_INVALID_JSON; } - info->lineNum = payloadNum; - info->dataFormat = true; + if (unlikely(info->lines != NULL)) { + for (int i = 0; i < info->lineNum; i++) { + taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv); + if (info->lines[i].measureTagsLen != 0) taosMemoryFree(info->lines[i].measureTag); + } taosMemoryFree(info->lines); info->lines = NULL; } + info->lineNum = payloadNum; + info->dataFormat = true; + ret = smlClearForRerun(info); if (ret != TSDB_CODE_SUCCESS) { return ret; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index e79093398e..4bf4ae091f 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -168,6 +168,9 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_SUCCESS; } sMeta = smlBuildSTableMeta(info->dataFormat); + if(sMeta == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } sMeta->tableMeta = pTableMeta; taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES); for (int i = pTableMeta->tableInfo.numOfColumns; @@ -326,7 +329,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin info->currSTableMeta->uid = tinfo->uid; tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); if (tinfo->tableDataCtx == NULL) { - smlDestroyTableInfo(info, tinfo); + smlDestroyTableInfo(&tinfo); smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); return TSDB_CODE_SML_INVALID_DATA; } @@ -372,6 +375,9 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_SUCCESS; } *tmp = smlBuildSTableMeta(info->dataFormat); + if(*tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } (*tmp)->tableMeta = pTableMeta; taosHashPut(info->superTables, currElement->measure, currElement->measureLen, tmp, POINTER_BYTES); diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index 42b8001e59..6dd8842797 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -91,6 +91,9 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS return TSDB_CODE_SUCCESS; } sMeta = smlBuildSTableMeta(info->dataFormat); + if(sMeta == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } sMeta->tableMeta = pTableMeta; taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES); for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){ @@ -212,7 +215,7 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); if (tinfo->tableDataCtx == NULL) { smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); - smlDestroyTableInfo(info, tinfo); + smlDestroyTableInfo(&tinfo); return TSDB_CODE_SML_INVALID_DATA; } } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ab4663e5b3..b09d8f68ec 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -571,7 +571,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py -#,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_stable.py @@ -1089,7 +1089,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 4 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 4 From 8ee1288611ce3a922e296a6c1519df7a95aca43d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 25 May 2023 17:47:55 +0800 Subject: [PATCH 56/97] fix(cache): remove deleted data --- source/dnode/vnode/src/tsdb/tsdbCache.c | 62 ++++++++++++---------- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 6 +-- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index d33f04c145..830f44b411 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -937,14 +937,14 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE } // build keys & multi get from rocks - int num_keys = pTSchema->numOfCols; - char **keys_list = taosMemoryCalloc(num_keys * 2, sizeof(char *)); - size_t *keys_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t)); + int num_keys = pTSchema->numOfCols; + char **keys_list = taosMemoryCalloc(num_keys * 2, sizeof(char *)); + size_t *keys_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t)); + const size_t klen = ROCKS_KEY_LEN; for (int i = 0; i < num_keys; ++i) { int16_t cid = pTSchema->columns[i].colId; - size_t klen = ROCKS_KEY_LEN; - char *keys = taosMemoryCalloc(2, sizeof(SLastKey)); + char *keys = taosMemoryCalloc(2, sizeof(SLastKey)); ((SLastKey *)keys)[0] = (SLastKey){.ltype = 1, .uid = uid, .cid = cid}; ((SLastKey *)keys)[1] = (SLastKey){.ltype = 0, .uid = uid, .cid = cid}; @@ -960,39 +960,35 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE rocksMayWrite(pTsdb, true, false, false); rocksdb_multi_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, num_keys * 2, (const char *const *)keys_list, keys_list_sizes, values_list, values_list_sizes, errs); - for (int i = 0; i < num_keys; ++i) { - taosMemoryFree(keys_list[i]); - } for (int i = 0; i < num_keys * 2; ++i) { - rocksdb_free(errs[i]); + if (errs[i]) { + rocksdb_free(errs[i]); + } } - taosMemoryFree(keys_list); - taosMemoryFree(keys_list_sizes); taosMemoryFree(errs); rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; for (int i = 0; i < num_keys; ++i) { SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i]); if (NULL != pLastCol && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) { - SLastKey *key = &(SLastKey){.ltype = 1, .uid = uid, .cid = pLastCol->colVal.cid}; - size_t klen = ROCKS_KEY_LEN; - - rocksdb_writebatch_delete(wb, (char *)key, klen); - taosLRUCacheErase(pTsdb->lruCache, key, klen); + rocksdb_writebatch_delete(wb, keys_list[i], klen); } + taosLRUCacheErase(pTsdb->lruCache, keys_list[i], klen); pLastCol = tsdbCacheDeserialize(values_list[i + num_keys]); if (NULL != pLastCol && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) { - SLastKey *key = &(SLastKey){.ltype = 0, .uid = uid, .cid = pLastCol->colVal.cid}; - size_t klen = ROCKS_KEY_LEN; - - rocksdb_writebatch_delete(wb, (char *)key, klen); - taosLRUCacheErase(pTsdb->lruCache, key, klen); + rocksdb_writebatch_delete(wb, keys_list[num_keys + i], klen); } + taosLRUCacheErase(pTsdb->lruCache, keys_list[num_keys + i], klen); rocksdb_free(values_list[i]); rocksdb_free(values_list[i + num_keys]); } + for (int i = 0; i < num_keys; ++i) { + taosMemoryFree(keys_list[i]); + } + taosMemoryFree(keys_list); + taosMemoryFree(keys_list_sizes); taosMemoryFree(values_list); taosMemoryFree(values_list_sizes); @@ -1871,10 +1867,14 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie if (isLast && (pColData->flag & HAS_VALUE)) { skipBlock = false; break; - } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { + } /*else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { skipBlock = false; break; - } + }*/ + } + + if (!isLast) { + skipBlock = false; } if (skipBlock) { @@ -1908,6 +1908,9 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie if (checkRemainingRow) { bool skipBlock = true; int inputColIndex = 0; + if (aCols[0] == PRIMARYKEY_TIMESTAMP_COL_ID) { + ++inputColIndex; + } for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) { SColData *pColData = &state->pBlockData->aColData[colIndex]; int16_t cid = pColData->cid; @@ -1916,15 +1919,19 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie if (isLast && (pColData->flag & HAS_VALUE)) { skipBlock = false; break; - } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { + } /*else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { skipBlock = false; break; - } + }*/ ++inputColIndex; } } + if (!isLast) { + skipBlock = false; + } + if (skipBlock) { if (--state->iBlock < 0) { tsdbDataFReaderClose(state->pDataFReader); @@ -2144,7 +2151,8 @@ static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) { if (key->ts > pItemBack->ts) { return false; } else if (key->ts >= pItemFront->ts && key->ts <= pItemBack->ts) { - if (key->version <= pItemFront->version || (key->ts == pItemBack->ts && key->version <= pItemBack->version)) { + // if (key->version <= pItemFront->version || (key->ts == pItemBack->ts && key->version <= pItemBack->version)) { + if (key->version <= pItemFront->version || key->version <= pItemBack->version) { return true; } else { return false; @@ -2959,7 +2967,7 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, do { TSDBROW *pRow = NULL; - nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, true, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray)); + nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, false, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray)); if (!pRow) { break; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 80967a906f..6d223e00c5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -190,9 +190,9 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid tsdbCacheDeleteLast(pTsdb->lruCache, pTbData->uid, eKey); } */ - if (eKey >= pTbData->maxKey && sKey <= pTbData->maxKey) { - tsdbCacheDel(pTsdb, suid, uid, sKey, eKey); - } + // if (eKey >= pTbData->maxKey && sKey <= pTbData->maxKey) { + tsdbCacheDel(pTsdb, suid, uid, sKey, eKey); + //} tsdbTrace("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64 " at version %" PRId64, From c3be7b14bed50e8f680bd98fcb381c405cce9659 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 25 May 2023 17:51:03 +0800 Subject: [PATCH 57/97] fix: set correct function ptr. --- include/libs/executor/executor.h | 1 - include/libs/executor/storageapi.h | 42 +++++++++++----------- source/dnode/snode/inc/sndInt.h | 2 ++ source/dnode/snode/src/snode.c | 4 +-- source/dnode/vnode/inc/vnode.h | 4 ++- source/dnode/vnode/src/meta/metaQuery.c | 4 +-- source/dnode/vnode/src/tq/tqRead.c | 3 ++ source/dnode/vnode/src/vnd/vnodeInitApi.c | 12 +++++++ source/dnode/vnode/src/vnd/vnodeQuery.c | 8 +++++ source/libs/executor/src/executil.c | 6 ++-- source/libs/executor/src/executor.c | 15 ++++---- source/libs/executor/src/operator.c | 4 +-- source/libs/executor/src/querytask.c | 2 +- source/libs/executor/src/scanoperator.c | 42 +++++++++++----------- source/libs/executor/src/sysscanoperator.c | 20 +++++------ 15 files changed, 98 insertions(+), 71 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index c4c2c89fdb..e90bb4688b 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -42,7 +42,6 @@ typedef struct { typedef struct { void* tqReader; -// void* meta; void* config; void* vnode; void* mnd; diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 1240f536ae..5f7a91c057 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -84,7 +84,7 @@ typedef struct SMetaReader { SMetaEntry me; void * pBuf; int32_t szBuf; - struct SStorageAPI *storageAPI; + struct SStoreMeta* pAPI; } SMetaReader; typedef struct SMTbCursor { @@ -256,7 +256,7 @@ typedef struct SStoreCacheReader { void *(*closeReader)(void *pReader); int32_t (*retrieveRows)(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, SArray *pTableUidList); - void (*reuseReader)(void *pReader, void *pTableIdList, int32_t numOfTables); + int32_t (*reuseReader)(void *pReader, void *pTableIdList, int32_t numOfTables); } SStoreCacheReader; /*------------------------------------------------------------------------------------------------------------------*/ @@ -290,6 +290,7 @@ typedef struct SStoreTqReader { int32_t (*tqRetrieveBlock)(); bool (*tqReaderNextBlockInWal)(); bool (*tqNextBlockImpl)(); // todo remove it + SSDataBlock* (*tqGetResultBlock)(); void (*tqReaderSetColIdList)(); int32_t (*tqReaderSetQueryTableList)(); @@ -345,14 +346,6 @@ int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen); */ -typedef struct SStoreMetaReader { - void (*initReader)(SMetaReader *pReader, void *pMeta, int32_t flags); - void (*clearReader)(SMetaReader *pReader); - void (*readerReleaseLock)(SMetaReader *pReader); - int32_t (*getTableEntryByUid)(SMetaReader *pReader, tb_uid_t uid); - int32_t (*getTableEntryByName)(SMetaReader *pReader, const char *name); - int32_t (*getEntryGetUidCache)(SMetaReader *pReader, tb_uid_t uid); -} SStoreMetaReader; typedef struct SStoreMeta { SMTbCursor *(*openTableMetaCursor)(void *pVnode); // metaOpenTbCursor @@ -387,7 +380,7 @@ int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, in void *(*storeGetIndexInfo)(); void *(*getInvertIndex)(void* pVnode); int32_t (*getChildTableList)(void *pVnode, int64_t suid, SArray *list); // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] - int32_t (*storeGetTableList)(); // vnodeGetStbIdList & vnodeGetAllTableList + int32_t (*storeGetTableList)(void* pVnode, int8_t type, SArray* pList); // vnodeGetStbIdList & vnodeGetAllTableList void *storeGetVersionRange; void *storeGetLastTimestamp; @@ -405,9 +398,14 @@ int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list); */ } SStoreMeta; - - - +typedef struct SStoreMetaReader { + void (*initReader)(SMetaReader *pReader, void *pVnode, int32_t flags, SStoreMeta* pAPI); + void (*clearReader)(SMetaReader *pReader); + void (*readerReleaseLock)(SMetaReader *pReader); + int32_t (*getTableEntryByUid)(SMetaReader *pReader, tb_uid_t uid); + int32_t (*getTableEntryByName)(SMetaReader *pReader, const char *name); + int32_t (*getEntryGetUidCache)(SMetaReader *pReader, tb_uid_t uid); +} SStoreMetaReader; typedef struct SUpdateInfo { SArray *pTsBuckets; @@ -507,14 +505,14 @@ typedef struct SStateStore { } SStateStore; typedef struct SStorageAPI { - SStoreMeta metaFn; // todo: refactor - TsdReader tsdReader; - SStoreMetaReader metaReaderFn; - SStoreCacheReader cacheFn; - SStoreSnapshotFn snapshotFn; - SStoreTqReader tqReaderFn; - SStateStore stateStore; - SMetaDataFilterAPI metaFilter; + SStoreMeta metaFn; // todo: refactor + TsdReader tsdReader; + SStoreMetaReader metaReaderFn; + SStoreCacheReader cacheFn; + SStoreSnapshotFn snapshotFn; + SStoreTqReader tqReaderFn; + SStateStore stateStore; + SMetaDataFilterAPI metaFilter; SFunctionStateStore functionStore; } SStorageAPI; diff --git a/source/dnode/snode/inc/sndInt.h b/source/dnode/snode/inc/sndInt.h index 3fcee862a1..68f7f756d5 100644 --- a/source/dnode/snode/inc/sndInt.h +++ b/source/dnode/snode/inc/sndInt.h @@ -53,6 +53,8 @@ int32_t sndStopTaskOfStream(SStreamMeta* pMeta, int64_t streamId); int32_t sndResumeTaskOfStream(SStreamMeta* pMeta, int64_t streamId); #endif +void initStreamStateAPI(SStorageAPI* pAPI); + #ifdef __cplusplus } #endif diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 24e2b13f46..678dd34e4a 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -62,8 +62,7 @@ FAIL: } int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { - ASSERT(pTask->taskLevel == TASK_LEVEL__AGG); - ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0); + ASSERT(pTask->taskLevel == TASK_LEVEL__AGG && taosArrayGetSize(pTask->childEpInfo) != 0); pTask->refCnt = 1; pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; @@ -88,6 +87,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { int32_t numOfChildEp = taosArrayGetSize(pTask->childEpInfo); SReadHandle handle = { .vnode = NULL, .numOfVgroups = numOfChildEp, .pStateBackend = pTask->pState }; + initStreamStateAPI(&handle.api); pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, 0); ASSERT(pTask->exec.pExecutor); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 2723e869ff..e8ae545ea0 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -69,6 +69,7 @@ int64_t vnodeGetSyncHandle(SVnode *pVnode); void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot); void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables); int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen); +int32_t vnodeGetTableList(void* pVnode, int8_t type, SArray* pList); int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list); int32_t vnodeIsCatchUp(SVnode *pVnode); ESyncRole vnodeGetRole(SVnode *pVnode); @@ -105,7 +106,7 @@ typedef struct SMetaEntry SMetaEntry; #define META_READER_NOLOCK 0x1 -void _metaReaderInit(SMetaReader *pReader, void *pVnode, int32_t flags); +void _metaReaderInit(SMetaReader *pReader, void *pVnode, int32_t flags, SStoreMeta* pAPI); void metaReaderReleaseLock(SMetaReader *pReader); void metaReaderClear(SMetaReader *pReader); int32_t metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); @@ -257,6 +258,7 @@ int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id); bool tqNextBlockInWal(STqReader *pReader, const char *idstr); bool tqNextBlockImpl(STqReader *pReader, const char *idstr); SWalReader* tqGetWalReader(STqReader* pReader); +SSDataBlock* tqGetResultBlock (STqReader* pReader); int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, const char *id); int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 255fa10f57..7eb6460af5 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -17,14 +17,14 @@ #include "osMemory.h" #include "tencode.h" -void _metaReaderInit(SMetaReader* pReader, void* pVnode, int32_t flags) { +void _metaReaderInit(SMetaReader* pReader, void* pVnode, int32_t flags, SStoreMeta* pAPI) { SMeta* pMeta = ((SVnode*)pVnode)->pMeta; metaReaderInit(pReader, pMeta, flags); + pReader->pAPI = pAPI; } void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) { memset(pReader, 0, sizeof(*pReader)); - pReader->flags = flags; pReader->pMeta = pMeta; if (pReader->pMeta && !(flags & META_READER_NOLOCK)) { metaRLock(pMeta); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 2521d956a3..dbb7d564f8 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -446,6 +446,9 @@ SWalReader* tqGetWalReader(STqReader* pReader) { return pReader->pWalReader; } +SSDataBlock* tqGetResultBlock (STqReader* pReader) { + return pReader->pResBlock; +} bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { if (pReader->msg.msgStr == NULL) { diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 923e25f73b..5317c0e675 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -25,6 +25,7 @@ static void initStateStoreAPI(SStateStore* pStore); static void initMetaReaderAPI(SStoreMetaReader* pMetaReader); static void initMetaFilterAPI(SMetaDataFilterAPI* pFilter); static void initFunctionStateStore(SFunctionStateStore* pStore); +static void initCacheFn(SStoreCacheReader* pCache); void initStorageAPI(SStorageAPI* pAPI) { initTsdbReaderAPI(&pAPI->tsdReader); @@ -34,6 +35,7 @@ void initStorageAPI(SStorageAPI* pAPI) { initMetaReaderAPI(&pAPI->metaReaderFn); initMetaFilterAPI(&pAPI->metaFilter); initFunctionStateStore(&pAPI->functionStore); + initCacheFn(&pAPI->cacheFn); } void initTsdbReaderAPI(TsdReader* pReader) { @@ -83,6 +85,7 @@ void initMetadataAPI(SStoreMeta* pMeta) { pMeta->getTableNameByUid = metaGetTableNameByUid; pMeta->getTableSchema = tsdbGetTableSchema; // todo refactor + pMeta->storeGetTableList = vnodeGetTableList; } void initTqAPI(SStoreTqReader* pTq) { @@ -109,6 +112,8 @@ void initTqAPI(SStoreTqReader* pTq) { pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it + pTq->tqGetResultBlock = tqGetResultBlock; + pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; } @@ -213,4 +218,11 @@ void initMetaFilterAPI(SMetaDataFilterAPI* pFilter) { void initFunctionStateStore(SFunctionStateStore* pStore) { pStore->streamStateFuncPut = streamStateFuncPut; pStore->streamStateFuncGet = streamStateFuncGet; +} + +void initCacheFn(SStoreCacheReader* pCache) { + pCache->openReader = tsdbCacherowsReaderOpen; + pCache->closeReader = tsdbCacherowsReaderClose; + pCache->retrieveRows = tsdbRetrieveCacheRows; + pCache->reuseReader = tsdbReuseCacherowsReader; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index ac3e632172..022fc4c951 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -431,6 +431,14 @@ void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t* num } } +int32_t vnodeGetTableList(void* pVnode, int8_t type, SArray* pList) { + if (type == TSDB_SUPER_TABLE) { + return vnodeGetStbIdList(pVnode, 0, pList); + } else { + return TSDB_CODE_INVALID_PARA; + } +} + int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) { SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, uid, 1); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index ebe3d69dbb..562a6daf8a 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -263,7 +263,7 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { STagVal tagVal = {0}; tagVal.cid = pSColumnNode->colId; - const char* p = mr->storageAPI->metaFn.extractTagVal(mr->me.ctbEntry.pTags, pSColumnNode->node.resType.type, &tagVal); + const char* p = mr->pAPI->extractTagVal(mr->me.ctbEntry.pTags, pSColumnNode->node.resType.type, &tagVal); if (p == NULL) { res->node.resType.type = TSDB_DATA_TYPE_NULL; } else if (pSColumnNode->node.resType.type == TSDB_DATA_TYPE_JSON) { @@ -306,7 +306,7 @@ int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, int32_t code = TSDB_CODE_SUCCESS; SMetaReader mr = {0}; - pAPI->metaReaderFn.initReader(&mr, metaHandle, 0); + pAPI->metaReaderFn.initReader(&mr, metaHandle, 0, &pAPI->metaFn); code = pAPI->metaReaderFn.getEntryGetUidCache(&mr, info->uid); if (TSDB_CODE_SUCCESS != code) { pAPI->metaReaderFn.clearReader(&mr); @@ -1168,7 +1168,7 @@ int32_t getGroupIdFromTagsVal(void* pVnode, uint64_t uid, SNodeList* pGroupNode, SStorageAPI* pAPI) { SMetaReader mr = {0}; - pAPI->metaReaderFn.initReader(&mr, pVnode, 0); + pAPI->metaReaderFn.initReader(&mr, pVnode, 0, &pAPI->metaFn); if (pAPI->metaReaderFn.getEntryGetUidCache(&mr, uid) != 0) { // table not exist pAPI->metaReaderFn.clearReader(&mr); return TSDB_CODE_PAR_TABLE_NOT_EXIST; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 53fad6503e..14f5c3f4fa 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -341,7 +341,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S // let's discard the tables those are not created according to the queried super table. SMetaReader mr = {0}; - pAPI->metaReaderFn.initReader(&mr, pScanInfo->readHandle.vnode, 0); + pAPI->metaReaderFn.initReader(&mr, pScanInfo->readHandle.vnode, 0, &pAPI->metaFn); for (int32_t i = 0; i < numOfUids; ++i) { uint64_t* id = (uint64_t*)taosArrayGet(tableIdList, i); @@ -1091,12 +1091,13 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pScanBaseInfo->dataReader); pScanBaseInfo->dataReader = NULL; - ASSERT(0); -// walReaderVerifyOffset(pInfo->tqReader->pWalReader, pOffset); -// if (tqReaderSeek(pInfo->tqReader, pOffset->version + 1, id) < 0) { -// qError("tqReaderSeek failed ver:%" PRId64 ", %s", pOffset->version + 1, id); -// return -1; -// } + SStoreTqReader* pReaderAPI = &pTaskInfo->storageAPI.tqReaderFn; + SWalReader* pWalReader = pReaderAPI->tqReaderGetWalReader(pInfo->tqReader); + walReaderVerifyOffset(pWalReader, pOffset); + if (pReaderAPI->tqReaderSeek(pInfo->tqReader, pOffset->version + 1, id) < 0) { + qError("tqReaderSeek failed ver:%" PRId64 ", %s", pOffset->version + 1, id); + return -1; + } } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { // iterate all tables from tableInfoList, and retrieve rows from each table one-by-one // those data are from the snapshot in tsdb, besides the data in the wal file. diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 8521fb623e..935e6a0b8b 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -381,9 +381,9 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR if (pBlockNode->tableType == TSDB_SUPER_TABLE) { SArray* pList = taosArrayInit(4, sizeof(STableKeyInfo)); - int32_t code = pTaskInfo->storageAPI.metaFn.storeGetTableList(pHandle->vnode, pBlockNode->uid, pList); + int32_t code = pTaskInfo->storageAPI.metaFn.getChildTableList(pHandle->vnode, pBlockNode->uid, pList); if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = terrno; + pTaskInfo->code = code; return NULL; } diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c index f79966a021..22d171e74a 100644 --- a/source/libs/executor/src/querytask.c +++ b/source/libs/executor/src/querytask.c @@ -122,7 +122,7 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo SStorageAPI* pAPI = &pTaskInfo->storageAPI; - pAPI->metaReaderFn.initReader(&mr, pHandle->vnode, 0); + pAPI->metaReaderFn.initReader(&mr, pHandle->vnode, 0, &pAPI->metaFn); int32_t code = pAPI->metaReaderFn.getEntryGetUidCache(&mr, pScanNode->uid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get the table meta, uid:0x%" PRIx64 ", suid:0x%" PRIx64 ", %s", pScanNode->uid, pScanNode->suid, diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 66e200c81b..8f11cd769a 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -531,7 +531,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int // 1. check if it is existed in meta cache if (pCache == NULL) { - pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0); + pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0, &pHandle->api.metaFn); code = pHandle->api.metaReaderFn.getEntryGetUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { // when encounter the TSDB_CODE_PAR_TABLE_NOT_EXIST error, we proceed. @@ -560,7 +560,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int h = taosLRUCacheLookup(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid)); if (h == NULL) { - pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0); + pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0, &pHandle->api.metaFn); code = pHandle->api.metaReaderFn.getEntryGetUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { @@ -1330,9 +1330,9 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr SColumnInfoData* pDestGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX); SColumnInfoData* pDestCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); SColumnInfoData* pDestCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); - int64_t version = pSrcBlock->info.version - 1; + int64_t ver = pSrcBlock->info.version - 1; for (int32_t i = 0; i < pSrcBlock->info.rows; i++) { - uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], version); + uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], ver); // gap must be 0. SSessionKey startWin = {0}; getCurSessionWindow(pInfo->windowSup.pStreamAggSup, startData[i], startData[i], groupId, &startWin); @@ -1378,13 +1378,13 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS ASSERT(pSrcStartTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); TSKEY* srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData; TSKEY* srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData; - int64_t version = pSrcBlock->info.version - 1; + int64_t ver = pSrcBlock->info.version - 1; if (pInfo->partitionSup.needCalc && srcStartTsCol[0] != srcEndTsCol[0]) { uint64_t srcUid = srcUidData[0]; TSKEY startTs = srcStartTsCol[0]; TSKEY endTs = srcEndTsCol[0]; - SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, srcUid, startTs, endTs, version); + SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, srcUid, startTs, endTs, ver); printDataBlock(pPreRes, "pre res"); blockDataCleanup(pSrcBlock); int32_t code = blockDataEnsureCapacity(pSrcBlock, pPreRes->info.rows); @@ -1422,7 +1422,7 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS uint64_t srcUid = srcUidData[i]; uint64_t groupId = srcGp[i]; if (groupId == 0) { - groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version); + groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], ver); } TSKEY calStartTs = srcStartTsCol[i]; colDataSetVal(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false); @@ -1459,13 +1459,13 @@ static int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pS ASSERT(pSrcStartTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); TSKEY* srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData; TSKEY* srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData; - int64_t version = pSrcBlock->info.version - 1; + int64_t ver = pSrcBlock->info.version - 1; for (int32_t i = 0; i < pSrcBlock->info.rows; i++) { uint64_t srcUid = srcUidData[i]; uint64_t groupId = srcGp[i]; char* tbname[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN] = {0}; if (groupId == 0) { - groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version); + groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], ver); } if (pInfo->tbnameCalSup.pExprInfo) { void* parTbname = NULL; @@ -1676,7 +1676,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { while (1) { bool hasResult = pAPI->tqReaderFn.tqReaderNextBlockInWal(pInfo->tqReader, id); - SSDataBlock* pRes = NULL; + SSDataBlock* pRes = pAPI->tqReaderFn.tqGetResultBlock(pInfo->tqReader); struct SWalReader* pWalReader = pAPI->tqReaderFn.tqReaderGetWalReader(pInfo->tqReader); // curVersion move to next, so currentOffset = curVersion - 1 @@ -2558,7 +2558,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { char str[512] = {0}; int32_t count = 0; SMetaReader mr = {0}; - pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, 0); + pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, 0, &pAPI->metaFn); while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { doTagScanOneTable(pOperator, pRes, count, &mr, &pTaskInfo->storageAPI); @@ -3185,6 +3185,7 @@ int32_t getTableCountScanSupp(SNodeList* groupTags, SName* tableName, SNodeList* qError("%s get table count scan supp. get inputs error", GET_TASKID(taskInfo)); return code; } + supp->dbNameSlotId = -1; supp->stbNameSlotId = -1; supp->tbCountSlotId = -1; @@ -3194,6 +3195,7 @@ int32_t getTableCountScanSupp(SNodeList* groupTags, SName* tableName, SNodeList* qError("%s get table count scan supp. get group tags slot id error", GET_TASKID(taskInfo)); return code; } + code = tblCountScanGetCountSlotId(pseudoCols, supp); if (code != TSDB_CODE_SUCCESS) { qError("%s get table count scan supp. get count error", GET_TASKID(taskInfo)); @@ -3378,7 +3380,7 @@ static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountSca if (pSupp->groupByStbName) { if (pInfo->stbUidList == NULL) { pInfo->stbUidList = taosArrayInit(16, sizeof(tb_uid_t)); - if (pAPI->metaFn.storeGetTableList(pInfo->readHandle.vnode, 0, pInfo->stbUidList, TSDB_SUPER_TABLE) < 0) { + if (pAPI->metaFn.storeGetTableList(pInfo->readHandle.vnode, TSDB_SUPER_TABLE, pInfo->stbUidList) < 0) { qError("vgId:%d, failed to get stb id list error: %s", vgId, terrstr()); } } @@ -3412,7 +3414,7 @@ static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanO if (strlen(pSupp->dbNameFilter) != 0) { if (strlen(pSupp->stbNameFilter) != 0) { - tb_uid_t uid = 0; + uint64_t uid = 0; pAPI->metaFn.getTableUidByName(pInfo->readHandle.vnode, pSupp->stbNameFilter, &uid); int64_t numOfChildTables = 0; @@ -3420,7 +3422,8 @@ static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanO fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, numOfChildTables, pRes); } else { - int64_t tbNumVnode = 0;//metaGetTbNum(pInfo->readHandle.vnode); + int64_t tbNumVnode = 0; + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, &tbNumVnode, NULL); fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } } else { @@ -3444,6 +3447,7 @@ static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, S int64_t numOfTables = 0;//metaGetNtbNum(pInfo->readHandle.vnode); pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, NULL, &numOfTables); + if (numOfTables != 0) { fillTableCountScanDataBlock(pSupp, dbName, "", numOfTables, pRes); } @@ -3456,18 +3460,16 @@ static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, S char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; if (pSupp->groupByDbName) { - snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, stbName); + snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, varDataVal(stbName)); } else { - snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s", stbName); + snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s", varDataVal(stbName)); } uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); pRes->info.id.groupId = groupId; - SMetaStbStats stats = {0}; -// metaGetStbStats(pInfo->readHandle.vnode, stbUid, &stats); - int64_t ctbNum = stats.ctbNum; - + int64_t ctbNum = 0; + int32_t code = pAPI->metaFn.getNumOfChildTables(pInfo->readHandle.vnode, stbUid, &ctbNum); fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes); } diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 40b45c25cd..fd766e21f8 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -466,7 +466,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { STR_TO_VARSTR(tableName, pInfo->req.filterTb); SMetaReader smrTable = {0}; - pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, 0); + pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn); int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrTable, pInfo->req.filterTb); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly @@ -486,7 +486,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { if (smrTable.me.type == TSDB_CHILD_TABLE) { int64_t suid = smrTable.me.ctbEntry.suid; pAPI->metaReaderFn.clearReader(&smrTable); - pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, 0); + pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn); code = pAPI->metaReaderFn.getTableEntryByUid(&smrTable, suid); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly @@ -569,7 +569,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { schemaRow = *(SSchemaWrapper**)schema; } else { SMetaReader smrSuperTable = {0}; - pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, 0); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn); int code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly @@ -658,7 +658,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { STR_TO_VARSTR(tableName, condTableName); SMetaReader smrChildTable = {0}; - pAPI->metaReaderFn.initReader(&smrChildTable, pInfo->readHandle.vnode, 0); + pAPI->metaReaderFn.initReader(&smrChildTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn); int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrChildTable, condTableName); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly @@ -676,7 +676,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { } SMetaReader smrSuperTable = {0}; - pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, META_READER_NOLOCK); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, META_READER_NOLOCK, &pAPI->metaFn); code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByUid @@ -715,7 +715,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); SMetaReader smrSuperTable = {0}; - pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, 0); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn); uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid); if (code != TSDB_CODE_SUCCESS) { @@ -1131,7 +1131,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { tb_uid_t* uid = taosArrayGet(pIdx->uids, i); SMetaReader mr = {0}; - pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, 0); + pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, 0, &pAPI->metaFn); ret = pAPI->metaReaderFn.getTableEntryByUid(&mr, *uid); if (ret < 0) { pAPI->metaReaderFn.clearReader(&mr); @@ -1159,7 +1159,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false); SMetaReader mr1 = {0}; - pAPI->metaReaderFn.initReader(&mr1, pInfo->readHandle.vnode, META_READER_NOLOCK); + pAPI->metaReaderFn.initReader(&mr1, pInfo->readHandle.vnode, META_READER_NOLOCK, &pAPI->metaFn); int64_t suid = mr.me.ctbEntry.suid; int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr1, suid); @@ -1338,7 +1338,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false); SMetaReader mr = {0}; - pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, META_READER_NOLOCK); + pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, META_READER_NOLOCK, &pAPI->metaFn); uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr, suid); @@ -2148,7 +2148,7 @@ static int32_t doGetTableRowSize(SReadHandle *pHandle, uint64_t uid, int32_t* ro *rowLen = 0; SMetaReader mr = {0}; - pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0); + pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0, &pHandle->api.metaFn); int32_t code = pHandle->api.metaReaderFn.getTableEntryByUid(&mr, uid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", uid, tstrerror(terrno), idstr); From 40fa0e06201153504d06d1bc0ff4aef896d3cdc9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 25 May 2023 18:07:36 +0800 Subject: [PATCH 58/97] fix:memory leak --- source/client/src/clientSml.c | 3 ++- source/client/src/clientSmlJson.c | 1 + source/client/src/clientSmlLine.c | 2 ++ source/client/src/clientSmlTelnet.c | 1 + 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 5748c06e8b..85cba9cda6 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -236,7 +236,7 @@ static void smlDestroySTableMeta(void *para) { taosHashCleanup(meta->colHash); taosArrayDestroy(meta->tags); taosArrayDestroy(meta->cols); - taosMemoryFree(meta->tableMeta); + taosMemoryFreeClear(meta->tableMeta); taosMemoryFree(meta); } @@ -1045,6 +1045,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { } } + taosMemoryFreeClear(sTableData->tableMeta); sTableData->tableMeta = pTableMeta; uDebug("SML:0x%" PRIx64 "modify schema uid:%" PRIu64 ", sversion:%d, tversion:%d", info->id, pTableMeta->uid, pTableMeta->sversion, pTableMeta->tversion) tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index ad5b199c4e..0f59505f8c 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -696,6 +696,7 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo } sMeta = smlBuildSTableMeta(info->dataFormat); if(sMeta == NULL){ + taosMemoryFreeClear(pTableMeta); return TSDB_CODE_OUT_OF_MEMORY; } sMeta->tableMeta = pTableMeta; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 4bf4ae091f..40e014458e 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -169,6 +169,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin } sMeta = smlBuildSTableMeta(info->dataFormat); if(sMeta == NULL){ + taosMemoryFreeClear(pTableMeta); return TSDB_CODE_OUT_OF_MEMORY; } sMeta->tableMeta = pTableMeta; @@ -376,6 +377,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin } *tmp = smlBuildSTableMeta(info->dataFormat); if(*tmp == NULL){ + taosMemoryFreeClear(pTableMeta); return TSDB_CODE_OUT_OF_MEMORY; } (*tmp)->tableMeta = pTableMeta; diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index 6dd8842797..b3f45a3107 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -92,6 +92,7 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS } sMeta = smlBuildSTableMeta(info->dataFormat); if(sMeta == NULL){ + taosMemoryFreeClear(pTableMeta); return TSDB_CODE_OUT_OF_MEMORY; } sMeta->tableMeta = pTableMeta; From 30de31e37431ac2e793769141692b0680d6e836f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 25 May 2023 18:17:09 +0800 Subject: [PATCH 59/97] fix(test): fix unit test link error. --- source/libs/stream/test/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/test/CMakeLists.txt b/source/libs/stream/test/CMakeLists.txt index 160cddbd4c..049bfbbb3a 100644 --- a/source/libs/stream/test/CMakeLists.txt +++ b/source/libs/stream/test/CMakeLists.txt @@ -15,11 +15,11 @@ ADD_EXECUTABLE(streamUpdateTest "tstreamUpdateTest.cpp") IF (TD_GRANT) TARGET_LINK_LIBRARIES(streamUpdateTest - PUBLIC os util common gtest gtest_main stream executor grant + PUBLIC os util common gtest gtest_main stream executor index grant ) ELSE () TARGET_LINK_LIBRARIES(streamUpdateTest - PUBLIC os util common gtest gtest_main stream executor + PUBLIC os util common gtest gtest_main stream executor index ) ENDIF() From 1090afd10884d91de4f074e793685100faa7e45e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 25 May 2023 18:46:53 +0800 Subject: [PATCH 60/97] fix(stream): add new file. --- source/dnode/snode/src/snodeInitApi.c | 109 ++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 source/dnode/snode/src/snodeInitApi.c diff --git a/source/dnode/snode/src/snodeInitApi.c b/source/dnode/snode/src/snodeInitApi.c new file mode 100644 index 0000000000..f5e9245252 --- /dev/null +++ b/source/dnode/snode/src/snodeInitApi.c @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "storageapi.h" +#include "tstreamUpdate.h" +#include "streamState.h" + +static void initStateStoreAPI(SStateStore* pStore); +static void initFunctionStateStore(SFunctionStateStore* pStore); + +void initStreamStateAPI(SStorageAPI* pAPI) { + initStateStoreAPI(&pAPI->stateStore); + initFunctionStateStore(&pAPI->functionStore); +} + +void initStateStoreAPI(SStateStore* pStore) { + pStore->streamFileStateInit = streamFileStateInit; + pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF; + + pStore->streamStateGetByPos = streamStateGetByPos; + + pStore->streamStatePutParName = streamStatePutParName; + pStore->streamStateGetParName = streamStateGetParName; + + pStore->streamStateAddIfNotExist = streamStateAddIfNotExist; + pStore->streamStateReleaseBuf = streamStateReleaseBuf; + pStore->streamStateFreeVal = streamStateFreeVal; + + pStore->streamStatePut = streamStatePut; + pStore->streamStateGet = streamStateGet; + pStore->streamStateCheck = streamStateCheck; + pStore->streamStateGetByPos = streamStateGetByPos; + pStore->streamStateDel = streamStateDel; + pStore->streamStateClear = streamStateClear; + pStore->streamStateSaveInfo = streamStateSaveInfo; + pStore->streamStateGetInfo = streamStateGetInfo; + pStore->streamStateSetNumber = streamStateSetNumber; + + pStore->streamStateFillPut = streamStateFillPut; + pStore->streamStateFillGet = streamStateFillGet; + pStore->streamStateFillDel = streamStateFillDel; + + pStore->streamStateCurNext = streamStateCurNext; + pStore->streamStateCurPrev = streamStateCurPrev; + + pStore->streamStateGetAndCheckCur = streamStateGetAndCheckCur; + pStore->streamStateSeekKeyNext = streamStateSeekKeyNext; + pStore->streamStateFillSeekKeyNext = streamStateFillSeekKeyNext; + pStore->streamStateFillSeekKeyPrev = streamStateFillSeekKeyPrev; + pStore->streamStateFreeCur = streamStateFreeCur; + + pStore->streamStateGetGroupKVByCur = streamStateGetGroupKVByCur; + pStore->streamStateGetKVByCur = streamStateGetKVByCur; + + pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist; + pStore->streamStateSessionPut = streamStateSessionPut; + pStore->streamStateSessionGet = streamStateSessionGet; + pStore->streamStateSessionDel = streamStateSessionDel; + pStore->streamStateSessionClear = streamStateSessionClear; + pStore->streamStateSessionGetKVByCur = streamStateSessionGetKVByCur; + pStore->streamStateStateAddIfNotExist = streamStateStateAddIfNotExist; + pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange; + + pStore->updateInfoInit = updateInfoInit; + pStore->updateInfoFillBlockData = updateInfoFillBlockData; + pStore->updateInfoIsUpdated = updateInfoIsUpdated; + pStore->updateInfoIsTableInserted = updateInfoIsTableInserted; + pStore->updateInfoDestroy = updateInfoDestroy; + + pStore->updateInfoInitP = updateInfoInitP; + pStore->updateInfoAddCloseWindowSBF = updateInfoAddCloseWindowSBF; + pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF; + pStore->updateInfoSerialize = updateInfoSerialize; + pStore->updateInfoDeserialize = updateInfoDeserialize; + + pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext; + pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev; + pStore->streamStateSessionSeekKeyCurrentNext = streamStateSessionSeekKeyCurrentNext; + + pStore->streamFileStateInit = streamFileStateInit; + + pStore->streamFileStateDestroy = streamFileStateDestroy; + pStore->streamFileStateClear = streamFileStateClear; + pStore->needClearDiskBuff = needClearDiskBuff; + + pStore->streamStateOpen = streamStateOpen; + pStore->streamStateClose = streamStateClose; + pStore->streamStateBegin = streamStateBegin; + pStore->streamStateCommit = streamStateCommit; + pStore->streamStateDestroy= streamStateDestroy; + pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint; +} + +void initFunctionStateStore(SFunctionStateStore* pStore) { + pStore->streamStateFuncPut = streamStateFuncPut; + pStore->streamStateFuncGet = streamStateFuncGet; +} \ No newline at end of file From 34224fbaa6446f16e0389004aa5bd0abb161b499 Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Thu, 25 May 2023 18:58:13 +0800 Subject: [PATCH 61/97] enh(taosAdapter): change taosAdapter to 3.0 branch --- cmake/taosadapter_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 391972960a..ef6ed4af1d 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG 283b50d + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 1e22d875b3d948dfe7c65d8a03481b7c017f7980 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 25 May 2023 20:00:26 +0800 Subject: [PATCH 62/97] fix: fix dead lock. --- source/dnode/vnode/src/meta/metaQuery.c | 1 + source/dnode/vnode/src/vnd/vnodeInitApi.c | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 7eb6460af5..26182baf7b 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -26,6 +26,7 @@ void _metaReaderInit(SMetaReader* pReader, void* pVnode, int32_t flags, SStoreMe void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) { memset(pReader, 0, sizeof(*pReader)); pReader->pMeta = pMeta; + pReader->flags = flags; if (pReader->pMeta && !(flags & META_READER_NOLOCK)) { metaRLock(pMeta); } diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 5317c0e675..00c1dacdfd 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -200,7 +200,6 @@ void initMetaReaderAPI(SStoreMetaReader* pMetaReader) { pMetaReader->clearReader = metaReaderClear; pMetaReader->getTableEntryByUid = metaReaderGetTableEntryByUid; - pMetaReader->clearReader = metaReaderClear; pMetaReader->getEntryGetUidCache = metaReaderGetTableEntryByUidCache; pMetaReader->getTableEntryByName = metaGetTableEntryByName; From efa1570b6e076c79e72317f4471d39da02a28c29 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 25 May 2023 20:10:40 +0800 Subject: [PATCH 63/97] prompt message when 2 mnode --- source/dnode/mnode/impl/src/mndMnode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 19c3d59167..d0b10a5768 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -695,7 +695,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode if (totalMnodes == 2) { if (force) { mError("cant't force drop dnode, since a mnode on it and replica is 2"); - terrno = TSDB_CODE_DNODE_OFFLINE; + terrno = TSDB_CODE_MNODE_ONLY_TWO_MNODE; return -1; } mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes); From 469b2b573c1ac673cbe54307d3613eafa133319c Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 25 May 2023 21:49:28 +0800 Subject: [PATCH 64/97] fix: fix error during emptyTsRange.sim --- source/libs/scalar/src/filter.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index ae4712b9ba..8a97f5fae2 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3546,7 +3546,7 @@ bool fltSclLessPoint(SFltSclPoint *pt1, SFltSclPoint *pt2) { if (pt1->start && pt2->start) { return !pt1->excl && pt2->excl; } else if (pt1->start) { - return pt1->excl && !pt2->excl; + return !pt1->excl && !pt2->excl; } else if (pt2->start) { return pt1->excl || pt2->excl; } @@ -3971,7 +3971,7 @@ static int32_t fltSclGetDatumValueFromPoint(SFltSclPoint *point, SFltSclDatum *d getDataMin(d->type.type, &(d->i)); } - if ((point->val.kind == FLT_SCL_DATUM_KIND_INT64) || (point->val.kind = FLT_SCL_DATUM_KIND_UINT64)) { + if (IS_NUMERIC_TYPE(d->type.type)) { if (point->excl) { if (point->start) { ++d->i; @@ -3980,7 +3980,7 @@ static int32_t fltSclGetDatumValueFromPoint(SFltSclPoint *point, SFltSclDatum *d } } } else { - qError("not supported kind %d when get datum from point", point->val.kind); + qError("not supported type %d when get datum from point", d->type.type); } return TSDB_CODE_SUCCESS; } @@ -4001,14 +4001,17 @@ int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) { if (taosArrayGetSize(points) == 2) { SFltSclPoint *startPt = taosArrayGet(points, 0); SFltSclPoint *endPt = taosArrayGet(points, 1); - SFltSclDatum start; - SFltSclDatum end; + SFltSclDatum start; + SFltSclDatum end; fltSclGetDatumValueFromPoint(startPt, &start); fltSclGetDatumValueFromPoint(endPt, &end); win->skey = start.i; win->ekey = end.i; *isStrict = true; goto _return; + } else if (taosArrayGetSize(points) == 0) { + *win = TSWINDOW_DESC_INITIALIZER; + goto _return; } } *win = TSWINDOW_INITIALIZER; From cbbd09b85c0fbe7038cfbfd3402888b217638acf Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 25 May 2023 22:37:49 +0800 Subject: [PATCH 65/97] fix: for logical and condition, each element shall be collectable operator --- source/libs/scalar/src/filter.c | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 8a97f5fae2..d45abb1b87 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3971,7 +3971,7 @@ static int32_t fltSclGetDatumValueFromPoint(SFltSclPoint *point, SFltSclDatum *d getDataMin(d->type.type, &(d->i)); } - if (IS_NUMERIC_TYPE(d->type.type)) { + if (IS_INTEGER_TYPE(d->type.type) || IS_TIMESTAMP_TYPE(d->type.type)) { if (point->excl) { if (point->start) { ++d->i; @@ -4460,26 +4460,35 @@ int32_t fltSclProcessCNF(SArray *sclOpListCNF, SArray *colRangeList) { return TSDB_CODE_SUCCESS; } -static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) { +static bool fltSclIsCollectableNode(SNode *pNode) { if (nodeType(pNode) != QUERY_NODE_OPERATOR) { - return TSDB_CODE_SUCCESS; + return false; } SOperatorNode *pOper = (SOperatorNode *)pNode; if (pOper->pLeft == NULL || pOper->pRight == NULL) { - return TSDB_CODE_SUCCESS; + return false; } if (!(pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL || pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL || pOper->opType == OP_TYPE_NOT_EQUAL || pOper->opType == OP_TYPE_EQUAL)) { - return TSDB_CODE_SUCCESS; + return false; } if (!(nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_VALUE)) { + return false; + } + return true; +} + +static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) { + if (!fltSclIsCollectableNode(pNode)) { return TSDB_CODE_SUCCESS; } + SOperatorNode *pOper = (SOperatorNode *)pNode; + SValueNode *valNode = (SValueNode *)pOper->pRight; if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { SFltSclOperator sclOp = {.colNode = (SColumnNode *)nodesCloneNode(pOper->pLeft), @@ -4501,6 +4510,11 @@ static int32_t fltSclCollectOperatorsFromLogicCond(SNode *pNode, SArray *sclOpLi return TSDB_CODE_SUCCESS; } SNode *pExpr = NULL; + FOREACH(pExpr, pLogicCond->pParameterList) { + if (!fltSclIsCollectableNode(pExpr)) { + return TSDB_CODE_SUCCESS; + } + } FOREACH(pExpr, pLogicCond->pParameterList) { fltSclCollectOperatorFromNode(pExpr, sclOpList); } return TSDB_CODE_SUCCESS; } From e24ce2111ff85eac87306edfd49e869155554e37 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 26 May 2023 00:00:07 +0800 Subject: [PATCH 66/97] fix: set function ptr. --- include/libs/executor/storageapi.h | 14 +++++++------- source/dnode/vnode/inc/vnode.h | 4 ++-- source/dnode/vnode/src/meta/metaSnapshot.c | 6 +++--- source/dnode/vnode/src/tq/tqMeta.c | 2 ++ source/dnode/vnode/src/vnd/vnodeInitApi.c | 10 +++++++++- source/libs/executor/src/executor.c | 2 +- source/libs/executor/src/operator.c | 7 ++++--- source/libs/executor/src/scanoperator.c | 9 +++++---- 8 files changed, 33 insertions(+), 21 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 5f7a91c057..9ed47c94f8 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -196,8 +196,8 @@ typedef struct { // int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id); // bool tqNextBlockInWal(STqReader* pReader, const char* idstr); // bool tqNextBlockImpl(STqReader *pReader, const char* idstr); -// int32_t getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); -// SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx); +// int32_t getTableInfoFromSnapshot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); +// SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext *ctx); // int32_t setForSnapShot(SSnapContext *ctx, int64_t uid); // int32_t destroySnapContext(SSnapContext *ctx); @@ -310,15 +310,15 @@ typedef struct SStoreTqReader { typedef struct SStoreSnapshotFn { /* - int32_t getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); - SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx); + int32_t getTableInfoFromSnapshot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); + SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext *ctx); int32_t setForSnapShot(SSnapContext *ctx, int64_t uid); int32_t destroySnapContext(SSnapContext *ctx); */ int32_t (*createSnapshot)(); - void (*destroySnapshot)(); - SMetaTableInfo (*getTableInfoFromSnapshot)(); - int32_t (*getMetaInfoFromSnapshot)(); + int32_t (*destroySnapshot)(); + SMetaTableInfo (*getMetaTableInfoFromSnapshot)(); + int32_t (*getTableInfoFromSnapshot)(); } SStoreSnapshotFn; /** diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index e8ae545ea0..72c8dce52f 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -282,8 +282,8 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData); int32_t buildSnapContext(SVnode *pVnode, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta, SSnapContext **ctxRet); -int32_t getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); -SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx); +int32_t getTableInfoFromSnapshot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); +SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext *ctx); int32_t setForSnapShot(SSnapContext *ctx, int64_t uid); int32_t destroySnapContext(SSnapContext *ctx); diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index ff4355fdf8..f4e930e509 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -466,7 +466,7 @@ int32_t setForSnapShot(SSnapContext* ctx, int64_t uid) { return c; } -int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid) { +int32_t getTableInfoFromSnapshot(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid) { int32_t ret = 0; void* pKey = NULL; void* pVal = NULL; @@ -598,7 +598,7 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in return ret; } -SMetaTableInfo getUidfromSnapShot(SSnapContext* ctx) { +SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext* ctx) { SMetaTableInfo result = {0}; void* pKey = NULL; void* pVal = NULL; @@ -619,7 +619,7 @@ SMetaTableInfo getUidfromSnapShot(SSnapContext* ctx) { int32_t ret = MoveToPosition(ctx, idInfo->version, *uidTmp); if (ret != 0) { - metaDebug("tmqsnap getUidfromSnapShot not exist uid:%" PRIi64 " version:%" PRIi64, *uidTmp, idInfo->version); + metaDebug("tmqsnap getMetaTableInfoFromSnapshot not exist uid:%" PRIi64 " version:%" PRIi64, *uidTmp, idInfo->version); continue; } tdbTbcGet((TBC*)ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen); diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index d5cfa86e4c..be5b5706e2 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -304,6 +304,8 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { .version = handle.snapshotVer }; + initStorageAPI(&reader.api); + if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { handle.execHandle.task = qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, vgId, &handle.execHandle.numOfCols, 0); diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 00c1dacdfd..1cc985f7df 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -26,6 +26,7 @@ static void initMetaReaderAPI(SStoreMetaReader* pMetaReader); static void initMetaFilterAPI(SMetaDataFilterAPI* pFilter); static void initFunctionStateStore(SFunctionStateStore* pStore); static void initCacheFn(SStoreCacheReader* pCache); +static void initSnapshotFn(SStoreSnapshotFn* pSnapshot); void initStorageAPI(SStorageAPI* pAPI) { initTsdbReaderAPI(&pAPI->tsdReader); @@ -36,6 +37,7 @@ void initStorageAPI(SStorageAPI* pAPI) { initMetaFilterAPI(&pAPI->metaFilter); initFunctionStateStore(&pAPI->functionStore); initCacheFn(&pAPI->cacheFn); + initSnapshotFn(&pAPI->snapshotFn); } void initTsdbReaderAPI(TsdReader* pReader) { @@ -69,7 +71,6 @@ void initMetadataAPI(SStoreMeta* pMeta) { pMeta->getBasicInfo = vnodeGetInfo; pMeta->getNumOfChildTables = metaGetStbStats; -// pMeta->getNumOfRowsInMem = tsdbGetNumOfRowsInMemTable; pMeta->getChildTableList = vnodeGetCtbIdList; @@ -224,4 +225,11 @@ void initCacheFn(SStoreCacheReader* pCache) { pCache->closeReader = tsdbCacherowsReaderClose; pCache->retrieveRows = tsdbRetrieveCacheRows; pCache->reuseReader = tsdbReuseCacherowsReader; +} + +void initSnapshotFn(SStoreSnapshotFn* pSnapshot) { + pSnapshot->createSnapshot = setForSnapShot; + pSnapshot->destroySnapshot = destroySnapContext; + pSnapshot->getMetaTableInfoFromSnapshot = getMetaTableInfoFromSnapshot; + pSnapshot->getTableInfoFromSnapshot = getTableInfoFromSnapshot; } \ No newline at end of file diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 14f5c3f4fa..fb35b211c9 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1190,7 +1190,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT return -1; } - SMetaTableInfo mtInfo = pTaskInfo->storageAPI.snapshotFn.getTableInfoFromSnapshot(sContext); + SMetaTableInfo mtInfo = pTaskInfo->storageAPI.snapshotFn.getMetaTableInfoFromSnapshot(sContext); pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pInfo->dataReader); pInfo->dataReader = NULL; diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 935e6a0b8b..730252c7ee 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -380,17 +380,18 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR STableListInfo* pTableListInfo = tableListCreate(); if (pBlockNode->tableType == TSDB_SUPER_TABLE) { - SArray* pList = taosArrayInit(4, sizeof(STableKeyInfo)); + SArray* pList = taosArrayInit(4, sizeof(uint64_t)); int32_t code = pTaskInfo->storageAPI.metaFn.getChildTableList(pHandle->vnode, pBlockNode->uid, pList); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; + taosArrayDestroy(pList); return NULL; } size_t num = taosArrayGetSize(pList); for (int32_t i = 0; i < num; ++i) { - STableKeyInfo* p = taosArrayGet(pList, i); - tableListAddTableInfo(pTableListInfo, p->uid, 0); + uint64_t* id = taosArrayGet(pList, i); + tableListAddTableInfo(pTableListInfo, *id, 0); } taosArrayDestroy(pList); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8f11cd769a..3984e9a792 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2185,7 +2185,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { return pBlock; } - SMetaTableInfo mtInfo = pAPI->snapshotFn.getTableInfoFromSnapshot(pInfo->sContext); + SMetaTableInfo mtInfo = pAPI->snapshotFn.getMetaTableInfoFromSnapshot(pInfo->sContext); STqOffsetVal offset = {0}; if (mtInfo.uid == 0) { // read snapshot done, change to get data from wal qDebug("tmqsnap read snapshot done, change to get data from wal"); @@ -2203,8 +2203,8 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { int32_t dataLen = 0; int16_t type = 0; int64_t uid = 0; - if (pAPI->snapshotFn.getMetaInfoFromSnapshot(sContext, &data, &dataLen, &type, &uid) < 0) { - qError("tmqsnap getMetafromSnapShot error"); + if (pAPI->snapshotFn.getTableInfoFromSnapshot(sContext, &data, &dataLen, &type, &uid) < 0) { + qError("tmqsnap getTableInfoFromSnapshot error"); taosMemoryFreeClear(data); return NULL; } @@ -2253,6 +2253,7 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT pInfo->pTableListInfo = tableListCreate(); pInfo->vnode = pHandle->vnode; + pInfo->pAPI = &pTaskInfo->storageAPI; pInfo->sContext = pHandle->sContext; setOperatorInfo(pOperator, "RawScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo, @@ -3470,7 +3471,7 @@ static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, S int64_t ctbNum = 0; int32_t code = pAPI->metaFn.getNumOfChildTables(pInfo->readHandle.vnode, stbUid, &ctbNum); - fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes); + fillTableCountScanDataBlock(pSupp, dbName, varDataVal(stbName), ctbNum, pRes); } static void destoryTableCountScanOperator(void* param) { From ab4111b25e78f1697233fb6072ecacf2cccb6fe5 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 26 May 2023 08:24:27 +0800 Subject: [PATCH 67/97] enhance: add python udf sample that uses numpy --- tests/script/tsim/query/udfpy.sim | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/script/tsim/query/udfpy.sim b/tests/script/tsim/query/udfpy.sim index bf89ff96a7..394833b4bd 100644 --- a/tests/script/tsim/query/udfpy.sim +++ b/tests/script/tsim/query/udfpy.sim @@ -38,7 +38,6 @@ else endi sql create function pybitand as '/tmp/pyudf/pybitand.py' outputtype int language 'python'; sql create aggregate function pyl2norm as '/tmp/pyudf/pyl2norm.py' outputtype double bufSize 128 language 'python'; -sql create aggregate function pycumsum as '/tmp/pyudf/pycumsum.py' outputtype double bufSize 128 language 'python'; sql show functions; if $rows != 4 then @@ -282,9 +281,17 @@ if $data20 != 8.000000000 then return -1 endi +sql create aggregate function pycumsum as '/tmp/pyudf/pycumsum.py' outputtype double bufSize 128 language 'python'; sql select pycumsum(f2) from udf.t2 print ======= pycumsum print $rows $data00 +if $rows != 1 then + return -1 +endi +if $data00 != 20.000000000 then + return -1 +endi +sql drop function pycumsum sql create or replace function bit_and as '/tmp/udf/libbitand.so' outputtype int sql select func_version from information_schema.ins_functions where name='bit_and' From 75eb724c37297c6010b013914b09ea0e5c31065c Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Thu, 25 May 2023 17:28:50 +0800 Subject: [PATCH 68/97] fix tsdb read error version --- source/dnode/vnode/src/tq/tq.c | 1 + source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 0a3173b3cb..25df80479d 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1083,6 +1083,7 @@ int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t sversion, char* msg, int32_t qDebug("s-task:%s set the start wal offset to be:%"PRId64, pTask->id.idStr, sversion); walReaderSeekVer(pTask->exec.pWalReader, sversion); + pTask->chkInfo.currentVer = sversion; if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { streamMetaReleaseTask(pTq->pStreamMeta, pTask); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b4b090249e..2fb86db8b8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1848,7 +1848,7 @@ static bool isCleanFileDataBlock(STsdbReader* pReader, SFileDataBlockInfo* pBloc SDataBlockToLoadInfo info = {0}; getBlockToLoadInfo(&info, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader, pReader); bool isCleanFileBlock = !(info.overlapWithNeighborBlock || info.hasDupTs || info.overlapWithKeyInBuf || - info.overlapWithDelInfo || info.overlapWithLastBlock); + info.overlapWithDelInfo || info.overlapWithLastBlock || info.partiallyRequired); return isCleanFileBlock; } From 974b59c238173ad468872626fb167235ba82d3a3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 26 May 2023 10:10:10 +0800 Subject: [PATCH 69/97] fix(query): fix bug in iterating the table list. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 62b121e4d7..ea84910d17 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3433,6 +3433,7 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { if (!hasNexTable) { return TSDB_CODE_SUCCESS; } + pBlockScanInfo = pStatus->pTableIter; } initMemDataIterator(*pBlockScanInfo, pReader); From 208ba2ef4d507aec918d7001b399b9b68f5255ff Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 26 May 2023 10:45:37 +0800 Subject: [PATCH 70/97] fix:memory leak --- source/client/src/clientImpl.c | 1 + source/libs/parser/src/parTranslater.c | 11 ++++------- source/libs/parser/src/parUtil.c | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 82177302c7..5963e419e1 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1757,6 +1757,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int return TSDB_CODE_TSC_INTERNAL_ERROR; } + taosMemoryFreeClear(pResultInfo->convertJson); pResultInfo->convertJson = taosMemoryCalloc(1, dataLen); if (pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; char* p1 = pResultInfo->convertJson; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 864513f15d..39dab556f3 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -8008,6 +8008,9 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { isJson = true; code = buildJsonTagVal(pCxt, pTagSchema, pVal, pTagArray, ppTag); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode((SNode*)pVal); + } taosArrayPush(tagName, pTagSchema->name); } else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL && !pVal->isNull) { char* tmpVal = nodesGetValueFromNode(pVal); @@ -8328,13 +8331,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS SArray* pTagVals = taosArrayInit(1, sizeof(STagVal)); int32_t code = TSDB_CODE_SUCCESS; STag* pTag = NULL; - do { - code = parseJsontoTagData(pStmt->pVal->literal, pTagVals, &pTag, &pCxt->msgBuf); - if (TSDB_CODE_SUCCESS != code) { - break; - } - } while (0); - + code = parseJsontoTagData(pStmt->pVal->literal, pTagVals, &pTag, &pCxt->msgBuf); taosArrayDestroy(pTagVals); if (code != TSDB_CODE_SUCCESS) { return code; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 5597bd3df8..8fa92a9d2e 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -416,7 +416,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi end: taosHashCleanup(keyHash); if (retCode == TSDB_CODE_SUCCESS) { - tTagNew(pTagVals, 1, true, ppTag); + retCode = tTagNew(pTagVals, 1, true, ppTag); } for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { STagVal* p = (STagVal*)taosArrayGet(pTagVals, i); From 81c796953e5233d1a2603eab13b71e7f25d12a0b Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 26 May 2023 10:55:52 +0800 Subject: [PATCH 71/97] fix: address sanitizer error --- source/libs/executor/src/executorInt.c | 7 ++++++- source/libs/scalar/src/sclvector.c | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index d229f7d0ee..66ad3ca892 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -573,8 +573,13 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD if (colDataIsNull_var(pDst, j)) { colDataSetNull_var(pDst, numOfRows); } else { + // fix address sanitizer error. p1 may point to memory that will change during realloc of colDataSetVal, first copy it to p2 char* p1 = colDataGetVarData(pDst, j); - colDataSetVal(pDst, numOfRows, p1, false); + int32_t len = varDataTLen(p1); + char* p2 = taosMemoryMalloc(len); + varDataCopy(p2, p1); + colDataSetVal(pDst, numOfRows, p2, false); + taosMemoryFree(p2); } numOfRows += 1; j += 1; diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 6983f6313b..e2496e15c3 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1788,6 +1788,7 @@ void vectorIsNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, ++pOut->numOfQualified; } colDataSetInt8(pOut->columnData, i, &v); + colDataClearNull_f(pOut->columnData->nullbitmap, i); } pOut->numOfRows = pLeft->numOfRows; } @@ -1799,6 +1800,7 @@ void vectorNotNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut ++pOut->numOfQualified; } colDataSetInt8(pOut->columnData, i, &v); + colDataClearNull_f(pOut->columnData->nullbitmap, i); } pOut->numOfRows = pLeft->numOfRows; } From bb481c52b6c03c8b0c8f27877a7b93401e5c2eb3 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 26 May 2023 11:26:55 +0800 Subject: [PATCH 72/97] cache/skyline: fix skyline false condition --- source/dnode/vnode/src/tsdb/tsdbCache.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 830f44b411..8757661216 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -825,7 +825,7 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pCache, h); SLastCol lastCol = *pLastCol; - // reallocVarData(&lastCol.colVal); + reallocVarData(&lastCol.colVal); taosArrayPush(pLastArray, &lastCol); if (h) { @@ -2151,11 +2151,15 @@ static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) { if (key->ts > pItemBack->ts) { return false; } else if (key->ts >= pItemFront->ts && key->ts <= pItemBack->ts) { - // if (key->version <= pItemFront->version || (key->ts == pItemBack->ts && key->version <= pItemBack->version)) { - if (key->version <= pItemFront->version || key->version <= pItemBack->version) { + if (key->version <= pItemFront->version || (key->ts == pItemBack->ts && key->version <= pItemBack->version)) { + // if (key->version <= pItemFront->version || key->version <= pItemBack->version) { return true; } else { - return false; + if (*iSkyline > 1) { + --*iSkyline; + } else { + return false; + } } } else { if (*iSkyline > 1) { From 1e845acabe2e3f554646ab1b793931bae26ea362 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 26 May 2023 11:34:12 +0800 Subject: [PATCH 73/97] fix(tmq): allow seek before get assignment --- source/client/src/clientTmq.c | 6 ++++-- source/dnode/vnode/src/tq/tq.c | 2 ++ source/libs/stream/src/streamMeta.c | 6 +++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index aff5846092..59d411cde8 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -27,6 +27,8 @@ #define EMPTY_BLOCK_POLL_IDLE_DURATION 10 #define DEFAULT_AUTO_COMMIT_INTERVAL 5000 +#define OFFSET_IS_RESET_OFFSET(_of) ((_of) < 0) + typedef void (*__tmq_askep_fn_t)(tmq_t* pTmq, int32_t code, SDataBuf* pBuf, void* pParam); struct SMqMgmt { @@ -2626,12 +2628,12 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; int32_t type = pOffsetInfo->currentOffset.type; - if (type != TMQ_OFFSET__LOG) { + if (type != TMQ_OFFSET__LOG && !OFFSET_IS_RESET_OFFSET(type)) { tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, seek not allowed", tmq->consumerId, type); return TSDB_CODE_INVALID_PARA; } - if (offset < pOffsetInfo->walVerBegin || offset > pOffsetInfo->walVerEnd) { + if (type == TMQ_OFFSET__LOG && (offset < pOffsetInfo->walVerBegin || offset > pOffsetInfo->walVerEnd)) { tscError("consumer:0x%" PRIx64 " invalid seek params, offset:%" PRId64 ", valid range:[%" PRId64 ", %" PRId64 "]", tmq->consumerId, offset, pOffsetInfo->walVerBegin, pOffsetInfo->walVerEnd); return TSDB_CODE_INVALID_PARA; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 9025c16612..14deff7502 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1280,6 +1280,8 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessTaskDropReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; + tqDebug("vgId:%d receive msg to drop stream task:0x%x", TD_VID(pTq->pVnode), pReq->taskId); + streamMetaRemoveTask(pTq->pStreamMeta, pReq->taskId); return 0; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 33375fe921..98e63f7f51 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -268,12 +268,14 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); if (ppTask) { SStreamTask* pTask = *ppTask; + taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t)); tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), pMeta->txn); atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING); - int32_t num = taosArrayGetSize(pMeta->pTaskList); + + qDebug("s-task:%s set the drop task flag, remain running s-task:%d", pTask->id.idStr, num - 1); for (int32_t i = 0; i < num; ++i) { int32_t* pTaskId = taosArrayGet(pMeta->pTaskList, i); if (*pTaskId == taskId) { @@ -283,6 +285,8 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { } streamMetaReleaseTask(pMeta, pTask); + } else { + qDebug("vgId:%d failed to find the task:0x%x, it may be dropped already", pMeta->vgId, taskId); } taosWUnLockLatch(&pMeta->lock); From fcf2b58aad3010b10dbb9f24465ce7d6faf61eff Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 26 May 2023 12:48:37 +0800 Subject: [PATCH 74/97] check offline when force --- include/util/taoserror.h | 1 + source/dnode/mnode/impl/src/mndDnode.c | 11 +++++++++-- source/util/src/terror.c | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 636a5e63a7..9e5229870e 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -409,6 +409,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MNODE_ALREADY_IS_VOTER TAOS_DEF_ERROR_CODE(0, 0x0413) // internal #define TSDB_CODE_MNODE_ONLY_TWO_MNODE TAOS_DEF_ERROR_CODE(0, 0x0414) // internal #define TSDB_CODE_MNODE_NO_NEED_RESTORE TAOS_DEF_ERROR_CODE(0, 0x0415) // internal +#define TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0416) // vnode // #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index cee3b3c61d..99694ae4b8 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -986,8 +986,15 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { } int32_t numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id); - if ((numOfVnodes > 0 || pMObj != NULL || pSObj != NULL || pQObj != NULL) && !force) { - if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) { + if (numOfVnodes > 0 || pMObj != NULL || pSObj != NULL || pQObj != NULL) { + bool isonline = mndIsDnodeOnline(pDnode, taosGetTimestampMs()); + if (isonline && force) { + terrno = TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE; + mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), + numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); + goto _OVER; + } + if (!isonline && !force) { terrno = TSDB_CODE_DNODE_OFFLINE; mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 31727f7535..e28e67f83a 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -325,6 +325,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_NOT_CATCH_UP, "Mnode didn't catch th TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_ALREADY_IS_VOTER, "Mnode already is a leader") TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_ONLY_TWO_MNODE, "Only two mnodes exist") TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_NO_NEED_RESTORE, "No need to restore on this dnode") +TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE, "Please use this command when the dnode is offline") // vnode TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode is closed or removed") From 00810d1ffdb4a95bca7ab863532bdee7c7cb98e2 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 26 May 2023 13:10:49 +0800 Subject: [PATCH 75/97] fix: remove udfpy.sim pycumsum part --- tests/script/tsim/query/udfpy.sim | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/script/tsim/query/udfpy.sim b/tests/script/tsim/query/udfpy.sim index 394833b4bd..2a4daedd5d 100644 --- a/tests/script/tsim/query/udfpy.sim +++ b/tests/script/tsim/query/udfpy.sim @@ -281,17 +281,17 @@ if $data20 != 8.000000000 then return -1 endi -sql create aggregate function pycumsum as '/tmp/pyudf/pycumsum.py' outputtype double bufSize 128 language 'python'; -sql select pycumsum(f2) from udf.t2 -print ======= pycumsum -print $rows $data00 -if $rows != 1 then - return -1 -endi -if $data00 != 20.000000000 then - return -1 -endi -sql drop function pycumsum +#sql create aggregate function pycumsum as '/tmp/pyudf/pycumsum.py' outputtype double bufSize 128 language 'python'; +#sql select pycumsum(f2) from udf.t2 +#print ======= pycumsum +#print $rows $data00 +#if $rows != 1 then +# return -1 +#endi +#if $data00 != 20.000000000 then +# return -1 +#endi +#sql drop function pycumsum sql create or replace function bit_and as '/tmp/udf/libbitand.so' outputtype int sql select func_version from information_schema.ins_functions where name='bit_and' From 70bcbafbb8b3ee3adde1bd1813857fec9c9732da Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 26 May 2023 13:36:56 +0800 Subject: [PATCH 76/97] fix: json value length is not same as binary/nchar length --- source/libs/executor/src/executorInt.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 66ad3ca892..ecd1874fa5 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -575,9 +575,14 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD } else { // fix address sanitizer error. p1 may point to memory that will change during realloc of colDataSetVal, first copy it to p2 char* p1 = colDataGetVarData(pDst, j); - int32_t len = varDataTLen(p1); + int32_t len = 0; + if (pDst->info.type == TSDB_DATA_TYPE_JSON) { + len = getJsonValueLen(p1); + } else { + len = varDataTLen(p1); + } char* p2 = taosMemoryMalloc(len); - varDataCopy(p2, p1); + memcpy(p2, p1, len); colDataSetVal(pDst, numOfRows, p2, false); taosMemoryFree(p2); } From a8599d78d1e0d0d5305b90375df15a5aef5da6bc Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 26 May 2023 14:02:46 +0800 Subject: [PATCH 77/97] docs: fix timeline function super table explaination --- docs/en/12-taos-sql/10-function.md | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index a204415d65..b469da8f55 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -1008,8 +1008,7 @@ SAMPLE(expr, k) **More explanations**: -This function cannot be used in expression calculation. -- Must be used with `PARTITION BY tbname` when it's used on a STable to force the result on each single timeline +- This function cannot be used in expression calculation. ### TAIL @@ -1088,7 +1087,6 @@ CSUM(expr) - Arithmetic operation can't be performed on the result of `csum` function - Can only be used with aggregate functions This function can be used with supertables and standard tables. -- Must be used with `PARTITION BY tbname` when it's used on a STable to force the result on each single timeline ### DERIVATIVE @@ -1112,7 +1110,6 @@ ignore_negative: { **More explanation**: -- It can be used together with `PARTITION BY tbname` against a STable. - It can be used together with a selected column. For example: select \_rowts, DERIVATIVE() from. ### DIFF @@ -1175,7 +1172,6 @@ MAVG(expr, k) - Arithmetic operation can't be performed on the result of `MAVG`. - Can only be used with data columns, can't be used with tags. - Can't be used with aggregate functions. -- Must be used with `PARTITION BY tbname` when it's used on a STable to force the result on each single timeline ### STATECOUNT @@ -1201,7 +1197,6 @@ STATECOUNT(expr, oper, val) **More explanations**: -- Must be used together with `PARTITION BY tbname` when it's used on a STable to force the result into each single timeline] - Can't be used with window operation, like interval/state_window/session_window @@ -1229,7 +1224,6 @@ STATEDURATION(expr, oper, val, unit) **More explanations**: -- Must be used together with `PARTITION BY tbname` when it's used on a STable to force the result into each single timeline] - Can't be used with window operation, like interval/state_window/session_window @@ -1247,7 +1241,6 @@ TWA(expr) **Applicable table types**: standard tables and supertables -- Must be used together with `PARTITION BY tbname` to force the result into each single timeline. ## System Information Functions From 217efa830e8eb74e092b0ddd8c6bcbf80287497a Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 26 May 2023 14:02:46 +0800 Subject: [PATCH 78/97] docs: fix timeline function super table explaination --- docs/zh/12-taos-sql/10-function.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 248554f931..b4785dc5e6 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -1001,7 +1001,6 @@ SAMPLE(expr, k) **使用说明**： - 不能参与表达式计算；该函数可以应用在普通表和超级表上； -- 使用在超级表上的时候，需要搭配 PARTITION by tbname 使用，将结果强制规约到单个时间线。 ### TAIL @@ -1080,7 +1079,6 @@ CSUM(expr) - 不支持 +、-、*、/ 运算，如 csum(col1) + csum(col2)。 - 只能与聚合（Aggregation）函数一起使用。 该函数可以应用在普通表和超级表上。 -- 使用在超级表上的时候，需要搭配 PARTITION BY tbname使用，将结果强制规约到单个时间线。 ### DERIVATIVE @@ -1104,7 +1102,6 @@ ignore_negative: { **使用说明**: -- DERIVATIVE 函数可以在由 PARTITION BY 划分出单独时间线的情况下用于超级表（也即 PARTITION BY tbname）。 - 可以与选择相关联的列一起使用。 例如: select \_rowts, DERIVATIVE() from。 ### DIFF @@ -1167,7 +1164,6 @@ MAVG(expr, k) - 不支持 +、-、*、/ 运算，如 mavg(col1, k1) + mavg(col2, k1); - 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用； -- 使用在超级表上的时候，需要搭配 PARTITION BY tbname使用，将结果强制规约到单个时间线。 ### STATECOUNT @@ -1193,7 +1189,6 @@ STATECOUNT(expr, oper, val) **使用说明**： -- 该函数可以应用在普通表上，在由 PARTITION BY 划分出单独时间线的情况下用于超级表（也即 PARTITION BY tbname） - 不能和窗口操作一起使用，例如 interval/state_window/session_window。 @@ -1221,7 +1216,6 @@ STATEDURATION(expr, oper, val, unit) **使用说明**： -- 该函数可以应用在普通表上，在由 PARTITION BY 划分出单独时间线的情况下用于超级表（也即 PARTITION BY tbname） - 不能和窗口操作一起使用，例如 interval/state_window/session_window。 @@ -1239,8 +1233,6 @@ TWA(expr) **适用于**：表和超级表。 -**使用说明**： TWA 函数可以在由 PARTITION BY 划分出单独时间线的情况下用于超级表（也即 PARTITION BY tbname）。 - ## 系统信息函数 From 6e29d7879f9b9dec98f9d7faef4ac78aa0fafdeb Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 26 May 2023 14:13:01 +0800 Subject: [PATCH 79/97] cache/var-data: realloc var data --- source/dnode/vnode/src/tsdb/tsdbCache.c | 3 ++- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 24 ++++++++++----------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 8757661216..aea7c45bd6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -703,6 +703,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr *pTmpLastCol = *pLastCol; pLastCol = pTmpLastCol; + reallocVarData(&pLastCol->colVal); size_t charge = sizeof(*pLastCol); if (IS_VAR_DATA_TYPE(pLastCol->colVal.type)) { charge += pLastCol->colVal.value.nData; @@ -853,8 +854,8 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache SLastCol lastCol = *pLastCol; reallocVarData(&lastCol.colVal); - taosArraySet(pLastArray, idxKey->idx, &lastCol); + if (h) { taosLRUCacheRelease(pCache, h, false); } diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 0e0b7a2ffa..5df1ea0672 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -315,14 +315,14 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 tsdbCacheGetBatch(pr->pTsdb, pKeyInfo->uid, pRow, pr, ltype); // tsdbCacheGet(pr->pTsdb, pKeyInfo->uid, pRow, pr, ltype); if (TARRAY_SIZE(pRow) <= 0) { - // taosArrayClearEx(pRow, freeItem); - taosArrayClear(pRow); + taosArrayClearEx(pRow, freeItem); + // taosArrayClear(pRow); continue; } SLastCol* pColVal = taosArrayGet(pRow, 0); if (COL_VAL_IS_NONE(&pColVal->colVal)) { - // taosArrayClearEx(pRow, freeItem); - taosArrayClear(pRow); + taosArrayClearEx(pRow, freeItem); + // taosArrayClear(pRow); continue; } @@ -381,8 +381,8 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } } - // taosArrayClearEx(pRow, freeItem); - taosArrayClear(pRow); + taosArrayClearEx(pRow, freeItem); + // taosArrayClear(pRow); } if (hasRes) { @@ -394,20 +394,20 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype); if (TARRAY_SIZE(pRow) <= 0) { - // taosArrayClearEx(pRow, freeItem); - taosArrayClear(pRow); + taosArrayClearEx(pRow, freeItem); + // taosArrayClear(pRow); continue; } SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, 0); if (COL_VAL_IS_NONE(&pColVal->colVal)) { - // taosArrayClearEx(pRow, freeItem); - taosArrayClear(pRow); + taosArrayClearEx(pRow, freeItem); + // taosArrayClear(pRow); continue; } saveOneRow(pRow, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr); - // taosArrayClearEx(pRow, freeItem); - taosArrayClear(pRow); + taosArrayClearEx(pRow, freeItem); + // taosArrayClear(pRow); taosArrayPush(pTableUidList, &uid); From f61f37c936b53ea89d5a6ce3c1229fda8f3465ed Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 26 May 2023 14:20:01 +0800 Subject: [PATCH 80/97] fix(tmq): set the correct offset rsp when no poll occuring. --- source/client/test/clientTests.cpp | 15 ++++++++++++--- source/dnode/vnode/src/tq/tq.c | 9 ++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index b04727bfc0..ccc17289b0 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -1100,7 +1100,7 @@ TEST(clientCase, sub_tb_test) { // 创建订阅 topics 列表 tmq_list_t* topicList = tmq_list_new(); - tmq_list_append(topicList, "topic_t1"); + tmq_list_append(topicList, "t1"); // 启动订阅 tmq_subscribe(tmq, topicList); @@ -1118,7 +1118,7 @@ TEST(clientCase, sub_tb_test) { tmq_topic_assignment* pAssign = NULL; int32_t numOfAssign = 0; - int32_t code = tmq_get_topic_assignment(tmq, "topic_t1", &pAssign, &numOfAssign); + int32_t code = tmq_get_topic_assignment(tmq, "t1", &pAssign, &numOfAssign); if (code != 0) { printf("error occurs:%s\n", tmq_err2str(code)); tmq_consumer_close(tmq); @@ -1127,7 +1127,16 @@ TEST(clientCase, sub_tb_test) { return; } - tmq_offset_seek(tmq, "topic_t1", pAssign[0].vgId, 0); + tmq_offset_seek(tmq, "t1", pAssign[0].vgId, 4); + + code = tmq_get_topic_assignment(tmq, "t1", &pAssign, &numOfAssign); + if (code != 0) { + printf("error occurs:%s\n", tmq_err2str(code)); + tmq_consumer_close(tmq); + taos_close(pConn); + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); + return; + } while (1) { TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 14deff7502..c6077a3f8f 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -510,8 +510,6 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { int64_t sver = 0, ever = 0; walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); - int64_t currentVer = walReaderGetCurrentVer(pHandle->execHandle.pTqReader->pWalReader); - SMqDataRsp dataRsp = {0}; tqInitDataRsp(&dataRsp, &req); @@ -537,7 +535,12 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { dataRsp.rspOffset.type = TMQ_OFFSET__LOG; if (reqOffset.type == TMQ_OFFSET__LOG) { - dataRsp.rspOffset.version = currentVer; // return current consume offset value + int64_t currentVer = walReaderGetCurrentVer(pHandle->execHandle.pTqReader->pWalReader); + if (currentVer == -1) { // not start to read data from wal yet, return req offset directly + dataRsp.rspOffset.version = reqOffset.version; + } else { + dataRsp.rspOffset.version = currentVer; // return current consume offset value + } } else if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { dataRsp.rspOffset.version = sver; // not consume yet, set the earliest position } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { From b83bf9163ec854d2fb857666fd50693df16572fa Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 26 May 2023 14:46:32 +0800 Subject: [PATCH 81/97] fix: restore scalar mode computation and add filterScalarMode config variable and add test case --- include/common/tglobal.h | 1 + source/common/src/tglobal.c | 5 + source/libs/scalar/src/filter.c | 10 +- tests/parallel_test/cases.task | 2 + tests/script/tsim/parser/condition_scl.sim | 136 +++++++++++++++++++ tests/script/tsim/query/emptyTsRange_scl.sim | 21 +++ 6 files changed, 172 insertions(+), 3 deletions(-) create mode 100644 tests/script/tsim/parser/condition_scl.sim create mode 100644 tests/script/tsim/query/emptyTsRange_scl.sim diff --git a/include/common/tglobal.h b/include/common/tglobal.h index abee37d122..157e37f080 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -180,6 +180,7 @@ extern int32_t tsRpcRetryInterval; extern bool tsDisableStream; extern int64_t tsStreamBufferSize; extern int64_t tsCheckpointInterval; +extern bool tsFilterScalarMode; // #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 9ff297c5b4..c648f8551a 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -208,6 +208,7 @@ char tsUdfdLdLibPath[512] = ""; bool tsDisableStream = false; int64_t tsStreamBufferSize = 128 * 1024 * 1024; int64_t tsCheckpointInterval = 3 * 60 * 60 * 1000; +bool tsFilterScalarMode = false; #ifndef _STORAGE int32_t taosSetTfsCfg(SConfig *pCfg) { @@ -522,6 +523,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, 0) != 0) return -1; + if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, 0) != 0) return -1; + GRANT_CFG_ADD; return 0; } @@ -898,6 +901,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsStreamBufferSize = cfgGetItem(pCfg, "streamBufferSize")->i64; tsCheckpointInterval = cfgGetItem(pCfg, "checkpointInterval")->i64; + tsFilterScalarMode = cfgGetItem(pCfg, "filterScalarMode")->bval; + GRANT_CFG_GET; return 0; } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index d45abb1b87..9a488edd67 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -14,6 +14,7 @@ */ #include #include "os.h" +#include "tglobal.h" #include "thash.h" // #include "queryLog.h" #include "filter.h" @@ -4510,7 +4511,7 @@ static int32_t fltSclCollectOperatorsFromLogicCond(SNode *pNode, SArray *sclOpLi return TSDB_CODE_SUCCESS; } SNode *pExpr = NULL; - FOREACH(pExpr, pLogicCond->pParameterList) { + FOREACH(pExpr, pLogicCond->pParameterList) { if (!fltSclIsCollectableNode(pExpr)) { return TSDB_CODE_SUCCESS; } @@ -4614,8 +4615,11 @@ int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pInfo, uint32_t options) stat.info = info; FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat)); - - info->scalarMode = true; + if (tsFilterScalarMode) { + info->scalarMode = true; + } else { + info->scalarMode = stat.scalarMode; + } fltDebug("scalar mode: %d", info->scalarMode); if (!info->scalarMode) { diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ab4663e5b3..6037a14c8a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1274,6 +1274,7 @@ ,,y,script,./test.sh -f tsim/parser/columnValue_tinyint.sim ,,y,script,./test.sh -f tsim/parser/columnValue_unsign.sim ,,y,script,./test.sh -f tsim/parser/condition.sim +,,y,script,./test.sh -f tsim/parser/condition_scl.sim ,,y,script,./test.sh -f tsim/parser/constCol.sim ,,y,script,./test.sh -f tsim/parser/create_db.sim ,,y,script,./test.sh -f tsim/parser/create_mt.sim @@ -1353,6 +1354,7 @@ ,,y,script,./test.sh -f tsim/query/event.sim ,,y,script,./test.sh -f tsim/query/forceFill.sim ,,y,script,./test.sh -f tsim/query/emptyTsRange.sim +,,y,script,./test.sh -f tsim/query/emptyTsRange_scl.sim ,,y,script,./test.sh -f tsim/query/partitionby.sim ,,y,script,./test.sh -f tsim/query/tableCount.sim ,,y,script,./test.sh -f tsim/query/tag_scan.sim diff --git a/tests/script/tsim/parser/condition_scl.sim b/tests/script/tsim/parser/condition_scl.sim new file mode 100644 index 0000000000..f377988006 --- /dev/null +++ b/tests/script/tsim/parser/condition_scl.sim @@ -0,0 +1,136 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c filterScalarMode -v 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql drop database if exists cdb +sql create database if not exists cdb +sql use cdb +sql create table stb1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(10), t3 double) +sql create table tb1 using stb1 tags(1,'1',1.0) +sql create table tb2 using stb1 tags(2,'2',2.0) +sql create table tb3 using stb1 tags(3,'3',3.0) +sql create table tb4 using stb1 tags(4,'4',4.0) +sql create table tb5 using stb1 tags(5,'5',5.0) +sql create table tb6 using stb1 tags(6,'6',6.0) + +sql insert into tb1 values ('2021-05-05 18:19:00',1,1.0,1,1,1,1.0,true ,'1','1') +sql insert into tb1 values ('2021-05-05 18:19:01',2,2.0,2,2,2,2.0,true ,'2','2') +sql insert into tb1 values ('2021-05-05 18:19:02',3,3.0,3,3,3,3.0,false,'3','3') +sql insert into tb1 values ('2021-05-05 18:19:03',4,4.0,4,4,4,4.0,false,'4','4') +sql insert into tb1 values ('2021-05-05 18:19:04',11,11.0,11,11,11,11.0,true ,'11','11') +sql insert into tb1 values ('2021-05-05 18:19:05',12,12.0,12,12,12,12.0,true ,'12','12') +sql insert into tb1 values ('2021-05-05 18:19:06',13,13.0,13,13,13,13.0,false,'13','13') +sql insert into tb1 values ('2021-05-05 18:19:07',14,14.0,14,14,14,14.0,false,'14','14') +sql insert into tb2 values ('2021-05-05 18:19:08',21,21.0,21,21,21,21.0,true ,'21','21') +sql insert into tb2 values ('2021-05-05 18:19:09',22,22.0,22,22,22,22.0,true ,'22','22') +sql insert into tb2 values ('2021-05-05 18:19:10',23,23.0,23,23,23,23.0,false,'23','23') +sql insert into tb2 values ('2021-05-05 18:19:11',24,24.0,24,24,24,24.0,false,'24','24') +sql insert into tb3 values ('2021-05-05 18:19:12',31,31.0,31,31,31,31.0,true ,'31','31') +sql insert into tb3 values ('2021-05-05 18:19:13',32,32.0,32,32,32,32.0,true ,'32','32') +sql insert into tb3 values ('2021-05-05 18:19:14',33,33.0,33,33,33,33.0,false,'33','33') +sql insert into tb3 values ('2021-05-05 18:19:15',34,34.0,34,34,34,34.0,false,'34','34') +sql insert into tb4 values ('2021-05-05 18:19:16',41,41.0,41,41,41,41.0,true ,'41','41') +sql insert into tb4 values ('2021-05-05 18:19:17',42,42.0,42,42,42,42.0,true ,'42','42') +sql insert into tb4 values ('2021-05-05 18:19:18',43,43.0,43,43,43,43.0,false,'43','43') +sql insert into tb4 values ('2021-05-05 18:19:19',44,44.0,44,44,44,44.0,false,'44','44') +sql insert into tb5 values ('2021-05-05 18:19:20',51,51.0,51,51,51,51.0,true ,'51','51') +sql insert into tb5 values ('2021-05-05 18:19:21',52,52.0,52,52,52,52.0,true ,'52','52') +sql insert into tb5 values ('2021-05-05 18:19:22',53,53.0,53,53,53,53.0,false,'53','53') +sql insert into tb5 values ('2021-05-05 18:19:23',54,54.0,54,54,54,54.0,false,'54','54') +sql insert into tb6 values ('2021-05-05 18:19:24',61,61.0,61,61,61,61.0,true ,'61','61') +sql insert into tb6 values ('2021-05-05 18:19:25',62,62.0,62,62,62,62.0,true ,'62','62') +sql insert into tb6 values ('2021-05-05 18:19:26',63,63.0,63,63,63,63.0,false,'63','63') +sql insert into tb6 values ('2021-05-05 18:19:27',64,64.0,64,64,64,64.0,false,'64','64') +sql insert into tb6 values ('2021-05-05 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) + +sql create table stb2 (ts timestamp, u1 int unsigned, u2 bigint unsigned, u3 smallint unsigned, u4 tinyint unsigned, ts2 timestamp) TAGS(t1 int unsigned, t2 bigint unsigned, t3 timestamp, t4 int) +sql create table tb2_1 using stb2 tags(1,1,'2021-05-05 18:38:38',1) +sql create table tb2_2 using stb2 tags(2,2,'2021-05-05 18:58:58',2) + +sql insert into tb2_1 values ('2021-05-05 18:19:00',1,2,3,4,'2021-05-05 18:28:01') +sql insert into tb2_1 values ('2021-05-05 18:19:01',5,6,7,8,'2021-05-05 18:28:02') +sql insert into tb2_1 values ('2021-05-05 18:19:02',2,2,3,4,'2021-05-05 18:28:03') +sql insert into tb2_1 values ('2021-05-05 18:19:03',5,6,7,8,'2021-05-05 18:28:04') +sql insert into tb2_1 values ('2021-05-05 18:19:04',3,2,3,4,'2021-05-05 18:28:05') +sql insert into tb2_1 values ('2021-05-05 18:19:05',5,6,7,8,'2021-05-05 18:28:06') +sql insert into tb2_1 values ('2021-05-05 18:19:06',4,2,3,4,'2021-05-05 18:28:07') +sql insert into tb2_1 values ('2021-05-05 18:19:07',5,6,7,8,'2021-05-05 18:28:08') +sql insert into tb2_1 values ('2021-05-05 18:19:08',5,2,3,4,'2021-05-05 18:28:09') +sql insert into tb2_1 values ('2021-05-05 18:19:09',5,6,7,8,'2021-05-05 18:28:10') +sql insert into tb2_1 values ('2021-05-05 18:19:10',6,2,3,4,'2021-05-05 18:28:11') +sql insert into tb2_2 values ('2021-05-05 18:19:11',5,6,7,8,'2021-05-05 18:28:12') +sql insert into tb2_2 values ('2021-05-05 18:19:12',7,2,3,4,'2021-05-05 18:28:13') +sql insert into tb2_2 values ('2021-05-05 18:19:13',5,6,7,8,'2021-05-05 18:28:14') +sql insert into tb2_2 values ('2021-05-05 18:19:14',8,2,3,4,'2021-05-05 18:28:15') +sql insert into tb2_2 values ('2021-05-05 18:19:15',5,6,7,8,'2021-05-05 18:28:16') + +sql create table stb3 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(10), t3 double) +sql create table tb3_1 using stb3 tags(1,'1',1.0) +sql create table tb3_2 using stb3 tags(2,'2',2.0) + +sql insert into tb3_1 values ('2021-01-05 18:19:00',1,1.0,1,1,1,1.0,true ,'1','1') +sql insert into tb3_1 values ('2021-02-05 18:19:01',2,2.0,2,2,2,2.0,true ,'2','2') +sql insert into tb3_1 values ('2021-03-05 18:19:02',3,3.0,3,3,3,3.0,false,'3','3') +sql insert into tb3_1 values ('2021-04-05 18:19:03',4,4.0,4,4,4,4.0,false,'4','4') +sql insert into tb3_1 values ('2021-05-05 18:19:28',5,NULL,5,NULL,5,NULL,true,NULL,'5') +sql insert into tb3_1 values ('2021-06-05 18:19:28',NULL,6.0,NULL,6,NULL,6.0,NULL,'6',NULL) +sql insert into tb3_1 values ('2021-07-05 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) +sql insert into tb3_2 values ('2021-01-06 18:19:00',11,11.0,11,11,11,11.0,true ,'11','11') +sql insert into tb3_2 values ('2021-02-06 18:19:01',12,12.0,12,12,12,12.0,true ,'12','12') +sql insert into tb3_2 values ('2021-03-06 18:19:02',13,13.0,13,13,13,13.0,false,'13','13') +sql insert into tb3_2 values ('2021-04-06 18:19:03',14,14.0,14,14,14,14.0,false,'14','14') +sql insert into tb3_2 values ('2021-05-06 18:19:28',15,NULL,15,NULL,15,NULL,true,NULL,'15') +sql insert into tb3_2 values ('2021-06-06 18:19:28',NULL,16.0,NULL,16,NULL,16.0,NULL,'16',NULL) +sql insert into tb3_2 values ('2021-07-06 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) + +sql create table stb4 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9),c10 binary(16300)) TAGS(t1 int, t2 binary(10), t3 double) +sql create table tb4_0 using stb4 tags(0,'0',0.0) +sql create table tb4_1 using stb4 tags(1,'1',1.0) +sql create table tb4_2 using stb4 tags(2,'2',2.0) +sql create table tb4_3 using stb4 tags(3,'3',3.0) +sql create table tb4_4 using stb4 tags(4,'4',4.0) + +$i = 0 +$ts0 = 1625850000000 +$blockNum = 5 +$delta = 0 +$tbname0 = tb4_ +$a = 0 +$b = 200 +$c = 400 +while $i < $blockNum + $x = 0 + $rowNum = 1200 + while $x < $rowNum + $ts = $ts0 + $x + $a = $a + 1 + $b = $b + 1 + $c = $c + 1 + $d = $x / 10 + $tin = $rowNum + $binary = 'binary . $c + $binary = $binary . ' + $nchar = 'nchar . $c + $nchar = $nchar . ' + $tbname = 'tb4_ . $i + $tbname = $tbname . ' + sql insert into $tbname values ( $ts , $a , $b , $c , $d , $d , $c , true, $binary , $nchar , $binary ) + $x = $x + 1 + endw + + $i = $i + 1 + $ts0 = $ts0 + 259200000 +endw + +run tsim/parser/condition_query.sim + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sql connect + +run tsim/parser/condition_query.sim + diff --git a/tests/script/tsim/query/emptyTsRange_scl.sim b/tests/script/tsim/query/emptyTsRange_scl.sim new file mode 100644 index 0000000000..43734b047d --- /dev/null +++ b/tests/script/tsim/query/emptyTsRange_scl.sim @@ -0,0 +1,21 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c filterScalarMode -v 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql drop database if exists db1; +sql create database if not exists db1; +sql use db1; +sql create stable sta (ts timestamp, f1 double, f2 binary(200)) tags(t1 int); +sql create table tba1 using sta tags(1); +sql insert into tba1 values ('2022-04-26 15:15:01', 1.0, "a"); +sql insert into tba1 values ('2022-04-26 15:15:02', 2.0, "b"); +sql insert into tba1 values ('2022-04-26 15:15:04', 4.0, "b"); +sql insert into tba1 values ('2022-04-26 15:15:05', 5.0, "b"); +sql select last_row(*) from sta where ts >= 1678901803783 and ts <= 1678901803783 and _c0 <= 1678901803782 interval(10d,8d) fill(linear) order by _wstart desc; +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From b89f1652fcc16487de8e4fc5194c08316bc376dd Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 26 May 2023 14:52:14 +0800 Subject: [PATCH 82/97] fix: remove tbl count scan modification from the PR --- source/libs/planner/src/planOptimizer.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 4a803065ed..72e5081ab9 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2632,13 +2632,11 @@ static bool tbCntScanOptIsEligibleConds(STbCntScanOptInfo* pInfo, SNode* pCondit return false; } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pConditions)) { - return tbCntScanOptIsEligibleLogicCond(pInfo, (SLogicConditionNode*)pConditions) && - LIST_LENGTH(pInfo->pAgg->pGroupKeys) == 0; + return tbCntScanOptIsEligibleLogicCond(pInfo, (SLogicConditionNode*)pConditions); } if (QUERY_NODE_OPERATOR == nodeType(pConditions)) { - return tbCntScanOptIsEligibleOpCond((SOperatorNode*)pConditions) && - LIST_LENGTH(pInfo->pAgg->pGroupKeys) == 0; + return tbCntScanOptIsEligibleOpCond((SOperatorNode*)pConditions); } return false; From 5ebe75599e4a7f853726acd14661599c3f56d74e Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Fri, 26 May 2023 10:09:05 +0800 Subject: [PATCH 83/97] fix pause issue --- source/libs/stream/src/streamExec.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 637d661343..95b97e080a 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -327,7 +327,11 @@ int32_t streamExecForAll(SStreamTask* pTask) { while (1) { if (streamTaskShouldPause(&pTask->status)) { - return 0; + if (batchSize > 1) { + break; + } else { + return 0; + } } SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); From 34b9b322cdcbd17dcb83a4e9a4d343257d64945b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 26 May 2023 15:01:57 +0800 Subject: [PATCH 84/97] cache/var-data: realloc for cache reading --- source/dnode/vnode/src/tsdb/tsdbCache.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index aea7c45bd6..63629f8623 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -790,7 +790,9 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA code = -1; } - taosArraySet(pLastArray, idxKey->idx, pLastCol); + SLastCol lastCol = *pLastCol; + reallocVarData(&lastCol.colVal); + taosArraySet(pLastArray, idxKey->idx, &lastCol); taosArrayRemove(remainCols, j); taosMemoryFree(values_list[i]); From 436717008663a19655d67ddee463b13890cfb621 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Fri, 26 May 2023 19:16:23 +0800 Subject: [PATCH 85/97] doc: refine PDF user guide --- docs/en/07-develop/09-udf.md | 564 ++++++++++++++++++++++++++++++++--- docs/zh/07-develop/09-udf.md | 563 +++++++++++++++++++++++++++++++--- 2 files changed, 1046 insertions(+), 81 deletions(-) diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md index b0124b9c3f..92bd4efd25 100644 --- a/docs/en/07-develop/09-udf.md +++ b/docs/en/07-develop/09-udf.md @@ -270,29 +270,92 @@ select max_vol(vol1,vol2,vol3,deviceid) from battery; ## Implement a UDF in Python +### Prepare Environment + +1. Prepare Python Environment + +Please follow standard procedure of python environment preparation. + +2. Install Python package `taospyudf` + +```shell +pip3 install taospyudf +``` + +During this process, some C++ code needs to be compiled. So it's required to have `cmake` and `gcc` on your system. The compiled `libtaospyudf.so` will be automatically copied to `/usr/local/lib` path. If you are not root user, please use `sudo`. After installation is done, please check using the command below. + +```shell +root@slave11 ~/udf $ ls -l /usr/local/lib/libtaos* +-rw-r--r-- 1 root root 671344 May 24 22:54 /usr/local/lib/libtaospyudf.so +``` + +Then execute the command below. + +```shell +ldconfig +``` + +3. If you want to utilize some 3rd party python packages using `PYTHONPATH`, please set configuration parameter `UdfdLdLibPath` to the value of `PYTHONPATH` before starting `taosd`. + +4. Launch `taosd` service + +Please refer to [Get Started](../get-started) + +### Interface definition + +#### Introduction to Interface + Implement the specified interface functions when implementing a UDF in Python. - implement `process` function for the scalar UDF. - implement `start`, `reduce`, `finish` for the aggregate UDF. - implement `init` for initialization and `destroy` for termination. -### Implement a Scalar UDF in Python +#### Scalar UDF Interface The implementation of a scalar UDF is described as follows: +```Python +def process(input: datablock) -> tuple[output_type]: +``` + +Description: this function prcesses datablock, which is the input; you can use datablock.data(row, col) to access the python object at location(row,col); the output is a tuple object consisted of objects of type outputtype + +### Aggregate UDF Interface + +The implementation of an aggregate function is described as follows: + +```Python +def start() -> bytes: +def reduce(inputs: datablock, buf: bytes) -> bytes +def finish(buf: bytes) -> output_type: +``` + +Description: first the start() is invoked to generate the initial result `buffer`; then the input data is divided into multiple row blocks, and reduce() is invoked for each block `inputs` and current intermediate result `buf`; finally finish() is invoked to generate the final result from intermediate `buf`, the final result can only contains 0 or 1 data. + +#### Initialization and Cleanup Interface + +```python +def init() +def destroy() +``` + +Description: init() does the work of initialization before processing any data; destroy() does the work of cleanup after the data is processed. + +### Python UDF Template + +#### Scalar Template + ```Python def init(): # initialization def destroy(): # destroy -def process(input: datablock) -> tuple[output_type]: - # process input datablock, - # datablock.data(row, col) is to access the python object in location(row,col) - # return tuple object consisted of object of type outputtype +def process(input: datablock) -> tuple[output_type]: ``` -### Implement an Aggregate UDF in Python +Note：process() must be implemeted, init() and destroy() must be defined too but they can do nothing. -The implementation of an aggregate function is described as follows: +#### Aggregate Template ```Python def init(): @@ -311,33 +374,7 @@ def finish(buf: bytes) -> output_type: #return obj of type outputtype ``` -### Python UDF Interface Definition - -#### Scalar interface -```Python -def process(input: datablock) -> tuple[output_type]: -``` -- `input` is a data block two-dimension matrix-like object, of which method `data(row, col)` returns the Python object located at location (`row`, `col`) -- return a Python tuple object, of which each item is a Python object of type `output_type` - -#### Aggregate Interface -```Python -def start() -> bytes: -def reduce(input: datablock, buf: bytes) -> bytes -def finish(buf: bytes) -> output_type: -``` - -- first `start()` is called to return the initial result in type `bytes` -- then the input data are divided into multiple data blocks and for each block `input`, `reduce` is called with the data block `input` and the current result `buf` bytes and generates a new intermediate result buffer. -- finally, the `finish` function is called on the intermediate result `buf` and outputs 0 or 1 data of type `output_type` - - -#### Initialization and Cleanup Interface -```Python -def init() -def destroy() -``` -Implement `init` for initialization and `destroy` for termination. +Note: aggregate UDF requires init(), destroy(), start(), reduce() and finish() to be impemented. start() generates the initial result in buffer, then the input data is divided into multiple row data blocks, reduce() is invoked for each data block `inputs` and intermediate `buf`, finally finish() is invoked to generate final result from the intermediate result `buf`. ### Data Mapping between TDengine SQL and Python UDF @@ -353,15 +390,460 @@ The following table describes the mapping between TDengine SQL data type and Pyt |TIMESTAMP | int | |JSON and other types | Not Supported | -### Installing Python UDF -1. Install Python package `taospyudf` that executes Python UDF -```bash -sudo pip install taospyudf -ldconfig +### Development Guide + +In this section we will demonstrate 5 examples of developing UDF in Python language. In this guide, you will learn the development skills from easy case to hard case, the examples include: +1. A scalar function which accepts only one integer as input and outputs ln(n^2 + 1)。 +2. A scalar function which accepts n integers, like（x1, x2, ..., xn）and output the sum of the product of each input and its sequence number, i.e. x1 + 2 * x2 + ... + n * xn。 +3. A scalar function which accepts a timestamp and output the next closest Sunday of the timestamp. In this case, we will demonstrate how to use 3rd party library `moment`. +4. An aggregate function which calculates the difference between the maximum and the minimum of a specific column, i.e. same functionality of built-in spread(). + +In the guide, some debugging skills of using Python UDF will be explained too. + +We assume you are using Linux system and already have TDengine 3.0.4.0+ and Python 3.x. + +Note:**You can't use print() function to output log inside a UDF, you have to write the log to a specific file or use logging module of Python.** + +#### Sample 1: Simplest UDF + +This scalar UDF accepts an integer as input and output ln(n^2 + 1). + +Firstly, please compose a Python source code file in your system and save it, e.g. `/root/udf/myfun.py`, the code is like below. + +```python +from math import log + +def init(): + pass + +def destroy(): + pass + +def process(block): + rows, _ = block.shape() + return [log(block.data(i, 0) ** 2 + 1) for i in range(rows)] +``` + +This program consists of 3 functions, init() and destroy() do nothing, but they have to be defined even though there is nothing to do in them because they are critical parts of a python UDF. The most important function is process(), which accepts a data block and the data block object has two methods: +1. shape() returns the number of rows and the number of columns of the data block +2. data(i, j) returns the value at (i,j) in the block + +The output of the process() function of a scalar UDF returns exactly same number of data as the number of input rows. We will ignore the number of columns because we just want to compute on the first column. + +Then, we create the UDF using the SQL command below. + +```sql +create function myfun as '/root/udf/myfun.py' outputtype double language 'Python' +``` + +Here is the output example, it may change a little depending on your version being used. + +```shell + taos> create function myfun as '/root/udf/myfun.py' outputtype double language 'Python'; +Create OK, 0 row(s) affected (0.005202s) +``` + +Then, we used the `show` command to prove the creation of the UDF is successful. + +```text +taos> show functions; + name | +================================= + myfun | +Query OK, 1 row(s) in set (0.005767s) +``` + +Next, we can try to test the function. Before executing the UDF, we need to prepare some data using the command below in TDengine CLI. + +```sql +create database test; +create table t(ts timestamp, v1 int, v2 int, v3 int); +insert into t values('2023-05-01 12:13:14', 1, 2, 3); +insert into t values('2023-05-03 08:09:10', 2, 3, 4); +insert into t values('2023-05-10 07:06:05', 3, 4, 5); +``` + +Execute the UDF to test it: + +```sql +taos> select myfun(v1, v2) from t; + +DB error: udf function execution failure (0.011088s) +``` + +Unfortunately, the UDF execution failed. We need to check the log `udfd` daemon to find out why. + +```shell +tail -10 /var/log/taos/udfd.log +``` + +Below is the output. + +```text +05/24 22:46:28.733545 01665799 UDF ERROR can not load library libtaospyudf.so. error: operation not permitted +05/24 22:46:28.733561 01665799 UDF ERROR can not load python plugin. lib path libtaospyudf.so +``` + +From the error message we can find out that `libtaospyudf.so` was not loaded successfully. Please refer to the [Prepare Environment] section. + +After correcting environment issues, execute the UDF: + +```sql +taos> select myfun(v1) from t; + myfun(v1) | +============================ + 0.693147181 | + 1.609437912 | + 2.302585093 | +``` + +Now, we have finished the first PDF in Python, and learned some basic debugging skills. + +#### Sample 2: Abnormal Processing + +The `myfun` UDF example in sample 1 has passed, but it has two drawbacks. + +1. It accepts only one column of data as input, and doesn't throw exception if you passes multiple columns.这 + +```sql +taos> select myfun(v1, v2) from t; + myfun(v1, v2) | +============================ + 0.693147181 | + 1.609437912 | + 2.302585093 | +``` + +2. `null` value is not processed. We except the program to throw exception and terminate if `null` is passed as input. + +So, we try to optimize the process() function as below. + +```python +def process(block): + rows, cols = block.shape() + if cols > 1: + raise Exception(f"require 1 parameter but given {cols}") + return [ None if block.data(i, 0) is None else log(block.data(i, 0) ** 2 + 1) for i in range(rows)] +``` + +The update the UDF with command below. + +```sql +create or replace function myfun as '/root/udf/myfun.py' outputtype double language 'Python'; +``` + +At this time, if we pass two arguments to `myfun`, the execution would fail. + +```sql +taos> select myfun(v1, v2) from t; + +DB error: udf function execution failure (0.014643s) +``` + +However, the exception is not shown to end user, but displayed in the log file `/var/log/taos/taospyudf.log` + +```text +2023-05-24 23:21:06.790 ERROR [1666188] [doPyUdfScalarProc@507] call pyUdfScalar proc function. context 0x7faade26d180. error: Exception: require 1 parameter but given 2 + +At: + /var/lib/taos//.udf/myfun_3_1884e1281d9.py(12): process + +``` + +Now, we have learned how to update a UDF and check the log of a UDF. + +Note: Prior to TDengine 3.0.5.0 (excluding), updating a UDF requires to restart `taosd` service. After 3.0.5.0, restarting is not required. + +#### Sample 3: UDF with n arguments + +A UDF which accepts n intergers, likee (x1, x2, ..., xn) and output the sum of the product of each value and its sequence number: 1 * x1 + 2 * x2 + ... + n * xn. If there is `null` in the input, then the result is `null`. The difference from sample 1 is that it can accept any number of columns as input and process each column. Assume the program is written in /root/udf/nsum.py: + +```python +def init(): + pass + + +def destroy(): + pass + + +def process(block): + rows, cols = block.shape() + result = [] + for i in range(rows): + total = 0 + for j in range(cols): + v = block.data(i, j) + if v is None: + total = None + break + total += (j + 1) * block.data(i, j) + result.append(total) + return result +``` + +Crate and test the UDF: + +```sql +create function nsum as '/root/udf/nsum.py' outputtype double language 'Python'; +``` + +```sql +taos> insert into t values('2023-05-25 09:09:15', 6, null, 8); +Insert OK, 1 row(s) affected (0.003675s) + +taos> select ts, v1, v2, v3, nsum(v1, v2, v3) from t; + ts | v1 | v2 | v3 | nsum(v1, v2, v3) | +================================================================================================ + 2023-05-01 12:13:14.000 | 1 | 2 | 3 | 14.000000000 | + 2023-05-03 08:09:10.000 | 2 | 3 | 4 | 20.000000000 | + 2023-05-10 07:06:05.000 | 3 | 4 | 5 | 26.000000000 | + 2023-05-25 09:09:15.000 | 6 | NULL | 8 | NULL | +Query OK, 4 row(s) in set (0.010653s) +``` + +Sample 4: Utilize 3rd party package + +A UDF which accepts a timestamp and output the next closed Sunday. This sample requires to use third party package `moment`, you need to install it firslty. + +```shell +pip3 install moment +``` + +Then compose the Python code in /root/udf/nextsunday.py + +```python +import moment + + +def init(): + pass + + +def destroy(): + pass + + +def process(block): + rows, cols = block.shape() + if cols > 1: + raise Exception("require only 1 parameter") + if not type(block.data(0, 0)) is int: + raise Exception("type error") + return [moment.unix(block.data(i, 0)).replace(weekday=7).format('YYYY-MM-DD') + for i in range(rows)] +``` + +UDF framework will map the TDengine timestamp to Python int type, so this function only accepts an integer representing millisecond. process() firstly validates the parameters, then use `moment` to replace the time, format the result and output. + +Create and test the UDF. + +```sql +create function nextsunday as '/root/udf/nextsunday.py' outputtype binary(10) language 'Python'; +``` + +If your `taosd` is started using `systemd`, you man encounter the error below. Next we will show how to debug. + +```sql +taos> select ts, nextsunday(ts) from t; + +DB error: udf function execution failure (1.123615s) +``` + +```shell + tail -20 taospyudf.log +2023-05-25 11:42:34.541 ERROR [1679419] [PyUdf::PyUdf@217] py udf load module failure. error ModuleNotFoundError: No module named 'moment' +``` + +This is because `moment` doesn't exist in the default library search path of python UDF, please check the log file `taosdpyudf.log`. + +```shell +grep 'sys path' taospyudf.log | tail -1 +``` + +```text +2023-05-25 10:58:48.554 INFO [1679419] [doPyOpen@592] python sys path: ['', '/lib/python38.zip', '/lib/python3.8', '/lib/python3.8/lib-dynload', '/lib/python3/dist-packages', '/var/lib/taos//.udf'] +``` + +You may find that the default library search path is `/lib/python3/dist-packages` (just for example, it may be different in your system), but `moment` is installed to `/usr/local/lib/python3.8/dist-packages` (for example, it may be different in your system). Then we change the library search path of python UDF. + +Check `sys.path` + +```python +>>> import sys +>>> ":".join(sys.path) +'/usr/lib/python3.8:/usr/lib/python3.8/lib-dynload:/usr/local/lib/python3.8/dist-packages:/usr/lib/python3/dist-packages' +``` + +Copy the output and edit /var/taos/taos.cfg to add below configuration parameter. + +```shell +UdfdLdLibPath /usr/lib/python3.8:/usr/lib/python3.8/lib-dynload:/usr/local/lib/python3.8/dist-packages:/usr/lib/python3/dist-packages +``` + +Save it, then restart `taosd`, using `systemctl restart taosd`, and test again, it will succeed this time. + +```sql +taos> select ts, nextsunday(ts) from t; + ts | nextsunday(ts) | +=========================================== + 2023-05-01 12:13:14.000 | 2023-05-07 | + 2023-05-03 08:09:10.000 | 2023-05-07 | + 2023-05-10 07:06:05.000 | 2023-05-14 | + 2023-05-25 09:09:15.000 | 2023-05-28 | +Query OK, 4 row(s) in set (1.011474s) +``` + +#### Sample 5: Aggregate Function + +An aggregate function which calculates the difference of the maximum and the minimum in a column. An aggregate funnction takes multiple rows as input and output only one data. The execution process of an aggregate UDF is like map-reduce, the framework divides the input into multiple parts, each mapper processes one block and the reducer aggregates the result of the mappers. The reduce() of Python UDF has the functionality of both map() and reduce(). The reduce() takes two arguments: the data to be processed; and the result of other tasks executing reduce(). For exmaple, assume the code is in `/root/udf/myspread.py`. + +```python +import io +import math +import pickle + +LOG_FILE: io.TextIOBase = None + + +def init(): + global LOG_FILE + LOG_FILE = open("/var/log/taos/spread.log", "wt") + log("init function myspead success") + + +def log(o): + LOG_FILE.write(str(o) + '\n') + + +def destroy(): + log("close log file: spread.log") + LOG_FILE.close() + + +def start(): + return pickle.dumps((-math.inf, math.inf)) + + +def reduce(block, buf): + max_number, min_number = pickle.loads(buf) + log(f"initial max_number={max_number}, min_number={min_number}") + rows, _ = block.shape() + for i in range(rows): + v = block.data(i, 0) + if v > max_number: + log(f"max_number={v}") + max_number = v + if v < min_number: + log(f"min_number={v}") + min_number = v + return pickle.dumps((max_number, min_number)) + + +def finish(buf): + max_number, min_number = pickle.loads(buf) + return max_number - min_number +``` + +In this example, we implemented an aggregate function, and added some logging. +1. init() opens a file for logging +2. log() is the function for logging, it converts the input object to string and output with an end of line +3. destroy() closes the log file \ +4. start() returns the initial buffer for storing the intermediate result +5. reduce() processes each daa block and aggregates the result +6. finish() converts the final buffer() to final result\ + +Create the UDF. + +```sql +create or replace aggregate function myspread as '/root/udf/myspread.py' outputtype double bufsize 128 language 'Python'; +``` + +This SQL command has two important different points from the command creating scalar UDF. +1. keyword `aggregate` is used +2. keyword `bufsize` is used to specify the memory size for storing the intermediate result. In this example, the result is 32 bytes, but we specified 128 bytes for `bufsize`. You can use the `python` CLI to print actual size. + +```python +>>> len(pickle.dumps((12345.6789, 23456789.9877))) +32 +``` + +Test this function, you can see the result is same as built-in spread() function. \ + +```sql +taos> select myspread(v1) from t; + myspread(v1) | +============================ + 5.000000000 | +Query OK, 1 row(s) in set (0.013486s) + +taos> select spread(v1) from t; + spread(v1) | +============================ + 5.000000000 | +Query OK, 1 row(s) in set (0.005501s) +``` + +At last, check the log file, we can see that the reduce() function is executed 3 times, max value is updated 3 times and min value is updated only one time. + +```shell +root@slave11 /var/log/taos $ cat spread.log +init function myspead success +initial max_number=-inf, min_number=inf +max_number=1 +min_number=1 +initial max_number=1, min_number=1 +max_number=2 +max_number=3 +initial max_number=3, min_number=1 +max_number=6 +close log file: spread.log +``` + +### SQL Commands + +1. Create Scalar UDF + +```sql +CREATE FUNCTION function_name AS library_path OUTPUTTYPE output_type LANGUAGE 'Python'; +``` + +2. Create Aggregate UDF + +```sql +CREATE AGGREGATE FUNCTION function_name library_path OUTPUTTYPE output_type LANGUAGE 'Python'; +``` + +3. Update Scalar UDF + +```sql +CREATE OR REPLACE FUNCTION function_name AS OUTPUTTYPE int LANGUAGE 'Python'; +``` + +4. Update Aggregate UDF + +```sql +CREATE OR REPLACE AGGREGATE FUNCTION function_name AS OUTPUTTYPE BUFSIZE buf_size int LANGUAGE 'Python'; +``` + +Note: If keyword `AGGREGATE` used, the UDF will be treated as aggregate UDF despite what it was before; Similarly, if there is no keyword `aggregate`, the UDF will be treated as scalar function despite what it was before. 更 + +5. Show the UDF + +The version of a UDF is increased by one every time it's updated. + +```sql +select * from ins_functions \G; +``` + +6. Show and Drop existing UDF + +```sql +SHOW functions; +DROP FUNCTION function_name; ``` -2. If PYTHONPATH is needed to find Python packages when the Python UDF executes, include the PYTHONPATH contents into the udfdLdLibPath variable of the taos.cfg configuration file -### Python UDF Sample Code +### More Python UDF Samples #### Scalar Function [pybitand](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pybitand.py) The `pybitand` function implements bitwise addition for multiple columns. If there is only one column, the column is returned. The `pybitand` function ignores null values. diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index 92f5d2a857..61592b4ac5 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -271,26 +271,88 @@ select max_vol(vol1,vol2,vol3,deviceid) from battery; ## 用 Python 语言实现 UDF +### 准备环境 + +1. 准备好 Python 运行环境 + +2. 安装 Python 包 `taospyudf` + +```shell +pip3 install taospyudf +``` + +安装过程中会编译 C++ 源码，因此系统上要有 cmake 和 gcc。编译生成的 libtaospyudf.so 文件自动会被复制到 /usr/local/lib/ 目录，因此如果是非 root 用户，安装时需加 sudo。安装完可以检查这个目录是否有了这个文件: + +```shell +root@slave11 ~/udf $ ls -l /usr/local/lib/libtaos* +-rw-r--r-- 1 root root 671344 May 24 22:54 /usr/local/lib/libtaospyudf.so +``` + +然后执行命令 +```shell +ldconfig +``` + +3. 如果 Python UDF 程序执行时，通过 PYTHONPATH 引用其它的包，可以设置 taos.cfg 的 UdfdLdLibPath 变量为PYTHONPATH的内容 + +4. 启动 `taosd` 服务 +细节请参考 [快速开始](../get-started) + +### 接口定义 + +#### 接口概述 + 使用 Python 语言实现 UDF 时，需要实现规定的接口函数 - 标量函数需要实现标量接口函数 process 。 - 聚合函数需要实现聚合接口函数 start ，reduce ，finish。 - 如果需要初始化，实现 init；如果需要清理工作，实现 destroy。 -### 用 Python 实现标量函数 +#### 标量函数接口 +```Python +def process(input: datablock) -> tuple[output_type]: +``` + +说明： + - input:datablock 类似二维矩阵，通过成员方法 data(row,col)返回位于 row 行，col 列的 python 对象 + - 返回值是一个 Python 对象元组，每个元素类型为输出类型。 + +#### 聚合函数接口 +```Python +def start() -> bytes: +def reduce(inputs: datablock, buf: bytes) -> bytes +def finish(buf: bytes) -> output_type: +``` + +说明： + - 首先调用 start 生成最初结果 buffer + - 然后输入数据会被分为多个行数据块，对每个数据块 inputs 和当前中间结果 buf 调用 reduce，得到新的中间结果 + - 最后再调用 finish 从中间结果 buf 产生最终输出，最终输出只能含 0 或 1 条数据。 + +#### 初始化和销毁接口 +```Python +def init() +def destroy() +``` + +说明： + - init 完成初始化工作 + - destroy 完成清理工作 + +### 标量函数实现模板 标量函数实现模版如下 + ```Python def init(): # initialization def destroy(): # destroy -def process(input: datablock) -> tuple[output_type]: - # process input datablock, - # datablock.data(row, col) is to access the python object in location(row,col) - # return tuple object consisted of object of type outputtype +def process(input: datablock) -> tuple[output_type]: ``` -### 用 Python 实现聚合函数 +注意：定义标题函数最重要是要实现 process 函数，同时必须定义 init 和 destroy 函数即使什么都不做 + +### 聚合函数实现模板 聚合函数实现模版如下 ```Python @@ -310,34 +372,9 @@ def finish(buf: bytes) -> output_type: #return obj of type outputtype ``` -### Python UDF 接口函数定义 +注意：定义聚合函数最重要是要实现 start, reduce 和 finish，且必须定义 init 和 destroy 函数。start 生成最初结果 buffer，然后输入数据会被分为多个行数据块，对每个数据块 inputs 和当前中间结果 buf 调用 reduce，得到新的中间结果，最后再调用 finish 从中间结果 buf 产生最终输出。 -#### 标量函数接口 -```Python -def process(input: datablock) -> tuple[output_type]: -``` -- input:datablock 类似二维矩阵，通过成员方法 data(row,col)返回位于 row 行，col 列的 python 对象 -- 返回值是一个 Python 对象元组，每个元素类型为输出类型。 - -#### 聚合函数接口 -```Python -def start() -> bytes: -def reduce(inputs: datablock, buf: bytes) -> bytes -def finish(buf: bytes) -> output_type: -``` - -首先调用 start 生成最初结果 buffer，然后输入数据会被分为多个行数据块，对每个数据块 inputs 和当前中间结果 buf 调用 reduce，得到新的中间结果，最后再调用 finish 从中间结果 buf 产生最终输出，最终输出只能含 0 或 1 条数据。 - - -#### 初始化和销毁接口 -```Python -def init() -def destroy() -``` - -其中 init 完成初始化工作。 destroy 完成清理工作。 - -### Python 和 TDengine之间的数据类型映射 +### 数据类型映射 下表描述了TDengine SQL数据类型和Python数据类型的映射。任何类型的NULL值都映射成Python的None值。 @@ -351,15 +388,461 @@ def destroy() |TIMESTAMP | int | |JSON and other types | 不支持 | -### Python UDF 环境的安装 -1. 安装 taospyudf 包。此包执行Python UDF程序。 -```bash -sudo pip install taospyudf -ldconfig +### 开发指南 + +本文内容由浅入深包括 4 个示例程序： +1. 定义一个只接收一个整数的标量函数： 输入 n， 输出 ln(n^2 + 1)。 +2. 定义一个接收 n 个整数的标量函数， 输入 （x1, x2, ..., xn）, 输出每个值和它们的序号的乘积的和： x1 + 2 * x2 + ... + n * xn。 +3. 定义一个标量函数，输入一个时间戳，输出距离这个时间最近的下一个周日。完成这个函数要用到第三方库 moment。我们在这个示例中讲解使用第三方库的注意事项。 +4. 定义一个聚合函数，计算某一列最大值和最小值的差, 也就是实现 TDengien 内置的 spread 函数。 +同时也包含大量实用的 debug 技巧。 +本文假设你用的是 Linux 系统，且已安装好了 TDengine 3.0.4.0+ 和 Python 3.x。 + +注意：**UDF 内无法通过 print 函数输出日志，需要自己写文件或用 python 内置的 logging 库写文件**。 + +#### 最简单的 UDF + +编写一个只接收一个整数的 UDF 函数： 输入 n， 输出 ln(n^2 + 1)。 +首先编写一个 Python 文件，存在系统某个目录，比如 /root/udf/myfun.py 内容如下 + +```python +from math import log + +def init(): + pass + +def destroy(): + pass + +def process(block): + rows, _ = block.shape() + return [log(block.data(i, 0) ** 2 + 1) for i in range(rows)] ``` -2. 如果 Python UDF 程序执行时，通过 PYTHONPATH 引用其它的包，可以设置 taos.cfg 的 UdfdLdLibPath 变量为PYTHONPATH的内容 + +这个文件包含 3 个函数， init 和 destroy 都是空函数，它们是 UDF 的生命周期函数，即使什么都不做也要定义。最关键的是 process 函数， 它接受一个数据块，这个数据块对象有两个方法： +1. shape() 返回数据块的行数和列数 +2. data(i, j) 返回 i 行 j 列的数据 +标量函数的 process 方法传人的数据块有多少行，就需要返回多少个数据。上述代码中我们忽略的列数，因为我们只想对每行的第一个数做计算。 +接下来我们创建对应的 UDF 函数，在 TDengine CLI 中执行下面语句： + +```sql +create function myfun as '/root/udf/myfun.py' outputtype double language 'Python' +``` +其输出如下 + +```shell + taos> create function myfun as '/root/udf/myfun.py' outputtype double language 'Python'; +Create OK, 0 row(s) affected (0.005202s) +``` + +看起来很顺利，接下来 show 一下系统中所有的自定义函数，确认创建成功： + +```text +taos> show functions; + name | +================================= + myfun | +Query OK, 1 row(s) in set (0.005767s) +``` + +接下来就来测试一下这个函数，测试之前先执行下面的 SQL 命令，制造些测试数据，在 TDengine CLI 中执行下述命令 + +```sql +create database test; +create table t(ts timestamp, v1 int, v2 int, v3 int); +insert into t values('2023-05-01 12:13:14', 1, 2, 3); +insert into t values('2023-05-03 08:09:10', 2, 3, 4); +insert into t values('2023-05-10 07:06:05', 3, 4, 5); +``` + +测试 myfun 函数： + +```sql +taos> select myfun(v1, v2) from t; + +DB error: udf function execution failure (0.011088s) +``` + +不幸的是执行失败了，什么原因呢？ +查看 udfd 进程的日志 + +```shell +tail -10 /var/log/taos/udfd.log +``` + +发现以下错误信息： + +```text +05/24 22:46:28.733545 01665799 UDF ERROR can not load library libtaospyudf.so. error: operation not permitted +05/24 22:46:28.733561 01665799 UDF ERROR can not load python plugin. lib path libtaospyudf.so +``` + +错误很明确：没有加载到 Python 插件 libtaospyudf.so，如果遇到此错误，请参考前面的准备环境一节。 + +修复环境错误后再次执行，如下： + +```sql +taos> select myfun(v1) from t; + myfun(v1) | +============================ + 0.693147181 | + 1.609437912 | + 2.302585093 | +``` + +至此，我们完成了第一个 UDF 😊，并学会了简单的 debug 方法。 + +#### 示例二：异常处理 + +上面的 myfun 虽然测试测试通过了，但是有两个缺点： + +1. 这个标量函数只接受 1 列数据作为输入，如果用户传入了多列也不会抛异常。 + +```sql +taos> select myfun(v1, v2) from t; + myfun(v1, v2) | +============================ + 0.693147181 | + 1.609437912 | + 2.302585093 | +``` + +2. 没有处理 null 值。我们期望如果输入有 null，则会抛异常终止执行。 +因此 process 函数改进如下： + +```python +def process(block): + rows, cols = block.shape() + if cols > 1: + raise Exception(f"require 1 parameter but given {cols}") + return [ None if block.data(i, 0) is None else log(block.data(i, 0) ** 2 + 1) for i in range(rows)] +``` + +然后执行下面的语句更新已有的 UDF： + +```sql +create or replace function myfun as '/root/udf/myfun.py' outputtype double language 'Python'; +``` + +再传入 myfun 两个参数，就会执行失败了 + +```sql +taos> select myfun(v1, v2) from t; + +DB error: udf function execution failure (0.014643s) +``` + +但遗憾的是我们自定义的异常信息没有展示给用户，而是在插件的日志文件 /var/log/taos/taospyudf.log 中： + +```text +2023-05-24 23:21:06.790 ERROR [1666188] [doPyUdfScalarProc@507] call pyUdfScalar proc function. context 0x7faade26d180. error: Exception: require 1 parameter but given 2 + +At: + /var/lib/taos//.udf/myfun_3_1884e1281d9.py(12): process + +``` + +至此，我们学会了如何更新 UDF，并查看 UDF 输出的错误日志。 +（注：如果 UDF 更新后未生效，在 TDengine 3.0.5.0 以前（不含）的版本中需要重启 taosd，在 3.0.5.0 及之后的版本中不需要重启 taosd 即可生效。） + +#### 示例三： 接收 n 个参数的 UDF + +编写一个 UDF：输入（x1, x2, ..., xn）, 输出每个值和它们的序号的乘积的和： 1 * x1 + 2 * x2 + ... + n * xn。如果 x1 至 xn 中包含 null，则结果为 null。 +这个示例与示例一的区别是，可以接受任意多列作为输入，且要处理每一列的值。编写 UDF 文件 /root/udf/nsum.py： + +```python +def init(): + pass + + +def destroy(): + pass + + +def process(block): + rows, cols = block.shape() + result = [] + for i in range(rows): + total = 0 + for j in range(cols): + v = block.data(i, j) + if v is None: + total = None + break + total += (j + 1) * block.data(i, j) + result.append(total) + return result +``` + +创建 UDF： + +```sql +create function nsum as '/root/udf/nsum.py' outputtype double language 'Python'; +``` + +测试 UDF： + +```sql +taos> insert into t values('2023-05-25 09:09:15', 6, null, 8); +Insert OK, 1 row(s) affected (0.003675s) + +taos> select ts, v1, v2, v3, nsum(v1, v2, v3) from t; + ts | v1 | v2 | v3 | nsum(v1, v2, v3) | +================================================================================================ + 2023-05-01 12:13:14.000 | 1 | 2 | 3 | 14.000000000 | + 2023-05-03 08:09:10.000 | 2 | 3 | 4 | 20.000000000 | + 2023-05-10 07:06:05.000 | 3 | 4 | 5 | 26.000000000 | + 2023-05-25 09:09:15.000 | 6 | NULL | 8 | NULL | +Query OK, 4 row(s) in set (0.010653s) +``` + +示例三：使用第三方库 + +编写一个 UDF，输入一个时间戳，输出距离这个时间最近的下一个周日。比如今天是 2023-05-25， 则下一个周日是 2023-05-28。 +完成这个函数要用到第三方库 momen。先安装这个库： + +```shell +pip3 install moment +``` + +然后编写 UDF 文件 /root/udf/nextsunday.py + +```python +import moment + + +def init(): + pass + + +def destroy(): + pass + + +def process(block): + rows, cols = block.shape() + if cols > 1: + raise Exception("require only 1 parameter") + if not type(block.data(0, 0)) is int: + raise Exception("type error") + return [moment.unix(block.data(i, 0)).replace(weekday=7).format('YYYY-MM-DD') + for i in range(rows)] +``` + +UDF 框架会将 TDengine 的 timestamp 类型映射为 Python 的 int 类型，所以这个函数只接受一个表示毫秒数的整数。process 方法先做参数检查，然后用 moment 包替换时间的星期为星期日，最后格式化输出。输出的字符串长度是固定的10个字符长，因此可以这样创建 UDF 函数： + +```sql +create function nextsunday as '/root/udf/nextsunday.py' outputtype binary(10) language 'Python'; +``` + +此时测试函数，如果你是用 systemctl 启动的 taosd，肯定会遇到错误： + +```sql +taos> select ts, nextsunday(ts) from t; + +DB error: udf function execution failure (1.123615s) +``` + +```shell + tail -20 taospyudf.log +2023-05-25 11:42:34.541 ERROR [1679419] [PyUdf::PyUdf@217] py udf load module failure. error ModuleNotFoundError: No module named 'moment' +``` + +这是因为 “moment” 所在位置不在 python udf 插件默认的库搜索路径中。怎么确认这一点呢？通过以下命令搜索 taospyudf.log: + +```shell +grep 'sys path' taospyudf.log | tail -1 +``` + +输出如下 + +```text +2023-05-25 10:58:48.554 INFO [1679419] [doPyOpen@592] python sys path: ['', '/lib/python38.zip', '/lib/python3.8', '/lib/python3.8/lib-dynload', '/lib/python3/dist-packages', '/var/lib/taos//.udf'] +``` + +发现 python udf 插件默认搜索的第三方库安装路径是： /lib/python3/dist-packages，而 moment 默认安装到了 /usr/local/lib/python3.8/dist-packages。下面我们修改 python udf 插件默认的库搜索路径。 +先打开 python3 命令行，查看当前的 sys.path + +```python +>>> import sys +>>> ":".join(sys.path) +'/usr/lib/python3.8:/usr/lib/python3.8/lib-dynload:/usr/local/lib/python3.8/dist-packages:/usr/lib/python3/dist-packages' +``` + +复制上面脚本的输出的字符串，然后编辑 /var/taos/taos.cfg 加入以下配置： + +```shell +UdfdLdLibPath /usr/lib/python3.8:/usr/lib/python3.8/lib-dynload:/usr/local/lib/python3.8/dist-packages:/usr/lib/python3/dist-packages +``` + +保存后执行 systemctl restart taosd, 再测试就不报错了： + +```sql +taos> select ts, nextsunday(ts) from t; + ts | nextsunday(ts) | +=========================================== + 2023-05-01 12:13:14.000 | 2023-05-07 | + 2023-05-03 08:09:10.000 | 2023-05-07 | + 2023-05-10 07:06:05.000 | 2023-05-14 | + 2023-05-25 09:09:15.000 | 2023-05-28 | +Query OK, 4 row(s) in set (1.011474s) +``` + +#### 示例五：聚合函数 + +编写一个聚合函数，计算某一列最大值和最小值的差。 +聚合函数与标量函数的区别是：标量函数是多行输入对应多个输出，聚合函数是多行输入对应一个输出。聚合函数的执行过程有点像经典的 map-reduce 框架的执行过程，框架把数据分成若干块，每个 mapper 处理一个块，reducer 再把 mapper 的结果做聚合。不一样的地方在于，对于 TDengine Python UDF 中的 reduce 函数既有 map 的功能又有 reduce 的功能。reduce 函数接受两个参数：一个是自己要处理的数据，一个是别的任务执行 reduce 函数的处理结果。如下面的示例 /root/udf/myspread.py: + +```python +import io +import math +import pickle + +LOG_FILE: io.TextIOBase = None + + +def init(): + global LOG_FILE + LOG_FILE = open("/var/log/taos/spread.log", "wt") + log("init function myspead success") + + +def log(o): + LOG_FILE.write(str(o) + '\n') + + +def destroy(): + log("close log file: spread.log") + LOG_FILE.close() + + +def start(): + return pickle.dumps((-math.inf, math.inf)) + + +def reduce(block, buf): + max_number, min_number = pickle.loads(buf) + log(f"initial max_number={max_number}, min_number={min_number}") + rows, _ = block.shape() + for i in range(rows): + v = block.data(i, 0) + if v > max_number: + log(f"max_number={v}") + max_number = v + if v < min_number: + log(f"min_number={v}") + min_number = v + return pickle.dumps((max_number, min_number)) + + +def finish(buf): + max_number, min_number = pickle.loads(buf) + return max_number - min_number +``` + +在这个示例中我们不光定义了一个聚合函数，还添加记录执行日志的功能，讲解如下： +1. init 函数不再是空函数，而是打开了一个文件用于写执行日志 +2. log 函数是记录日志的工具，自动将传入的对象转成字符串，加换行符输出 +3. destroy 函数用来在执行结束关闭文件 +4. start 返回了初始的 buffer，用来存聚合函数的中间结果，我们把最大值初始化为负无穷大，最小值初始化为正无穷大 +5. reduce 处理每个数据块并聚合结果 +6. finish 函数将最终的 buffer 转换成最终的输出 +执行下面的 SQL语句创建对应的 UDF： + +```sql +create or replace aggregate function myspread as '/root/udf/myspread.py' outputtype double bufsize 128 language 'Python'; +``` + +这个 SQL 语句与创建标量函数的 SQL 语句有两个重要区别： +1. 增加了 aggregate 关键字 +2. 增加了 bufsize 关键字，用来指定存储中间结果的内存大小，这个数值可以大于实际使用的数值。本例中间结果是两个浮点数组成的 tuple，序列化后实际占用大小只有 32 个字节，但指定的 bufsize 是128，可以用 python 命令行打印实际占用的字节数 + +```python +>>> len(pickle.dumps((12345.6789, 23456789.9877))) +32 +``` + +测试这个函数，可以看到 myspread 的输出结果和内置的 spread 函数的输出结果是一致的。 + +```sql +taos> select myspread(v1) from t; + myspread(v1) | +============================ + 5.000000000 | +Query OK, 1 row(s) in set (0.013486s) + +taos> select spread(v1) from t; + spread(v1) | +============================ + 5.000000000 | +Query OK, 1 row(s) in set (0.005501s) +``` + +最后，查看我们自己打印的执行日志，从日志可以看出，reduce 函数被执行了 3 次。执行过程中 max 值被更新了 4 次， min 值只被更新 1 次。 + +```shell +root@slave11 /var/log/taos $ cat spread.log +init function myspead success +initial max_number=-inf, min_number=inf +max_number=1 +min_number=1 +initial max_number=1, min_number=1 +max_number=2 +max_number=3 +initial max_number=3, min_number=1 +max_number=6 +close log file: spread.log +``` + +通过这个示例，我们学会了如何定义聚合函数，并打印自定义的日志信息。 + +### SQL 命令 + +1. 创建标量函数的语法 + +```sql +CREATE FUNCTION function_name AS library_path OUTPUTTYPE output_type LANGUAGE 'Python'; +``` + +2. 创建聚合函数的语法 + +```sql +CREATE AGGREGATE FUNCTION function_name library_path OUTPUTTYPE output_type LANGUAGE 'Python'; +``` + +3. 更新标量函数 + +```sql +CREATE OR REPLACE FUNCTION function_name AS OUTPUTTYPE int LANGUAGE 'Python'; +``` + +4. 更新聚合函数 + +```sql +CREATE OR REPLACE AGGREGATE FUNCTION function_name AS OUTPUTTYPE BUFSIZE buf_size int LANGUAGE 'Python'; +``` + +注意：如果加了 “AGGREGATE” 关键字，更新之后函数将被当作聚合函数，无论之前是什么类型的函数。相反，如果没有加 “AGGREGATE” 关键字，更新之后的函数将被当作标量函数，无论之前是什么类型的函数。 + +5. 查看函数信息 + + 同名的 UDF 每更新一次，版本号会增加 1。 + +```sql +select * from ins_functions \G; +``` + +6. 查看和删除已有的 UDF + +```sql +SHOW functions; +DROP FUNCTION function_name; +``` + + +上面的命令可以查看 UDF 的完整信息 -### Python UDF 示例代码 +### 更多 Python UDF 示例代码 #### 标量函数示例 [pybitand](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pybitand.py) pybitand 实现多列的按位与功能。如果只有一列，返回这一列。pybitand 忽略空值。 From 730e087d5e72d14aff0578e31d4eda3dd2f52c2e Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Fri, 26 May 2023 19:21:13 +0800 Subject: [PATCH 86/97] Update 09-udf.md --- docs/en/07-develop/09-udf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md index 92bd4efd25..f5f1230026 100644 --- a/docs/en/07-develop/09-udf.md +++ b/docs/en/07-develop/09-udf.md @@ -320,7 +320,7 @@ def process(input: datablock) -> tuple[output_type]: Description: this function prcesses datablock, which is the input; you can use datablock.data(row, col) to access the python object at location(row,col); the output is a tuple object consisted of objects of type outputtype -### Aggregate UDF Interface +#### Aggregate UDF Interface The implementation of an aggregate function is described as follows: From 290a14e2d31d074daaea7b9e801c19f9a40bef91 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Fri, 26 May 2023 19:22:47 +0800 Subject: [PATCH 87/97] Update 09-udf.md --- docs/en/07-develop/09-udf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md index f5f1230026..67109834ba 100644 --- a/docs/en/07-develop/09-udf.md +++ b/docs/en/07-develop/09-udf.md @@ -602,7 +602,7 @@ taos> select ts, v1, v2, v3, nsum(v1, v2, v3) from t; Query OK, 4 row(s) in set (0.010653s) ``` -Sample 4: Utilize 3rd party package +#### Sample 4: Utilize 3rd party package A UDF which accepts a timestamp and output the next closed Sunday. This sample requires to use third party package `moment`, you need to install it firslty. From 3b0cee7866c81e7a3edd97b6feeb849323245ad9 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Fri, 26 May 2023 19:24:46 +0800 Subject: [PATCH 88/97] Update 09-udf.md --- docs/zh/07-develop/09-udf.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index 61592b4ac5..cce9c0b24c 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -338,7 +338,9 @@ def destroy() - init 完成初始化工作 - destroy 完成清理工作 -### 标量函数实现模板 +### Python UDF 函数模板 + +#### 标量函数实现模板 标量函数实现模版如下 @@ -352,7 +354,7 @@ def process(input: datablock) -> tuple[output_type]: 注意：定义标题函数最重要是要实现 process 函数，同时必须定义 init 和 destroy 函数即使什么都不做 -### 聚合函数实现模板 +#### 聚合函数实现模板 聚合函数实现模版如下 ```Python @@ -596,7 +598,7 @@ taos> select ts, v1, v2, v3, nsum(v1, v2, v3) from t; Query OK, 4 row(s) in set (0.010653s) ``` -示例三：使用第三方库 +#### 示例四：使用第三方库 编写一个 UDF，输入一个时间戳，输出距离这个时间最近的下一个周日。比如今天是 2023-05-25， 则下一个周日是 2023-05-28。 完成这个函数要用到第三方库 momen。先安装这个库： From 5abfff6b53718d35fe592361d3d40345f5148b26 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Fri, 26 May 2023 19:30:34 +0800 Subject: [PATCH 89/97] Update 09-udf.md --- docs/zh/07-develop/09-udf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index cce9c0b24c..d964a2a1bd 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -871,7 +871,7 @@ pyl2norm 实现了输入列的所有数据的二阶范数，即对每个数据 -### 聚合函数示例 [pycumsum](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pycumsum.py) +#### 聚合函数示例 [pycumsum](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pycumsum.py) pycumsum 使用 numpy 计算输入列所有数据的累积和。
From b38e2530191715ec0e3db9e2ff97347d5c7c349a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 26 May 2023 22:51:58 +0800 Subject: [PATCH 90/97] fix(stream): set the skip offset when resume stream task. --- include/libs/wal/wal.h | 3 +++ source/dnode/vnode/src/tq/tq.c | 27 ++++++++++++++++----------- source/dnode/vnode/src/tq/tqRestore.c | 10 ++++++++++ source/libs/wal/src/walRead.c | 8 ++++++++ 4 files changed, 37 insertions(+), 11 deletions(-) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 35a6838b2e..a081527144 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -149,6 +149,7 @@ struct SWalReader { TdFilePtr pIdxFile; int64_t curFileFirstVer; int64_t curVersion; + int64_t skipToVersion; // skip data and jump to destination version, usually used by stream resume ignoring untreated data int64_t capacity; TdThreadMutex mutex; SWalFilterCond cond; @@ -200,6 +201,8 @@ int32_t walReaderSeekVer(SWalReader *pRead, int64_t ver); int32_t walNextValidMsg(SWalReader *pRead); int64_t walReaderGetCurrentVer(const SWalReader *pReader); int64_t walReaderGetValidFirstVer(const SWalReader *pReader); +int64_t walReaderGetSkipToVersion(SWalReader *pReader); +void walReaderSetSkipToVersion(SWalReader *pReader, int64_t ver); void walReaderValidVersionRange(SWalReader *pReader, int64_t *sver, int64_t *ever); void walReaderVerifyOffset(SWalReader *pWalReader, STqOffsetVal* pOffset); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c6077a3f8f..633b15a1fb 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1303,19 +1303,22 @@ int32_t tqProcessTaskPauseReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg int32_t tqProcessTaskResumeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { SVResumeStreamTaskReq* pReq = (SVResumeStreamTaskReq*)msg; + + int32_t vgId = pTq->pStreamMeta->vgId; SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->taskId); if (pTask) { atomic_store_8(&pTask->status.taskStatus, pTask->status.keepTaskStatus); // no lock needs to secure the access of the version - if (pReq->igUntreated && pTask->taskLevel == TASK_LEVEL__SOURCE) { // discard all the data when the stream task is suspended. - pTask->chkInfo.currentVer = sversion; - walReaderSeekVer(pTask->exec.pWalReader, sversion); - tqDebug("vgId:%d s-task:%s resume to normal from the latest version:%" PRId64 ", vnode ver:%" PRId64 ", schedStatus:%d", pTq->pStreamMeta->vgId, - pTask->id.idStr, pTask->chkInfo.currentVer, sversion, pTask->status.schedStatus); + if (pReq->igUntreated && pTask->taskLevel == TASK_LEVEL__SOURCE) { + // discard all the data when the stream task is suspended. + walReaderSetSkipToVersion(pTask->exec.pWalReader, sversion); + tqDebug("vgId:%d s-task:%s resume to exec, prev paused version:%" PRId64 ", start from vnode ver:%" PRId64 + ", schedStatus:%d", + vgId, pTask->id.idStr, pTask->chkInfo.currentVer, sversion, pTask->status.schedStatus); } else { // from the previous paused version and go on - tqDebug("vgId:%d s-task:%s resume to normal from paused ver:%" PRId64 ", vnode ver:%" PRId64 ", schedStatus:%d", pTq->pStreamMeta->vgId, - pTask->id.idStr, pTask->chkInfo.currentVer, sversion, pTask->status.schedStatus); + tqDebug("vgId:%d s-task:%s resume to exec, from paused ver:%" PRId64 ", vnode ver:%" PRId64 ", schedStatus:%d", + vgId, pTask->id.idStr, pTask->chkInfo.currentVer, sversion, pTask->status.schedStatus); } if (pTask->taskLevel == TASK_LEVEL__SOURCE && taosQueueItemSize(pTask->inputQueue->queue) == 0) { @@ -1324,6 +1327,8 @@ int32_t tqProcessTaskResumeReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms streamSchedExec(pTask); } streamMetaReleaseTask(pTq->pStreamMeta, pTask); + } else { + tqError("vgId:%d failed to find the s-task:0x%x for resume stream task", vgId, pReq->taskId); } return 0; @@ -1432,7 +1437,7 @@ int32_t tqStartStreamTasks(STQ* pTq) { int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); if (numOfTasks == 0) { tqInfo("vgId:%d no stream tasks exist", vgId); - taosWUnLockLatch(&pTq->pStreamMeta->lock); + taosWUnLockLatch(&pMeta->lock); return 0; } @@ -1440,7 +1445,7 @@ int32_t tqStartStreamTasks(STQ* pTq) { if (pMeta->walScanCounter > 1) { tqDebug("vgId:%d wal read task has been launched, remain scan times:%d", vgId, pMeta->walScanCounter); - taosWUnLockLatch(&pTq->pStreamMeta->lock); + taosWUnLockLatch(&pMeta->lock); return 0; } @@ -1448,7 +1453,7 @@ int32_t tqStartStreamTasks(STQ* pTq) { if (pRunReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("vgId:%d failed to create msg to start wal scanning to launch stream tasks, code:%s", vgId, terrstr()); - taosWUnLockLatch(&pTq->pStreamMeta->lock); + taosWUnLockLatch(&pMeta->lock); return -1; } @@ -1459,7 +1464,7 @@ int32_t tqStartStreamTasks(STQ* pTq) { SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &msg); - taosWUnLockLatch(&pTq->pStreamMeta->lock); + taosWUnLockLatch(&pMeta->lock); return 0; } diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index 6e3036a66b..fe80f48691 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -87,6 +87,16 @@ static int32_t doSetOffsetForWalReader(SStreamTask *pTask, int32_t vgId) { } } + int64_t skipToVer = walReaderGetSkipToVersion(pTask->exec.pWalReader); + if (skipToVer != 0 && skipToVer > pTask->chkInfo.currentVer) { + int32_t code = walReaderSeekVer(pTask->exec.pWalReader, skipToVer); + if (code != TSDB_CODE_SUCCESS) { // no data in wal, quit + return code; + } + + tqDebug("vgId:%d s-task:%s wal reader jump to ver:%" PRId64, vgId, pTask->id.idStr, skipToVer); + } + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index c29d82bcf3..0eb3d6ef09 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -108,6 +108,14 @@ int32_t walNextValidMsg(SWalReader *pReader) { int64_t walReaderGetCurrentVer(const SWalReader *pReader) { return pReader->curVersion; } int64_t walReaderGetValidFirstVer(const SWalReader *pReader) { return walGetFirstVer(pReader->pWal); } +void walReaderSetSkipToVersion(SWalReader *pReader, int64_t ver) { atomic_store_64(&pReader->skipToVersion, ver); } + +// this function is NOT multi-thread safe, and no need to be. +int64_t walReaderGetSkipToVersion(SWalReader *pReader) { + int64_t newVersion = pReader->skipToVersion; + pReader->skipToVersion = 0; + return newVersion; +} void walReaderValidVersionRange(SWalReader *pReader, int64_t *sver, int64_t *ever) { *sver = walGetFirstVer(pReader->pWal); From c91029f92fcb369633a564efc0583a99c91cac3f Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 27 May 2023 19:28:47 +0800 Subject: [PATCH 91/97] docs: fix typos (#21503) --- docs/en/07-develop/09-udf.md | 28 ++++++++++++------------- docs/en/14-reference/12-config/index.md | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md index b0124b9c3f..5a98ff2789 100644 --- a/docs/en/07-develop/09-udf.md +++ b/docs/en/07-develop/09-udf.md @@ -10,7 +10,7 @@ User-defined functions can be scalar functions or aggregate functions. Scalar fu TDengine supports user-defined functions written in C or Python. This document describes the usage of user-defined functions. -## Implement a UDF in C +## Implement a UDF in C When you create a user-defined function, you must implement standard interface functions: - For scalar functions, implement the `scalarfn` interface function. @@ -111,13 +111,13 @@ Interface functions return a value that indicates whether the operation was succ For information about the parameters for interface functions, see Data Model #### Scalar Interface - `int32_t scalarfn(SUdfDataBlock* inputDataBlock, SUdfColumn *resultColumn)` - + `int32_t scalarfn(SUdfDataBlock* inputDataBlock, SUdfColumn *resultColumn)` + Replace `scalarfn` with the name of your function. This function performs scalar calculations on data blocks. You can configure a value through the parameters in the `resultColumn` structure. The parameters in the function are defined as follows: - inputDataBlock: The data block to input. - - resultColumn: The column to output. The column to output. + - resultColumn: The column to output. The column to output. #### Aggregate Interface @@ -197,7 +197,7 @@ The data structure is described as follows: - The SUdfDataBlock block includes the number of rows (numOfRows) and the number of columns (numCols). udfCols[i] (0 <= i <= numCols-1) indicates that each column is of type SUdfColumn. - SUdfColumn includes the definition of the data type of the column (colMeta) and the data in the column (colData). - The member definitions of SUdfColumnMeta are the same as the data type definitions in `taos.h`. -- The data in SUdfColumnData can become longer. varLenCol indicates variable-length data, and fixLenCol indicates fixed-length data. +- The data in SUdfColumnData can become longer. varLenCol indicates variable-length data, and fixLenCol indicates fixed-length data. - SUdfInterBuf defines the intermediate structure `buffer` and the number of results in the buffer `numOfResult`. Additional functions are defined in `taosudf.h` to make it easier to work with these structures. @@ -285,9 +285,9 @@ def init(): def destroy(): # destroy def process(input: datablock) -> tuple[output_type]: - # process input datablock, + # process input datablock, # datablock.data(row, col) is to access the python object in location(row,col) - # return tuple object consisted of object of type outputtype + # return tuple object consisted of object of type outputtype ``` ### Implement an Aggregate UDF in Python @@ -303,12 +303,12 @@ def start() -> bytes: #return serialize(init_state) def reduce(inputs: datablock, buf: bytes) -> bytes # deserialize buf to state - # reduce the inputs and state into new_state. - # use inputs.data(i,j) to access python ojbect of location(i,j) + # reduce the inputs and state into new_state. + # use inputs.data(i,j) to access python object of location(i,j) # serialize new_state into new_state_bytes - return new_state_bytes + return new_state_bytes def finish(buf: bytes) -> output_type: - #return obj of type outputtype + #return obj of type outputtype ``` ### Python UDF Interface Definition @@ -328,7 +328,7 @@ def finish(buf: bytes) -> output_type: ``` - first `start()` is called to return the initial result in type `bytes` -- then the input data are divided into multiple data blocks and for each block `input`, `reduce` is called with the data block `input` and the current result `buf` bytes and generates a new intermediate result buffer. +- then the input data are divided into multiple data blocks and for each block `input`, `reduce` is called with the data block `input` and the current result `buf` bytes and generates a new intermediate result buffer. - finally, the `finish` function is called on the intermediate result `buf` and outputs 0 or 1 data of type `output_type` @@ -337,7 +337,7 @@ def finish(buf: bytes) -> output_type: def init() def destroy() ``` -Implement `init` for initialization and `destroy` for termination. +Implement `init` for initialization and `destroy` for termination. ### Data Mapping between TDengine SQL and Python UDF @@ -360,7 +360,7 @@ sudo pip install taospyudf ldconfig ``` 2. If PYTHONPATH is needed to find Python packages when the Python UDF executes, include the PYTHONPATH contents into the udfdLdLibPath variable of the taos.cfg configuration file - + ### Python UDF Sample Code #### Scalar Function [pybitand](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pybitand.py) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 3ce63fb6cc..52ded6208a 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -111,7 +111,7 @@ The parameters described in this document by the effect that they have on the sy | Attribute | Description | | ------------- | ---------------------------------------------- | | Applicable | Client/Server | -| Meaning | The maximum waiting time to get avaliable conn | +| Meaning | The maximum waiting time to get available conn | | Value Range | 10-50000000(ms) | | Default Value | 500000 | From 884e7d0db669c60458af1287e0b4d003ab3fd9d5 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 28 May 2023 00:16:16 +0800 Subject: [PATCH 92/97] docs: fix udf zh typos (#21504) * docs: fix typos * docs: fix udf typos --- docs/zh/07-develop/09-udf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index 92f5d2a857..e43275d5e0 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -303,7 +303,7 @@ def start() -> bytes: def reduce(inputs: datablock, buf: bytes) -> bytes # deserialize buf to state # reduce the inputs and state into new_state. - # use inputs.data(i,j) to access python ojbect of location(i,j) + # use inputs.data(i,j) to access python object of location(i,j) # serialize new_state into new_state_bytes return new_state_bytes def finish(buf: bytes) -> output_type: From c7e2347dfdea75a813ee0ddf5a7cc7648cd4f677 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Mon, 29 May 2023 08:14:36 +0800 Subject: [PATCH 93/97] Update 09-udf.md --- docs/en/07-develop/09-udf.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md index 67109834ba..66e944d654 100644 --- a/docs/en/07-develop/09-udf.md +++ b/docs/en/07-develop/09-udf.md @@ -295,7 +295,7 @@ Then execute the command below. ldconfig ``` -3. If you want to utilize some 3rd party python packages using `PYTHONPATH`, please set configuration parameter `UdfdLdLibPath` to the value of `PYTHONPATH` before starting `taosd`. +3. If you want to utilize some 3rd party python packages in your Python UDF, please set configuration parameter `UdfdLdLibPath` to the value of `PYTHONPATH` before starting `taosd`. 4. Launch `taosd` service @@ -503,7 +503,7 @@ Now, we have finished the first PDF in Python, and learned some basic debugging The `myfun` UDF example in sample 1 has passed, but it has two drawbacks. -1. It accepts only one column of data as input, and doesn't throw exception if you passes multiple columns.这 +1. It accepts only one column of data as input, and doesn't throw exception if you passes multiple columns. ```sql taos> select myfun(v1, v2) from t; From cb82453a43ca42354eea7046ee9f64825c4566ea Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Mon, 29 May 2023 08:21:00 +0800 Subject: [PATCH 94/97] Update 09-udf.md --- docs/en/07-develop/09-udf.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md index 66e944d654..d7a3fbd032 100644 --- a/docs/en/07-develop/09-udf.md +++ b/docs/en/07-develop/09-udf.md @@ -503,7 +503,7 @@ Now, we have finished the first PDF in Python, and learned some basic debugging The `myfun` UDF example in sample 1 has passed, but it has two drawbacks. -1. It accepts only one column of data as input, and doesn't throw exception if you passes multiple columns. +1. It the program accepts only one column of data as input, but it doesn't throw exception if you passes multiple columns. ```sql taos> select myfun(v1, v2) from t; @@ -514,7 +514,7 @@ taos> select myfun(v1, v2) from t; 2.302585093 | ``` -2. `null` value is not processed. We except the program to throw exception and terminate if `null` is passed as input. +2. `null` value is not processed. We expect the program to throw exception and terminate if `null` is passed as input. So, we try to optimize the process() function as below. @@ -642,7 +642,7 @@ Create and test the UDF. create function nextsunday as '/root/udf/nextsunday.py' outputtype binary(10) language 'Python'; ``` -If your `taosd` is started using `systemd`, you man encounter the error below. Next we will show how to debug. +If your `taosd` is started using `systemd`, you may encounter the error below. Next we will show how to debug. ```sql taos> select ts, nextsunday(ts) from t; @@ -826,7 +826,7 @@ CREATE OR REPLACE FUNCTION function_name AS OUTPUTTYPE int LANGUAGE 'Python'; CREATE OR REPLACE AGGREGATE FUNCTION function_name AS OUTPUTTYPE BUFSIZE buf_size int LANGUAGE 'Python'; ``` -Note: If keyword `AGGREGATE` used, the UDF will be treated as aggregate UDF despite what it was before; Similarly, if there is no keyword `aggregate`, the UDF will be treated as scalar function despite what it was before. 更 +Note: If keyword `AGGREGATE` used, the UDF will be treated as aggregate UDF despite what it was before; Similarly, if there is no keyword `aggregate`, the UDF will be treated as scalar function despite what it was before. 5. Show the UDF From 6ccefe9891302176f54228cfffe446ed156c1d8b Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Mon, 29 May 2023 08:23:31 +0800 Subject: [PATCH 95/97] Update 09-udf.md --- docs/en/07-develop/09-udf.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md index d7a3fbd032..51c4c112c0 100644 --- a/docs/en/07-develop/09-udf.md +++ b/docs/en/07-develop/09-udf.md @@ -667,7 +667,7 @@ grep 'sys path' taospyudf.log | tail -1 You may find that the default library search path is `/lib/python3/dist-packages` (just for example, it may be different in your system), but `moment` is installed to `/usr/local/lib/python3.8/dist-packages` (for example, it may be different in your system). Then we change the library search path of python UDF. -Check `sys.path` +Check `sys.path`, which must include the packages you install with pip3 command previously, as shown below: ```python >>> import sys @@ -683,6 +683,8 @@ UdfdLdLibPath /usr/lib/python3.8:/usr/lib/python3.8/lib-dynload:/usr/local/lib/p Save it, then restart `taosd`, using `systemctl restart taosd`, and test again, it will succeed this time. +Note: If your cluster consists of multiple `taosd` instances, you have to repeat same process for each of them. + ```sql taos> select ts, nextsunday(ts) from t; ts | nextsunday(ts) | From 6a741cc9014e039df68343329031a05f0e87452e Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Mon, 29 May 2023 09:45:02 +0800 Subject: [PATCH 96/97] Update 09-udf.md --- docs/zh/07-develop/09-udf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index 8e345277b3..ae11273a39 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -296,7 +296,7 @@ ldconfig 3. 如果 Python UDF 程序执行时，通过 PYTHONPATH 引用其它的包，可以设置 taos.cfg 的 UdfdLdLibPath 变量为PYTHONPATH的内容 4. 启动 `taosd` 服务 -细节请参考 [快速开始](../get-started) +细节请参考 [快速开始](../../get-started) ### 接口定义 From b2d2e35450132649cfa5d12fadd1a20e449dcfd3 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Mon, 29 May 2023 09:52:03 +0800 Subject: [PATCH 97/97] Update 09-udf.md --- docs/en/07-develop/09-udf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md index 62cdc68e84..825d3c6f8b 100644 --- a/docs/en/07-develop/09-udf.md +++ b/docs/en/07-develop/09-udf.md @@ -299,7 +299,7 @@ ldconfig 4. Launch `taosd` service -Please refer to [Get Started](../get-started) +Please refer to [Get Started](../../get-started) ### Interface definition