From 73d4fa48f9378510fe885d7363faaeb7d6f111b7 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Mon, 8 Jul 2024 18:22:33 +0800 Subject: [PATCH] support month/year tsma interval --- include/common/ttime.h | 3 + source/common/src/ttime.c | 93 +++++++++++++++++++++++++ source/libs/parser/src/parTranslater.c | 65 +---------------- source/libs/planner/src/planOptimizer.c | 17 +++-- source/util/test/utilTests.cpp | 92 ++++++++++++++++++++++++ tests/system-test/2-query/tsma.py | 4 +- 6 files changed, 204 insertions(+), 70 deletions(-) diff --git a/include/common/ttime.h b/include/common/ttime.h index d890b729d4..f50b5ee9d7 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -127,6 +127,9 @@ int32_t TEST_char2ts(const char* format, int64_t* ts, int32_t precision, const c /// @return 0 success, other fail int32_t offsetOfTimezone(char* tzStr, int64_t* offset); +bool checkRecursiveTsmaInterval(int64_t baseInterval, int8_t baseUnit, int64_t interval, int8_t unit, int8_t precision, + bool checkEq); + #ifdef __cplusplus } #endif diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 52379f10d7..e7a0f51b9e 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -1969,3 +1969,96 @@ int32_t TEST_char2ts(const char* format, int64_t* ts, int32_t precision, const c taosArrayDestroy(formats); return code; } + +static int8_t UNIT_INDEX[26] = {/*a*/ 2, 0, -1, 6, -1, -1, -1, + /*h*/ 5, -1, -1, -1, -1, 4, 8, + /*o*/ -1, -1, -1, -1, 3, -1, + /*u*/ 1, -1, 7, -1, 9, -1}; + +#define GET_UNIT_INDEX(idx) UNIT_INDEX[(idx) - 97] + +static int64_t MATRIX[10][11] = { /* ns, us, ms, s, min, h, d, w, month, y*/ + /*ns*/ { 1, 1000, 0}, + /*us*/ {1000, 1, 1000, 0}, + /*ms*/ { 0, 1000, 1, 1000, 0}, + /*s*/ { 0, 0, 1000, 1, 60, 0}, + /*min*/ { 0, 0, 0, 60, 1, 60, 0}, + /*h*/ { 0, 0, 0, 0, 60, 1, 24, 0}, + /*d*/ { 0, 0, 0, 0, 0, 24, 1, 7, 1, 0}, + /*w*/ { 0, 0, 0, 0, 0, 0, 7, 1, -1, 0}, + /*mon*/ { 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0}, + /*y*/ { 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0}}; + +static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx, bool checkEq) { + if (MATRIX[baseIdx][idx] == -1) return false; + if (baseIdx == idx) { + if (interval < baseInterval) return false; + if (checkEq && interval == baseInterval) return false; + return interval % baseInterval == 0; + } + int8_t next = baseIdx + 1; + int64_t val = MATRIX[baseIdx][next]; + while (val != 0 && next <= idx) { + if (val == -1) { + next++; + val = MATRIX[baseIdx][next]; + continue; + } + if (val % baseInterval == 0 || baseInterval % val == 0) { + int8_t extra = baseInterval >= val ? 0 : 1; + if (!recursiveTsmaCheckRecursive(baseInterval / val + extra, next, interval, idx, (extra == 0) && checkEq)) { + next++; + val = MATRIX[baseIdx][next]; + continue; + } else { + return true; + } + } else { + return false; + } + } + return false; +} + +static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx, bool checkEq) { + if (MATRIX[baseIdx][idx] == -1) return false; + + if (baseIdx == idx) { + if (interval < baseInterval) return false; + if (checkEq && interval == baseInterval) return false; + return interval % baseInterval == 0; + } + + int8_t next = baseIdx - 1; + int64_t val = MATRIX[baseIdx][next]; + while (val != 0 && next >= 0) { + return recursiveTsmaCheckRecursiveReverse(baseInterval * val, next, interval, idx, checkEq); + } + return false; +} + +/* + * @breif check if tsma with param [interval], [unit] can create based on base tsma with baseInterval and baseUnit + * @param baseInterval, baseUnit, interval/unit of base tsma + * @param interval the tsma interval going to create. Not that if unit is not calander unit, then interval has already been + * translated to TICKS of [precision] + * @param unit the tsma unit going to create + * @precision the precision of this db + * @ret true the tsma can be created, else cannot + * */ +bool checkRecursiveTsmaInterval(int64_t baseInterval, int8_t baseUnit, int64_t interval, int8_t unit, int8_t precision, bool checkEq) { + bool baseIsCalendarDuration = IS_CALENDAR_TIME_DURATION(baseUnit); + if (!baseIsCalendarDuration) baseInterval = convertTimeFromPrecisionToUnit(baseInterval, precision, baseUnit); + bool isCalendarDuration = IS_CALENDAR_TIME_DURATION(unit); + if (!isCalendarDuration) interval = convertTimeFromPrecisionToUnit(interval, precision, unit); + + bool needCheckEq = baseIsCalendarDuration == isCalendarDuration && checkEq; + + int8_t baseIdx = GET_UNIT_INDEX(baseUnit), idx = GET_UNIT_INDEX(unit); + if (baseIdx <= idx) { + return recursiveTsmaCheckRecursive(baseInterval, baseIdx, interval, idx, needCheckEq); + } else { + return recursiveTsmaCheckRecursiveReverse(baseInterval, baseIdx, interval, idx, true); + } + return true; +} diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3e1e7896d6..dc8c9309c4 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -11125,68 +11125,6 @@ static int32_t rewriteTSMAFuncs(STranslateContext* pCxt, SCreateTSMAStmt* pStmt, return code; } -static int8_t UNIT_INDEX[26] = {/*a*/ 2, 0, -1, 6, -1, -1, -1, - /*h*/ 5, -1, -1, -1, -1, 4, 8, - /*o*/ -1, -1, -1, -1, 3, -1, - /*u*/ 1, -1, 7, -1, 9, -1}; -static int64_t MATRIX[10][11] = { /* ns, us, ms, s, min, h, d, w, month, y*/ - /*ns*/ { 1, 1000, 0}, - /*us*/ {1000, 1, 1000, 0}, - /*ms*/ { -1, 1000, 1, 1000, 0}, - /*s*/ { -1, -1, 1000, 1, 60, 0}, - /*min*/ { -1, -1, -1, 60, 1, 60, 0}, - /*h*/ { -1, -1, -1, -1, 60, 1, 24, 0}, - /*d*/ { -1, -1, -1, -1, -1, 24, 1, 7, 1, 0}, - /*w*/ { -1, -1, -1, -1, -1, -1, 7, 1, -1, 0}, - /*mon*/ { -1, -1, -1, -1, -1, -1, -1, -1, 1, 12, 0}, - /*y*/ { -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 0}}; - -static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx, int8_t precision) { - if (MATRIX[baseIdx][idx] == -1) return false; - if (baseIdx == idx) { - if (interval < baseInterval) return false; - return interval % baseInterval == 0; - } - int8_t next = baseIdx + 1; - while (MATRIX[baseIdx][next] != 0 && next <= idx) { - if (MATRIX[baseIdx][next] == -1) { - next++; - continue; - } - if (MATRIX[baseIdx][next] % baseInterval == 0) { - int8_t extra = baseInterval >= MATRIX[baseIdx][idx] ? 0 : 1; - if (!recursiveTsmaCheckRecursive(baseInterval / MATRIX[baseIdx][idx] + extra, next, interval, idx, precision)) { - next++; - continue; - } else { - return true; - } - } else { - return false; - } - } - return false; -} - -/* - * @breif check if tsma with param [interval], [unit] can create based on base tsma with baseInterval and baseUnit - * @param baseInterval, baseUnit, interval/unit of base tsma - * @param interval the tsma interval going to create. Not that if unit is not calander unit, then interval has already been - * translated to TICKS of [precision] - * @param unit the tsma unit going to create - * @precision the precision of this db - * @ret true the tsma can be created, else cannot - * */ -static bool checkRecursiveTsmaInterval(int64_t baseInterval, int8_t baseUnit, int64_t interval, int8_t unit, int8_t precision) { - int8_t baseIdx = UNIT_INDEX[baseUnit], idx = UNIT_INDEX[unit]; - if (baseIdx <= idx) { - return recursiveTsmaCheckRecursive(baseInterval, baseIdx, interval, idx, precision); - } else { - - } - return true; -} - static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStmt, SMCreateSmaReq* pReq, SName* useTbName) { SName name; @@ -11236,7 +11174,8 @@ static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStm pReq->recursiveTsma = true; tNameExtractFullName(useTbName, pReq->baseTsmaName); SValueNode* pInterval = (SValueNode*)pStmt->pOptions->pInterval; - if (pRecursiveTsma->interval < pInterval->datum.i && pInterval->datum.i % pRecursiveTsma->interval == 0) { + if (checkRecursiveTsmaInterval(pRecursiveTsma->interval, pRecursiveTsma->unit, pInterval->datum.i, + pInterval->unit, pDbInfo.precision, true)) { } else { code = TSDB_CODE_TSMA_INVALID_PARA; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 82b60fe580..2e731eb53c 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -6086,12 +6086,19 @@ static void clearTSMAOptCtx(STSMAOptCtx* pTsmaOptCtx) { taosMemoryFreeClear(pTsmaOptCtx->queryInterval); } -static bool tsmaOptCheckValidInterval(int64_t tsmaInterval, const STSMAOptCtx* pTsmaOptCtx) { +static bool tsmaOptCheckValidInterval(int64_t tsmaInterval, int8_t unit, const STSMAOptCtx* pTsmaOptCtx) { if (!pTsmaOptCtx->queryInterval) return true; - bool validInterval = pTsmaOptCtx->queryInterval->interval % tsmaInterval == 0; - bool validSliding = pTsmaOptCtx->queryInterval->sliding % tsmaInterval == 0; - bool validOffset = pTsmaOptCtx->queryInterval->offset % tsmaInterval == 0; + bool validInterval = checkRecursiveTsmaInterval(tsmaInterval, unit, pTsmaOptCtx->queryInterval->interval, + pTsmaOptCtx->queryInterval->intervalUnit, + pTsmaOptCtx->queryInterval->precision, false); + bool validSliding = + checkRecursiveTsmaInterval(tsmaInterval, unit, pTsmaOptCtx->queryInterval->sliding, + pTsmaOptCtx->queryInterval->slidingUnit, pTsmaOptCtx->queryInterval->precision, false); + bool validOffset = + pTsmaOptCtx->queryInterval->offset == 0 || + checkRecursiveTsmaInterval(tsmaInterval, unit, pTsmaOptCtx->queryInterval->offset, + pTsmaOptCtx->queryInterval->offsetUnit, pTsmaOptCtx->queryInterval->precision, false); return validInterval && validSliding && validOffset; } @@ -6191,7 +6198,7 @@ static int32_t tsmaOptFilterTsmas(STSMAOptCtx* pTsmaOptCtx) { continue; } // filter with interval - if (!tsmaOptCheckValidInterval(pTsma->interval, pTsmaOptCtx)) { + if (!tsmaOptCheckValidInterval(pTsma->interval, pTsma->unit, pTsmaOptCtx)) { continue; } // filter with funcs, note that tsma funcs has been sorted by funcId and ColId diff --git a/source/util/test/utilTests.cpp b/source/util/test/utilTests.cpp index 6e32167f88..cb78743018 100644 --- a/source/util/test/utilTests.cpp +++ b/source/util/test/utilTests.cpp @@ -2,6 +2,7 @@ #include #include #include +#include "ttime.h" #include "tarray.h" #include "tcompare.h" @@ -382,3 +383,94 @@ TEST(utilTest, intToHextStr) { ASSERT_STREQ(buf, destBuf); } } +static int64_t getIntervalValWithPrecision(int64_t interval, int8_t unit, int8_t precision) { + if (IS_CALENDAR_TIME_DURATION(unit)) { + return interval; + } + if(0 != getDuration(interval, unit, &interval, precision)) { + assert(0); + } + return interval; +} + +static bool tsmaIntervalCheck(int64_t baseInterval, int8_t baseUnit, int64_t interval, int8_t unit, int8_t precision) { + auto ret = checkRecursiveTsmaInterval(getIntervalValWithPrecision(baseInterval, baseUnit, precision), baseUnit, + getIntervalValWithPrecision(interval, unit, precision), unit, precision, true); + using namespace std; + cout << interval << unit << " on " << baseInterval << baseUnit << ": " << ret << endl; + return ret; +} + +TEST(tsma, reverse_unit) { + ASSERT_TRUE(tsmaIntervalCheck(1, 'm', 120, 's', TSDB_TIME_PRECISION_MILLI)); + ASSERT_TRUE(tsmaIntervalCheck(1, 'h', 120, 'm', TSDB_TIME_PRECISION_MILLI)); + ASSERT_TRUE(tsmaIntervalCheck(20, 's', 2 * 20 * 1000, 'a', TSDB_TIME_PRECISION_MILLI)); + ASSERT_TRUE(tsmaIntervalCheck(20, 's', 2 * 20 * 1000 * 1000, 'u', TSDB_TIME_PRECISION_MILLI)); + ASSERT_TRUE(tsmaIntervalCheck(20, 's', 2UL * 20UL * 1000UL * 1000UL * 1000UL, 'b', TSDB_TIME_PRECISION_MILLI)); + + ASSERT_FALSE(tsmaIntervalCheck(1, 'h', 60, 'm', TSDB_TIME_PRECISION_MILLI)); + ASSERT_FALSE(tsmaIntervalCheck(1, 'h', 6, 'm', TSDB_TIME_PRECISION_MILLI)); + + ASSERT_FALSE(tsmaIntervalCheck(2, 'h', 120, 'm', TSDB_TIME_PRECISION_MILLI)); + ASSERT_TRUE(tsmaIntervalCheck(2, 'h', 240, 'm', TSDB_TIME_PRECISION_MILLI)); + ASSERT_FALSE(tsmaIntervalCheck(1, 'd', 240, 'm', TSDB_TIME_PRECISION_MILLI)); + + ASSERT_FALSE(tsmaIntervalCheck(1, 'd', 1440, 'm', TSDB_TIME_PRECISION_MILLI)); + ASSERT_TRUE(tsmaIntervalCheck(1, 'd', 2880, 'm', TSDB_TIME_PRECISION_MILLI)); + + ASSERT_FALSE(tsmaIntervalCheck(1, 'y', 365, 'd', TSDB_TIME_PRECISION_MILLI)); + ASSERT_FALSE(tsmaIntervalCheck(1, 'n', 30, 'd', TSDB_TIME_PRECISION_MILLI)); + + ASSERT_TRUE(tsmaIntervalCheck(1, 'y', 24, 'n', TSDB_TIME_PRECISION_MILLI)); + + ASSERT_FALSE(tsmaIntervalCheck(55, 's', 55, 'm', TSDB_TIME_PRECISION_MILLI)); + ASSERT_TRUE(tsmaIntervalCheck(10, 's', 1, 'm', TSDB_TIME_PRECISION_MICRO)); + ASSERT_TRUE(tsmaIntervalCheck(10, 's', 2, 'm', TSDB_TIME_PRECISION_MICRO)); + ASSERT_TRUE(tsmaIntervalCheck(10, 's', 20, 'm', TSDB_TIME_PRECISION_MICRO)); + ASSERT_TRUE(tsmaIntervalCheck(10, 's', 50, 'm', TSDB_TIME_PRECISION_MICRO)); + + ASSERT_TRUE(tsmaIntervalCheck(120, 's', 30, 'm', TSDB_TIME_PRECISION_MICRO)); + ASSERT_TRUE(tsmaIntervalCheck(360, 's', 30, 'm', TSDB_TIME_PRECISION_MICRO)); + ASSERT_TRUE(tsmaIntervalCheck(600, 's', 30, 'm', TSDB_TIME_PRECISION_MICRO)); + ASSERT_FALSE(tsmaIntervalCheck(600, 's', 15, 'm', TSDB_TIME_PRECISION_MICRO)); + + ASSERT_TRUE(tsmaIntervalCheck(10, 's', 1, 'h', TSDB_TIME_PRECISION_MICRO)); + ASSERT_TRUE(tsmaIntervalCheck(15, 's', 1, 'h', TSDB_TIME_PRECISION_MICRO)); + ASSERT_FALSE(tsmaIntervalCheck(7*60, 's', 1, 'h', TSDB_TIME_PRECISION_MICRO)); + ASSERT_TRUE(tsmaIntervalCheck(10, 's', 1, 'd', TSDB_TIME_PRECISION_MICRO)); + ASSERT_TRUE(tsmaIntervalCheck(10, 's', 1, 'w', TSDB_TIME_PRECISION_MICRO)); + ASSERT_TRUE(tsmaIntervalCheck(1, 'd', 1, 'w', TSDB_TIME_PRECISION_MICRO)); + ASSERT_TRUE(tsmaIntervalCheck(10, 's', 1, 'n', TSDB_TIME_PRECISION_MICRO)); + ASSERT_TRUE(tsmaIntervalCheck(10, 's', 1, 'y', TSDB_TIME_PRECISION_MICRO)); + + ASSERT_TRUE(tsmaIntervalCheck(1, 'd', 1, 'w', TSDB_TIME_PRECISION_MICRO)); + ASSERT_TRUE(tsmaIntervalCheck(1, 'd', 1, 'n', TSDB_TIME_PRECISION_MICRO)); + ASSERT_TRUE(tsmaIntervalCheck(1, 'd', 2, 'n', TSDB_TIME_PRECISION_MICRO)); + ASSERT_FALSE(tsmaIntervalCheck(2, 'd', 2, 'n', TSDB_TIME_PRECISION_MICRO)); + ASSERT_FALSE(tsmaIntervalCheck(2, 'd', 2, 'y', TSDB_TIME_PRECISION_MICRO)); + ASSERT_FALSE(tsmaIntervalCheck(2, 'd', 1, 'y', TSDB_TIME_PRECISION_MICRO)); + + ASSERT_FALSE(tsmaIntervalCheck(1, 'w', 1, 'n', TSDB_TIME_PRECISION_NANO)); + ASSERT_FALSE(tsmaIntervalCheck(4, 'w', 1, 'n', TSDB_TIME_PRECISION_NANO)); + ASSERT_FALSE(tsmaIntervalCheck(1, 'w', 1, 'y', TSDB_TIME_PRECISION_NANO)); + + ASSERT_TRUE(tsmaIntervalCheck(1, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO)); + ASSERT_TRUE(tsmaIntervalCheck(2, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO)); + ASSERT_TRUE(tsmaIntervalCheck(3, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO)); + ASSERT_TRUE(tsmaIntervalCheck(4, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO)); + ASSERT_FALSE(tsmaIntervalCheck(5, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO)); + ASSERT_TRUE(tsmaIntervalCheck(6, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO)); + ASSERT_FALSE(tsmaIntervalCheck(7, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO)); + ASSERT_FALSE(tsmaIntervalCheck(8, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO)); + ASSERT_FALSE(tsmaIntervalCheck(9, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO)); + ASSERT_FALSE(tsmaIntervalCheck(10, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO)); + ASSERT_FALSE(tsmaIntervalCheck(11, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO)); + + ASSERT_FALSE(tsmaIntervalCheck(1, 'w', 1, 'w', TSDB_TIME_PRECISION_NANO)); + ASSERT_FALSE(tsmaIntervalCheck(120, 's', 2, 'm', TSDB_TIME_PRECISION_NANO)); + + ASSERT_FALSE(tsmaIntervalCheck(2, 'n', 2, 'n', TSDB_TIME_PRECISION_NANO)); + ASSERT_FALSE(tsmaIntervalCheck(2, 'y', 2, 'y', TSDB_TIME_PRECISION_NANO)); + ASSERT_FALSE(tsmaIntervalCheck(12, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO)); + ASSERT_TRUE(tsmaIntervalCheck(3, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO)); +} diff --git a/tests/system-test/2-query/tsma.py b/tests/system-test/2-query/tsma.py index 491a544097..a9d46abc0d 100644 --- a/tests/system-test/2-query/tsma.py +++ b/tests/system-test/2-query/tsma.py @@ -972,16 +972,16 @@ class TDTestCase: sql = 'select avg(c2), "recursive test.tsma4" from test.meters' ctx = TSMAQCBuilder().with_sql(sql).should_query_with_tsma( 'tsma4', UsedTsma.TS_MIN, UsedTsma.TS_MAX).get_qc() - #time.sleep(999999) self.tsma_tester.check_sql(sql, ctx) self.check(self.test_query_tsma_all(select_func_list)) self.create_recursive_tsma( - 'tsma4', 'tsma6', 'test', '1h', 'meters', tsma_func_list) + 'tsma4', 'tsma6', 'test', '5h', 'meters', tsma_func_list) ctx = TSMAQCBuilder().with_sql(sql).should_query_with_tsma( 'tsma6', UsedTsma.TS_MIN, UsedTsma.TS_MAX).get_qc() self.tsma_tester.check_sql(sql, ctx) self.check(self.test_query_tsma_all(select_func_list)) + #time.sleep(999999) tdSql.error('drop tsma test.tsma3', -2147482491) tdSql.error('drop tsma test.tsma4', -2147482491)