xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

support month/year tsma interval

This commit is contained in:
wangjiaming0909 2024-07-08 18:22:33 +08:00
parent dbc2e9ec75
commit 73d4fa48f9
6 changed files with 204 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/common/ttime.h
View File

@ -127,6 +127,9 @@ int32_t TEST_char2ts(const char* format, int64_t* ts, int32_t precision, const c
/// @return 0 success, other fail
int32_t offsetOfTimezone(char* tzStr, int64_t* offset);
bool checkRecursiveTsmaInterval(int64_t baseInterval, int8_t baseUnit, int64_t interval, int8_t unit, int8_t precision,
bool checkEq);
#ifdef __cplusplus
}
#endif

93
source/common/src/ttime.c
View File

@ -1969,3 +1969,96 @@ int32_t TEST_char2ts(const char* format, int64_t* ts, int32_t precision, const c
taosArrayDestroy(formats);
return code;
}
static int8_t UNIT_INDEX[26] = {/*a*/ 2, 0, -1, 6, -1, -1, -1,
/*h*/ 5, -1, -1, -1, -1, 4, 8,
/*o*/ -1, -1, -1, -1, 3, -1,
/*u*/ 1, -1, 7, -1, 9, -1};
#define GET_UNIT_INDEX(idx) UNIT_INDEX[(idx) - 97]
static int64_t MATRIX[10][11] = { /* ns, us, ms, s, min, h, d, w, month, y*/
/*ns*/ { 1, 1000, 0},
/*us*/ {1000, 1, 1000, 0},
/*ms*/ { 0, 1000, 1, 1000, 0},
/*s*/ { 0, 0, 1000, 1, 60, 0},
/*min*/ { 0, 0, 0, 60, 1, 60, 0},
/*h*/ { 0, 0, 0, 0, 60, 1, 24, 0},
/*d*/ { 0, 0, 0, 0, 0, 24, 1, 7, 1, 0},
/*w*/ { 0, 0, 0, 0, 0, 0, 7, 1, -1, 0},
/*mon*/ { 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0},
/*y*/ { 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0}};
static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx, bool checkEq) {
if (MATRIX[baseIdx][idx] == -1) return false;
if (baseIdx == idx) {
if (interval < baseInterval) return false;
if (checkEq && interval == baseInterval) return false;
return interval % baseInterval == 0;
}
int8_t next = baseIdx + 1;
int64_t val = MATRIX[baseIdx][next];
while (val != 0 && next <= idx) {
if (val == -1) {
next++;
val = MATRIX[baseIdx][next];
continue;
}
if (val % baseInterval == 0 || baseInterval % val == 0) {
int8_t extra = baseInterval >= val ? 0 : 1;
if (!recursiveTsmaCheckRecursive(baseInterval / val + extra, next, interval, idx, (extra == 0) && checkEq)) {
next++;
val = MATRIX[baseIdx][next];
continue;
} else {
return true;
}
} else {
return false;
}
}
return false;
}
static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx, bool checkEq) {
if (MATRIX[baseIdx][idx] == -1) return false;
if (baseIdx == idx) {
if (interval < baseInterval) return false;
if (checkEq && interval == baseInterval) return false;
return interval % baseInterval == 0;
}
int8_t next = baseIdx - 1;
int64_t val = MATRIX[baseIdx][next];
while (val != 0 && next >= 0) {
return recursiveTsmaCheckRecursiveReverse(baseInterval * val, next, interval, idx, checkEq);
}
return false;
}
/*
* @breif check if tsma with param [interval], [unit] can create based on base tsma with baseInterval and baseUnit
* @param baseInterval, baseUnit, interval/unit of base tsma
* @param interval the tsma interval going to create. Not that if unit is not calander unit, then interval has already been
* translated to TICKS of [precision]
* @param unit the tsma unit going to create
* @precision the precision of this db
* @ret true the tsma can be created, else cannot
* */
bool checkRecursiveTsmaInterval(int64_t baseInterval, int8_t baseUnit, int64_t interval, int8_t unit, int8_t precision, bool checkEq) {
bool baseIsCalendarDuration = IS_CALENDAR_TIME_DURATION(baseUnit);
if (!baseIsCalendarDuration) baseInterval = convertTimeFromPrecisionToUnit(baseInterval, precision, baseUnit);
bool isCalendarDuration = IS_CALENDAR_TIME_DURATION(unit);
if (!isCalendarDuration) interval = convertTimeFromPrecisionToUnit(interval, precision, unit);
bool needCheckEq = baseIsCalendarDuration == isCalendarDuration && checkEq;
int8_t baseIdx = GET_UNIT_INDEX(baseUnit), idx = GET_UNIT_INDEX(unit);
if (baseIdx <= idx) {
return recursiveTsmaCheckRecursive(baseInterval, baseIdx, interval, idx, needCheckEq);
} else {
return recursiveTsmaCheckRecursiveReverse(baseInterval, baseIdx, interval, idx, true);
}
return true;
}

65
source/libs/parser/src/parTranslater.c
View File

@ -11125,68 +11125,6 @@ static int32_t rewriteTSMAFuncs(STranslateContext* pCxt, SCreateTSMAStmt* pStmt,
return code;
}
static int8_t UNIT_INDEX[26] = {/*a*/ 2, 0, -1, 6, -1, -1, -1,
/*h*/ 5, -1, -1, -1, -1, 4, 8,
/*o*/ -1, -1, -1, -1, 3, -1,
/*u*/ 1, -1, 7, -1, 9, -1};
static int64_t MATRIX[10][11] = { /* ns, us, ms, s, min, h, d, w, month, y*/
/*ns*/ { 1, 1000, 0},
/*us*/ {1000, 1, 1000, 0},
/*ms*/ { -1, 1000, 1, 1000, 0},
/*s*/ { -1, -1, 1000, 1, 60, 0},
/*min*/ { -1, -1, -1, 60, 1, 60, 0},
/*h*/ { -1, -1, -1, -1, 60, 1, 24, 0},
/*d*/ { -1, -1, -1, -1, -1, 24, 1, 7, 1, 0},
/*w*/ { -1, -1, -1, -1, -1, -1, 7, 1, -1, 0},
/*mon*/ { -1, -1, -1, -1, -1, -1, -1, -1, 1, 12, 0},
/*y*/ { -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 0}};
static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx, int8_t precision) {
if (MATRIX[baseIdx][idx] == -1) return false;
if (baseIdx == idx) {
if (interval < baseInterval) return false;
return interval % baseInterval == 0;
}
int8_t next = baseIdx + 1;
while (MATRIX[baseIdx][next] != 0 && next <= idx) {
if (MATRIX[baseIdx][next] == -1) {
next++;
continue;
}
if (MATRIX[baseIdx][next] % baseInterval == 0) {
int8_t extra = baseInterval >= MATRIX[baseIdx][idx] ? 0 : 1;
if (!recursiveTsmaCheckRecursive(baseInterval / MATRIX[baseIdx][idx] + extra, next, interval, idx, precision)) {
next++;
continue;
} else {
return true;
}
} else {
return false;
}
}
return false;
}
/*
* @breif check if tsma with param [interval], [unit] can create based on base tsma with baseInterval and baseUnit
* @param baseInterval, baseUnit, interval/unit of base tsma
* @param interval the tsma interval going to create. Not that if unit is not calander unit, then interval has already been
* translated to TICKS of [precision]
* @param unit the tsma unit going to create
* @precision the precision of this db
* @ret true the tsma can be created, else cannot
* */
static bool checkRecursiveTsmaInterval(int64_t baseInterval, int8_t baseUnit, int64_t interval, int8_t unit, int8_t precision) {
int8_t baseIdx = UNIT_INDEX[baseUnit], idx = UNIT_INDEX[unit];
if (baseIdx <= idx) {
return recursiveTsmaCheckRecursive(baseInterval, baseIdx, interval, idx, precision);
} else {
}
return true;
}
static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStmt, SMCreateSmaReq* pReq,
SName* useTbName) {
SName name;
@ -11236,7 +11174,8 @@ static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStm
pReq->recursiveTsma = true;
tNameExtractFullName(useTbName, pReq->baseTsmaName);
SValueNode* pInterval = (SValueNode*)pStmt->pOptions->pInterval;
if (pRecursiveTsma->interval < pInterval->datum.i && pInterval->datum.i % pRecursiveTsma->interval == 0) {
if (checkRecursiveTsmaInterval(pRecursiveTsma->interval, pRecursiveTsma->unit, pInterval->datum.i,
pInterval->unit, pDbInfo.precision, true)) {
} else {
code = TSDB_CODE_TSMA_INVALID_PARA;
}

17
source/libs/planner/src/planOptimizer.c
View File

@ -6086,12 +6086,19 @@ static void clearTSMAOptCtx(STSMAOptCtx* pTsmaOptCtx) {
taosMemoryFreeClear(pTsmaOptCtx->queryInterval);
}
static bool tsmaOptCheckValidInterval(int64_t tsmaInterval, const STSMAOptCtx* pTsmaOptCtx) {
static bool tsmaOptCheckValidInterval(int64_t tsmaInterval, int8_t unit, const STSMAOptCtx* pTsmaOptCtx) {
if (!pTsmaOptCtx->queryInterval) return true;
bool validInterval = pTsmaOptCtx->queryInterval->interval % tsmaInterval == 0;
bool validSliding = pTsmaOptCtx->queryInterval->sliding % tsmaInterval == 0;
bool validOffset = pTsmaOptCtx->queryInterval->offset % tsmaInterval == 0;
bool validInterval = checkRecursiveTsmaInterval(tsmaInterval, unit, pTsmaOptCtx->queryInterval->interval,
pTsmaOptCtx->queryInterval->intervalUnit,
pTsmaOptCtx->queryInterval->precision, false);
bool validSliding =
checkRecursiveTsmaInterval(tsmaInterval, unit, pTsmaOptCtx->queryInterval->sliding,
pTsmaOptCtx->queryInterval->slidingUnit, pTsmaOptCtx->queryInterval->precision, false);
bool validOffset =
pTsmaOptCtx->queryInterval->offset == 0 ||
checkRecursiveTsmaInterval(tsmaInterval, unit, pTsmaOptCtx->queryInterval->offset,
pTsmaOptCtx->queryInterval->offsetUnit, pTsmaOptCtx->queryInterval->precision, false);
return validInterval && validSliding && validOffset;
}
@ -6191,7 +6198,7 @@ static int32_t tsmaOptFilterTsmas(STSMAOptCtx* pTsmaOptCtx) {
continue;
}
// filter with interval
if (!tsmaOptCheckValidInterval(pTsma->interval, pTsmaOptCtx)) {
if (!tsmaOptCheckValidInterval(pTsma->interval, pTsma->unit, pTsmaOptCtx)) {
continue;
}
// filter with funcs, note that tsma funcs has been sorted by funcId and ColId

92
source/util/test/utilTests.cpp
View File

@ -2,6 +2,7 @@
#include <stdlib.h>
#include <tutil.h>
#include <random>
#include "ttime.h"
#include "tarray.h"
#include "tcompare.h"
@ -382,3 +383,94 @@ TEST(utilTest, intToHextStr) {
ASSERT_STREQ(buf, destBuf);
}
}
static int64_t getIntervalValWithPrecision(int64_t interval, int8_t unit, int8_t precision) {
if (IS_CALENDAR_TIME_DURATION(unit)) {
return interval;
}
if(0 != getDuration(interval, unit, &interval, precision)) {
assert(0);
}
return interval;
}
static bool tsmaIntervalCheck(int64_t baseInterval, int8_t baseUnit, int64_t interval, int8_t unit, int8_t precision) {
auto ret = checkRecursiveTsmaInterval(getIntervalValWithPrecision(baseInterval, baseUnit, precision), baseUnit,
getIntervalValWithPrecision(interval, unit, precision), unit, precision, true);
using namespace std;
cout << interval << unit << " on " << baseInterval << baseUnit << ": " << ret << endl;
return ret;
}
TEST(tsma, reverse_unit) {
ASSERT_TRUE(tsmaIntervalCheck(1, 'm', 120, 's', TSDB_TIME_PRECISION_MILLI));
ASSERT_TRUE(tsmaIntervalCheck(1, 'h', 120, 'm', TSDB_TIME_PRECISION_MILLI));
ASSERT_TRUE(tsmaIntervalCheck(20, 's', 2 * 20 * 1000, 'a', TSDB_TIME_PRECISION_MILLI));
ASSERT_TRUE(tsmaIntervalCheck(20, 's', 2 * 20 * 1000 * 1000, 'u', TSDB_TIME_PRECISION_MILLI));
ASSERT_TRUE(tsmaIntervalCheck(20, 's', 2UL * 20UL * 1000UL * 1000UL * 1000UL, 'b', TSDB_TIME_PRECISION_MILLI));
ASSERT_FALSE(tsmaIntervalCheck(1, 'h', 60, 'm', TSDB_TIME_PRECISION_MILLI));
ASSERT_FALSE(tsmaIntervalCheck(1, 'h', 6, 'm', TSDB_TIME_PRECISION_MILLI));
ASSERT_FALSE(tsmaIntervalCheck(2, 'h', 120, 'm', TSDB_TIME_PRECISION_MILLI));
ASSERT_TRUE(tsmaIntervalCheck(2, 'h', 240, 'm', TSDB_TIME_PRECISION_MILLI));
ASSERT_FALSE(tsmaIntervalCheck(1, 'd', 240, 'm', TSDB_TIME_PRECISION_MILLI));
ASSERT_FALSE(tsmaIntervalCheck(1, 'd', 1440, 'm', TSDB_TIME_PRECISION_MILLI));
ASSERT_TRUE(tsmaIntervalCheck(1, 'd', 2880, 'm', TSDB_TIME_PRECISION_MILLI));
ASSERT_FALSE(tsmaIntervalCheck(1, 'y', 365, 'd', TSDB_TIME_PRECISION_MILLI));
ASSERT_FALSE(tsmaIntervalCheck(1, 'n', 30, 'd', TSDB_TIME_PRECISION_MILLI));
ASSERT_TRUE(tsmaIntervalCheck(1, 'y', 24, 'n', TSDB_TIME_PRECISION_MILLI));
ASSERT_FALSE(tsmaIntervalCheck(55, 's', 55, 'm', TSDB_TIME_PRECISION_MILLI));
ASSERT_TRUE(tsmaIntervalCheck(10, 's', 1, 'm', TSDB_TIME_PRECISION_MICRO));
ASSERT_TRUE(tsmaIntervalCheck(10, 's', 2, 'm', TSDB_TIME_PRECISION_MICRO));
ASSERT_TRUE(tsmaIntervalCheck(10, 's', 20, 'm', TSDB_TIME_PRECISION_MICRO));
ASSERT_TRUE(tsmaIntervalCheck(10, 's', 50, 'm', TSDB_TIME_PRECISION_MICRO));
ASSERT_TRUE(tsmaIntervalCheck(120, 's', 30, 'm', TSDB_TIME_PRECISION_MICRO));
ASSERT_TRUE(tsmaIntervalCheck(360, 's', 30, 'm', TSDB_TIME_PRECISION_MICRO));
ASSERT_TRUE(tsmaIntervalCheck(600, 's', 30, 'm', TSDB_TIME_PRECISION_MICRO));
ASSERT_FALSE(tsmaIntervalCheck(600, 's', 15, 'm', TSDB_TIME_PRECISION_MICRO));
ASSERT_TRUE(tsmaIntervalCheck(10, 's', 1, 'h', TSDB_TIME_PRECISION_MICRO));
ASSERT_TRUE(tsmaIntervalCheck(15, 's', 1, 'h', TSDB_TIME_PRECISION_MICRO));
ASSERT_FALSE(tsmaIntervalCheck(7*60, 's', 1, 'h', TSDB_TIME_PRECISION_MICRO));
ASSERT_TRUE(tsmaIntervalCheck(10, 's', 1, 'd', TSDB_TIME_PRECISION_MICRO));
ASSERT_TRUE(tsmaIntervalCheck(10, 's', 1, 'w', TSDB_TIME_PRECISION_MICRO));
ASSERT_TRUE(tsmaIntervalCheck(1, 'd', 1, 'w', TSDB_TIME_PRECISION_MICRO));
ASSERT_TRUE(tsmaIntervalCheck(10, 's', 1, 'n', TSDB_TIME_PRECISION_MICRO));
ASSERT_TRUE(tsmaIntervalCheck(10, 's', 1, 'y', TSDB_TIME_PRECISION_MICRO));
ASSERT_TRUE(tsmaIntervalCheck(1, 'd', 1, 'w', TSDB_TIME_PRECISION_MICRO));
ASSERT_TRUE(tsmaIntervalCheck(1, 'd', 1, 'n', TSDB_TIME_PRECISION_MICRO));
ASSERT_TRUE(tsmaIntervalCheck(1, 'd', 2, 'n', TSDB_TIME_PRECISION_MICRO));
ASSERT_FALSE(tsmaIntervalCheck(2, 'd', 2, 'n', TSDB_TIME_PRECISION_MICRO));
ASSERT_FALSE(tsmaIntervalCheck(2, 'd', 2, 'y', TSDB_TIME_PRECISION_MICRO));
ASSERT_FALSE(tsmaIntervalCheck(2, 'd', 1, 'y', TSDB_TIME_PRECISION_MICRO));
ASSERT_FALSE(tsmaIntervalCheck(1, 'w', 1, 'n', TSDB_TIME_PRECISION_NANO));
ASSERT_FALSE(tsmaIntervalCheck(4, 'w', 1, 'n', TSDB_TIME_PRECISION_NANO));
ASSERT_FALSE(tsmaIntervalCheck(1, 'w', 1, 'y', TSDB_TIME_PRECISION_NANO));
ASSERT_TRUE(tsmaIntervalCheck(1, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
ASSERT_TRUE(tsmaIntervalCheck(2, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
ASSERT_TRUE(tsmaIntervalCheck(3, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
ASSERT_TRUE(tsmaIntervalCheck(4, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
ASSERT_FALSE(tsmaIntervalCheck(5, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
ASSERT_TRUE(tsmaIntervalCheck(6, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
ASSERT_FALSE(tsmaIntervalCheck(7, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
ASSERT_FALSE(tsmaIntervalCheck(8, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
ASSERT_FALSE(tsmaIntervalCheck(9, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
ASSERT_FALSE(tsmaIntervalCheck(10, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
ASSERT_FALSE(tsmaIntervalCheck(11, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
ASSERT_FALSE(tsmaIntervalCheck(1, 'w', 1, 'w', TSDB_TIME_PRECISION_NANO));
ASSERT_FALSE(tsmaIntervalCheck(120, 's', 2, 'm', TSDB_TIME_PRECISION_NANO));
ASSERT_FALSE(tsmaIntervalCheck(2, 'n', 2, 'n', TSDB_TIME_PRECISION_NANO));
ASSERT_FALSE(tsmaIntervalCheck(2, 'y', 2, 'y', TSDB_TIME_PRECISION_NANO));
ASSERT_FALSE(tsmaIntervalCheck(12, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
ASSERT_TRUE(tsmaIntervalCheck(3, 'n', 1, 'y', TSDB_TIME_PRECISION_NANO));
}

4
tests/system-test/2-query/tsma.py
View File

@ -972,16 +972,16 @@ class TDTestCase:
sql = 'select avg(c2), "recursive test.tsma4" from test.meters'
ctx = TSMAQCBuilder().with_sql(sql).should_query_with_tsma(
'tsma4', UsedTsma.TS_MIN, UsedTsma.TS_MAX).get_qc()
#time.sleep(999999)
self.tsma_tester.check_sql(sql, ctx)
self.check(self.test_query_tsma_all(select_func_list))
self.create_recursive_tsma(
'tsma4', 'tsma6', 'test', '1h', 'meters', tsma_func_list)
'tsma4', 'tsma6', 'test', '5h', 'meters', tsma_func_list)
ctx = TSMAQCBuilder().with_sql(sql).should_query_with_tsma(
'tsma6', UsedTsma.TS_MIN, UsedTsma.TS_MAX).get_qc()
self.tsma_tester.check_sql(sql, ctx)
self.check(self.test_query_tsma_all(select_func_list))
#time.sleep(999999)
tdSql.error('drop tsma test.tsma3', -2147482491)
tdSql.error('drop tsma test.tsma4', -2147482491)