From 882cfb7deb545cf937a1f9ed7f843caab0832461 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Mon, 12 Aug 2024 08:53:27 +0800 Subject: [PATCH 1/2] 1. lock not released when return 2. fix tsma err msg 3. fix fill prev wrong results --- include/util/taoserror.h | 1 + source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 3 ++- source/libs/executor/src/tfill.c | 5 +++-- source/libs/parser/src/parTranslater.c | 2 +- source/util/src/terror.c | 1 + tests/system-test/2-query/tsma.py | 8 ++++---- tests/system-test/2-query/tsma2.py | 2 +- 7 files changed, 13 insertions(+), 9 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index b091d870ec..b8098cc96e 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -914,6 +914,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_TSMA_UNSUPPORTED_FUNC TAOS_DEF_ERROR_CODE(0, 0x3109) #define TSDB_CODE_TSMA_MUST_BE_DROPPED TAOS_DEF_ERROR_CODE(0, 0x3110) #define TSDB_CODE_TSMA_NAME_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x3111) +#define TSDB_CODE_TSMA_INVALID_RECURSIVE_INTERVAL TAOS_DEF_ERROR_CODE(0, 0x3112) //rsma #define TSDB_CODE_RSMA_INVALID_ENV TAOS_DEF_ERROR_CODE(0, 0x3150) diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 4dabffc10a..1fba59c9b2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -957,6 +957,7 @@ int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piM if (p->version <= ver) { void* px = taosArrayPush(pMemDelData, p); if (px == NULL) { + taosRUnLockLatch(&pMemTbData->lock); return terrno; } } @@ -1430,4 +1431,4 @@ bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const return doCheckDatablockOverlapWithoutVersion(pBlockScanInfo, pRecord, index); } -} \ No newline at end of file +} diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 3158c85987..e6c2ea1b45 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -331,8 +331,9 @@ static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SRowVa for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { int32_t type = pFillInfo->pFillCol[i].pExpr->pExpr->nodeType; if (type == QUERY_NODE_COLUMN || type == QUERY_NODE_OPERATOR || type == QUERY_NODE_FUNCTION) { - if (!pFillInfo->pFillCol[i].notFillCol && pFillInfo->type != TSDB_FILL_NEXT) { - continue; + if (!pFillInfo->pFillCol[i].notFillCol) { + if (FILL_IS_ASC_FILL(pFillInfo) && pFillInfo->type != TSDB_FILL_NEXT) continue; + if (!FILL_IS_ASC_FILL(pFillInfo) && pFillInfo->type != TSDB_FILL_PREV) continue; } int32_t srcSlotId = GET_DEST_SLOT_ID(&pFillInfo->pFillCol[i]); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 03c3355920..c86e2d4874 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -11864,7 +11864,7 @@ static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStm if (checkRecursiveTsmaInterval(pRecursiveTsma->interval, pRecursiveTsma->unit, pInterval->datum.i, pInterval->unit, pDbInfo.precision, true)) { } else { - code = TSDB_CODE_TSMA_INVALID_INTERVAL; + code = TSDB_CODE_TSMA_INVALID_RECURSIVE_INTERVAL; } } } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 7b9d48eb58..2f4414cd03 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -759,6 +759,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_INVALID_FUNC_PARAM, "Invalid tsma func p TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_UNSUPPORTED_FUNC, "Tsma func not supported") TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_MUST_BE_DROPPED, "Tsma must be dropped first") TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_NAME_TOO_LONG, "Tsma name too long") +TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_INVALID_RECURSIVE_INTERVAL,"Invalid recursive tsma interval") //rsma TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_ENV, "Invalid rsma env") diff --git a/tests/system-test/2-query/tsma.py b/tests/system-test/2-query/tsma.py index 2bf908e250..1b52302503 100644 --- a/tests/system-test/2-query/tsma.py +++ b/tests/system-test/2-query/tsma.py @@ -1476,18 +1476,18 @@ class TDTestCase: tdSql.error(sql, -2147473920) # syntax error sql = 'create recursive tsma tsma2 on test.tsma1 interval(1m)' - tdSql.error(sql, -2147471097) # invalid tsma interval + tdSql.error(sql, -2147471086) # invalid tsma interval sql = 'create recursive tsma tsma2 on test.tsma1 interval(7m)' - tdSql.error(sql, -2147471097) # invalid tsma interval + tdSql.error(sql, -2147471086) # invalid tsma interval sql = 'create recursive tsma tsma2 on test.tsma1 interval(11m)' - tdSql.error(sql, -2147471097) # invalid tsma interval + tdSql.error(sql, -2147471086) # invalid tsma interval self.create_recursive_tsma('tsma1', 'tsma2', 'test', '20m', 'meters') sql = 'create recursive tsma tsma3 on test.tsma2 interval(30m)' - tdSql.error(sql, -2147471097) # invalid tsma interval + tdSql.error(sql, -2147471086) # invalid tsma interval self.create_recursive_tsma('tsma2', 'tsma3', 'test', '40m', 'meters') diff --git a/tests/system-test/2-query/tsma2.py b/tests/system-test/2-query/tsma2.py index 404fb3f00c..f53b578012 100644 --- a/tests/system-test/2-query/tsma2.py +++ b/tests/system-test/2-query/tsma2.py @@ -852,7 +852,7 @@ class TDTestCase: ] tdSql.execute('use db') for (i, ri, ret) in examples: - self.test_create_recursive_tsma_interval(db, tb, func, i, ri, ret, -2147471097) + self.test_create_recursive_tsma_interval(db, tb, func, i, ri, ret, -2147471086) self.create_tsma('tsma1', db, tb, func, '1h') self.create_recursive_tsma('tsma1', 'tsma2', db, '1n', tb, func) From 19952995af675694dec8ef5a2175ba0c20e2f566 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Mon, 12 Aug 2024 15:27:15 +0800 Subject: [PATCH 2/2] fix fill order by returned wrong values --- source/libs/executor/src/filloperator.c | 2 + source/libs/executor/src/tfill.c | 13 +++++-- tests/system-test/2-query/fill.py | 11 ++++++ tests/system-test/2-query/fill_with_group.py | 41 +++++++++++++++++++- 4 files changed, 62 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index f61b514626..e39667531c 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -254,6 +254,8 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { (pInfo->pFillInfo->type != TSDB_FILL_NULL_F && pInfo->pFillInfo->type != TSDB_FILL_SET_VALUE_F)) { setOperatorCompleted(pOperator); return NULL; + } else if (pInfo->totalInputRows == 0 && taosFillNotStarted(pInfo->pFillInfo)) { + reviseFillStartAndEndKey(pInfo, order); } taosFillSetStartInfo(pInfo->pFillInfo, 0, pInfo->win.ekey); diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index e6c2ea1b45..b8ee5bf047 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -207,7 +207,7 @@ static void doFillOneRow(SFillInfo* pFillInfo, SSDataBlock* pBlock, SSDataBlock* setNotFillColumn(pFillInfo, pDstCol, index, i); } } else { - SRowVal* pRVal = FILL_IS_ASC_FILL(pFillInfo) ? &pFillInfo->prev : &pFillInfo->next; + SRowVal* pRVal = &pFillInfo->prev; SGroupKeys* pKey = taosArrayGet(pRVal->pRowVal, i); if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pKey->isNull) { colDataSetNULL(pDstCol, index); @@ -395,8 +395,8 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t ASSERT(pFillInfo->currentKey == ts); int32_t index = pBlock->info.rows; + int32_t nextRowIndex = pFillInfo->index + 1; if (pFillInfo->type == TSDB_FILL_NEXT) { - int32_t nextRowIndex = pFillInfo->index + 1; if ((pFillInfo->index + 1) < pFillInfo->numOfRows) { copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, &pFillInfo->next, false); } else { @@ -404,6 +404,11 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, &pFillInfo->next, true); } } + if (pFillInfo->type == TSDB_FILL_PREV) { + if (nextRowIndex + 1 >= pFillInfo->numOfRows && !FILL_IS_ASC_FILL(pFillInfo)) { + copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, &pFillInfo->next, true); + } + } // copy rows to dst buffer for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { @@ -418,7 +423,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t if (!colDataIsNull_s(pSrc, pFillInfo->index)) { code = colDataSetVal(pDst, index, src, false); QUERY_CHECK_CODE(code, lino, _end); - SRowVal* pRVal = FILL_IS_ASC_FILL(pFillInfo) ? &pFillInfo->prev : &pFillInfo->next; + SRowVal* pRVal = &pFillInfo->prev; saveColData(pRVal->pRowVal, i, src, false); if (pFillInfo->srcTsSlotId == dstSlotId) { pRVal->key = *(int64_t*)src; @@ -439,7 +444,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t code = colDataSetVal(pDst, index, src, isNull); QUERY_CHECK_CODE(code, lino, _end); - SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev.pRowVal : pFillInfo->next.pRowVal; + SArray* p = pFillInfo->prev.pRowVal; saveColData(p, i, src, isNull); // todo: } else if (pFillInfo->type == TSDB_FILL_NULL || pFillInfo->type == TSDB_FILL_NULL_F) { colDataSetNULL(pDst, index); diff --git a/tests/system-test/2-query/fill.py b/tests/system-test/2-query/fill.py index 64a43bd80a..31d2d92563 100644 --- a/tests/system-test/2-query/fill.py +++ b/tests/system-test/2-query/fill.py @@ -126,6 +126,17 @@ class TDTestCase: def test_fill_range(self): os.system('taosBenchmark -t 10 -n 10000 -v 8 -S 32000 -y') self.schedule_fill_test_tasks() + sql = "select first(_wstart), last(_wstart) from (select _wstart, count(*) from test.meters where ts >= '2019-09-19 23:54:00.000' and ts < '2019-09-20 01:00:00.000' interval(10s) sliding(10s) fill(VALUE_F, 122) order by _wstart) t" + tdSql.query(sql, queryTimes=1) + rows = tdSql.getRows() + first_ts = tdSql.queryResult[0][0] + last_ts = tdSql.queryResult[0][1] + + sql = "select first(_wstart), last(_wstart) from (select _wstart, count(*) from test.meters where ts >= '2019-09-19 23:54:00.000' and ts < '2019-09-20 01:00:00.000' interval(10s) sliding(10s) fill(VALUE_F, 122) order by _wstart desc) t" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(rows) + tdSql.checkData(0, 0, first_ts) + tdSql.checkData(0, 1, last_ts) tdSql.execute('drop database test') def run(self): diff --git a/tests/system-test/2-query/fill_with_group.py b/tests/system-test/2-query/fill_with_group.py index 47e70d3c0f..2139bbbfb3 100644 --- a/tests/system-test/2-query/fill_with_group.py +++ b/tests/system-test/2-query/fill_with_group.py @@ -165,7 +165,7 @@ class TDTestCase: sql = "select _wstart, count(*) from meters where ts >= '2018-09-20 00:00:00.000' and ts < '2018-09-20 01:00:00.000' interval(5m) fill(prev) order by _wstart desc;" tdSql.query(sql, queryTimes=1) tdSql.checkRows(12) - tdSql.checkData(0, 1, None) + tdSql.checkData(0, 1, 10) tdSql.checkData(1, 1, 10) tdSql.checkData(2, 1, 10) tdSql.checkData(3, 1, 10) @@ -198,6 +198,45 @@ class TDTestCase: tdSql.query(sql, queryTimes=1) tdSql.checkRows(60) + sql = "select _wstart, count(*) from meters where ts >= '2018-09-19 23:54:00.000' and ts < '2018-09-20 01:00:00.000' interval(5m) fill(next) order by _wstart asc;" + tdSql.query(sql, queryTimes=1) + for i in range(0, 13): + tdSql.checkData(i, 1, 10) + tdSql.checkData(13, 1, None) + sql = "select _wstart, count(*) from meters where ts >= '2018-09-19 23:54:00.000' and ts < '2018-09-20 01:00:00.000' interval(5m) fill(next) order by _wstart desc;" + tdSql.query(sql, queryTimes=1) + tdSql.checkData(0, 1, None) + for i in range(1, 14): + tdSql.checkData(i, 1, 10) + + sql = "select _wstart, count(*) from meters where ts >= '2018-09-19 23:54:00.000' and ts < '2018-09-20 01:00:00.000' interval(5m) fill(prev) order by _wstart asc;" + tdSql.query(sql, queryTimes=1) + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + for i in range(2, 14): + tdSql.checkData(i, 1, 10) + sql = "select _wstart, count(*) from meters where ts >= '2018-09-19 23:54:00.000' and ts < '2018-09-20 01:00:00.000' interval(5m) fill(prev) order by _wstart desc;" + tdSql.query(sql, queryTimes=1) + for i in range(0, 12): + tdSql.checkData(i, 1, 10) + tdSql.checkData(12, 1, None) + tdSql.checkData(13, 1, None) + + sql = "select _wstart, count(*) from meters where ts >= '2018-09-19 23:54:00.000' and ts < '2018-09-20 01:00:00.000' interval(5m) fill(linear) order by _wstart asc;" + tdSql.query(sql, queryTimes=1) + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + for i in range(2, 13): + tdSql.checkData(i, 1, 10) + tdSql.checkData(13, 1, None) + sql = "select _wstart, count(*) from meters where ts >= '2018-09-19 23:54:00.000' and ts < '2018-09-20 01:00:00.000' interval(5m) fill(linear) order by _wstart desc;" + tdSql.query(sql, queryTimes=1) + tdSql.checkData(0, 1, None) + for i in range(1, 12): + tdSql.checkData(i, 1, 10) + tdSql.checkData(12, 1, None) + tdSql.checkData(13, 1, None) + def run(self): self.prepareTestEnv() self.test_partition_by_with_interval_fill_prev_new_group_fill_error()