From a7d21c9ee73c24aa7da8430ebe10008963d75afb Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 8 Nov 2022 14:50:03 +0800 Subject: [PATCH 01/95] enh: interp fill linear ignore null values during calculation --- source/libs/executor/inc/executorimpl.h | 1 - source/libs/executor/src/timewindowoperator.c | 166 ++++-------------- 2 files changed, 33 insertions(+), 134 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index dedf1630af..03907a0d03 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -782,7 +782,6 @@ typedef struct STimeSliceOperatorInfo { SArray* pPrevRow; // SArray SArray* pNextRow; // SArray SArray* pLinearInfo; // SArray - bool fillLastPoint; bool isPrevRowSet; bool isNextRowSet; int32_t fillType; // fill type diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 88f3e4cff9..ef99e31f67 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1991,10 +1991,8 @@ static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock pSliceInfo->isNextRowSet = true; } -static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex, - bool isLastRow) { +static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) { int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - bool fillLastPoint = pSliceInfo->fillLastPoint; for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId); @@ -2002,30 +2000,21 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo // null data should not be kept since it can not be used to perform interpolation if (!colDataIsNull_s(pColInfoData, i)) { - if (isLastRow) { + if (pLinearInfo->start.key == INT64_MIN) { pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); memcpy(pLinearInfo->start.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); - } else if (fillLastPoint) { + } else if (pLinearInfo->end.key == INT64_MAX) { pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); } else { - pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); - pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex + 1); - - char* val; - val = colDataGetData(pColInfoData, rowIndex); - memcpy(pLinearInfo->start.val, val, pLinearInfo->bytes); - val = colDataGetData(pColInfoData, rowIndex + 1); - memcpy(pLinearInfo->end.val, val, pLinearInfo->bytes); + pLinearInfo->start.key = pLinearInfo->end.key; + pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); + memcpy(pLinearInfo->start.val, pLinearInfo->end.val, pLinearInfo->bytes); + memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); } - - pLinearInfo->hasNull = false; - } else { - pLinearInfo->hasNull = true; } } - pSliceInfo->fillLastPoint = isLastRow; } static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock) { @@ -2209,15 +2198,9 @@ static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB taosArrayPush(pInfo->pLinearInfo, &linearInfo); } - pInfo->fillLastPoint = false; - return TSDB_CODE_SUCCESS; } -static bool needToFillLastPoint(STimeSliceOperatorInfo* pSliceInfo) { - return (pSliceInfo->fillLastPoint == true && pSliceInfo->fillType == TSDB_FILL_LINEAR); -} - static int32_t initKeeperInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) { int32_t code; code = initPrevRowsKeeper(pInfo, pBlock); @@ -2273,15 +2256,6 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { for (int32_t i = 0; i < pBlock->info.rows; ++i) { int64_t ts = *(int64_t*)colDataGetData(pTsCol, i); - if (i == 0 && needToFillLastPoint(pSliceInfo)) { // first row in current block - doKeepLinearInfo(pSliceInfo, pBlock, i, false); - while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); - pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - } - } - if (pSliceInfo->current > pSliceInfo->win.ekey) { doSetOperatorCompleted(pOperator); break; @@ -2313,94 +2287,44 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { pResBlock->info.rows += 1; doKeepPrevRows(pSliceInfo, pBlock, i); + doKeepLinearInfo(pSliceInfo, pBlock, i); - // for linear interpolation, always fill value between this and next points; - // if its the first point in data block, also fill values between previous(if there's any) and this point; - // if its the last point in data block, no need to fill, but reserve this point as the start value and do - // the interpolation when processing next data block. - if (pSliceInfo->fillType == TSDB_FILL_LINEAR) { - pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - if (i < pBlock->info.rows - 1) { - doKeepLinearInfo(pSliceInfo, pBlock, i, false); - int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); - if (nextTs > pSliceInfo->current) { - while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); - pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, - pInterval->precision); - } - - if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); - break; - } - } - } else { // it is the last row of current block - // store ts value as start, and calculate interp value when processing next block - doKeepLinearInfo(pSliceInfo, pBlock, i, true); - } - } else { // non-linear interpolation - pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); - break; - } + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + if (pSliceInfo->current > pSliceInfo->win.ekey) { + doSetOperatorCompleted(pOperator); + break; } } else if (ts < pSliceInfo->current) { // in case of interpolation window starts and ends between two datapoints, fill(prev) need to interpolate doKeepPrevRows(pSliceInfo, pBlock, i); + doKeepLinearInfo(pSliceInfo, pBlock, i); - if (pSliceInfo->fillType == TSDB_FILL_LINEAR) { - // no need to increate pSliceInfo->current here - // pSliceInfo->current = - // taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - if (i < pBlock->info.rows - 1) { - doKeepLinearInfo(pSliceInfo, pBlock, i, false); - int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); - if (nextTs > pSliceInfo->current) { - while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); - pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, - pInterval->precision); - } + if (i < pBlock->info.rows - 1) { + // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate + doKeepNextRows(pSliceInfo, pBlock, i + 1); + int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); + if (nextTs > pSliceInfo->current) { + while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); + pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, + pInterval->precision); + } - if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); - break; - } + if (pSliceInfo->current > pSliceInfo->win.ekey) { + doSetOperatorCompleted(pOperator); + break; } } else { - // store ts value as start, and calculate interp value when processing next block - doKeepLinearInfo(pSliceInfo, pBlock, i, true); - } - } else { // non-linear interpolation - if (i < pBlock->info.rows - 1) { - // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate - doKeepNextRows(pSliceInfo, pBlock, i + 1); - int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); - if (nextTs > pSliceInfo->current) { - while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); - pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, - pInterval->precision); - } - - if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); - break; - } - } else { - // ignore current row, and do nothing - } - } else { // it is the last row of current block - doKeepPrevRows(pSliceInfo, pBlock, i); + // ignore current row, and do nothing } + } else { // it is the last row of current block + doKeepPrevRows(pSliceInfo, pBlock, i); } } else { // ts > pSliceInfo->current // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate doKeepNextRows(pSliceInfo, pBlock, i); + doKeepLinearInfo(pSliceInfo, pBlock, i); while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); @@ -2436,32 +2360,8 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { pResBlock->info.rows += 1; doKeepPrevRows(pSliceInfo, pBlock, i); - if (pSliceInfo->fillType == TSDB_FILL_LINEAR) { - pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - if (i < pBlock->info.rows - 1) { - doKeepLinearInfo(pSliceInfo, pBlock, i, false); - int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); - if (nextTs > pSliceInfo->current) { - while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); - pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, - pInterval->precision); - } - - if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); - break; - } - } - } else { // it is the last row of current block - // store ts value as start, and calculate interp value when processing next block - doKeepLinearInfo(pSliceInfo, pBlock, i, true); - } - } else { // non-linear interpolation - pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - } + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); } if (pSliceInfo->current > pSliceInfo->win.ekey) { From 6aca4b8a5cef4c71b4b2f30680f3e2c08fb2a424 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 8 Nov 2022 14:50:03 +0800 Subject: [PATCH 02/95] enh: interp fill linear ignore null values during calculation --- source/libs/executor/inc/tfill.h | 1 - source/libs/executor/src/timewindowoperator.c | 43 +++++++++++-------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index ed019be767..7e0866f545 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -36,7 +36,6 @@ typedef struct SFillColInfo { typedef struct SFillLinearInfo { SPoint start; SPoint end; - bool hasNull; int16_t type; int32_t bytes; } SFillLinearInfo; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index ef99e31f67..0950fcce2f 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1999,7 +1999,7 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, i); // null data should not be kept since it can not be used to perform interpolation - if (!colDataIsNull_s(pColInfoData, i)) { + if (!colDataIsNull_s(pColInfoData, rowIndex)) { if (pLinearInfo->start.key == INT64_MIN) { pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); memcpy(pLinearInfo->start.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); @@ -2017,7 +2017,8 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo } -static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock) { +static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, + SSDataBlock* pSrcBlock, int32_t index, bool beforeRange) { int32_t rows = pResBlock->info.rows; blockDataEnsureCapacity(pResBlock, rows + 1); // todo set the correct primary timestamp column @@ -2036,7 +2037,6 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; - // SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); switch (pSliceInfo->fillType) { case TSDB_FILL_NULL: { colDataAppendNULL(pDst, rows); @@ -2071,17 +2071,17 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp current.val = taosMemoryCalloc(pLinearInfo->bytes, 1); // before interp range, do not fill - if (start.key == INT64_MIN || end.key == INT64_MAX) { + SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot); + if (colDataIsNull_s(pSrc, index) || (!beforeRange && end.key == INT64_MAX)) { hasInterp = false; break; + } else if (start.key == INT64_MIN || (beforeRange && end.key == INT64_MAX)) { + //hasInterp = true; + return true; } - if (pLinearInfo->hasNull) { - colDataAppendNULL(pDst, rows); - } else { - taosGetLinearInterpolationVal(¤t, pLinearInfo->type, &start, &end, pLinearInfo->type); - colDataAppend(pDst, rows, (char*)current.val, false); - } + taosGetLinearInterpolationVal(¤t, pLinearInfo->type, &start, &end, pLinearInfo->type); + colDataAppend(pDst, rows, (char*)current.val, false); taosMemoryFree(current.val); break; @@ -2117,6 +2117,8 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp if (hasInterp) { pResBlock->info.rows += 1; } + + return hasInterp; } static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) { @@ -2192,7 +2194,6 @@ static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB linearInfo.end.key = INT64_MAX; linearInfo.start.val = taosMemoryCalloc(1, pColInfo->info.bytes); linearInfo.end.val = taosMemoryCalloc(1, pColInfo->info.bytes); - linearInfo.hasNull = false; linearInfo.type = pColInfo->info.type; linearInfo.bytes = pColInfo->info.bytes; taosArrayPush(pInfo->pLinearInfo, &linearInfo); @@ -2306,9 +2307,12 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); - pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, - pInterval->precision); + if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i, false) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { + break; + } else { + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + } } if (pSliceInfo->current > pSliceInfo->win.ekey) { @@ -2327,9 +2331,12 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doKeepLinearInfo(pSliceInfo, pBlock, i); while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); - pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i, true) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { + break; + } else { + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + } } // add current row if timestamp match @@ -2376,7 +2383,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { // except for fill(next), fill(linear) while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT && pSliceInfo->fillType != TSDB_FILL_LINEAR) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, NULL, 0, false); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); } From df3ae1341fe69192915a2ac8c30e3aab5a25c8a4 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 8 Nov 2022 14:50:03 +0800 Subject: [PATCH 03/95] add test cases --- tests/system-test/2-query/interp.py | 51 ++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index ce6b85c1cf..4db074011d 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -311,7 +311,7 @@ class TDTestCase: ## {. . .} tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") - tdSql.checkRows(12) + tdSql.checkRows(11) tdSql.checkData(0, 0, 5) tdSql.checkData(1, 0, 6) tdSql.checkData(2, 0, 7) @@ -354,7 +354,7 @@ class TDTestCase: ## ..{.} tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:13', '2020-02-01 00:00:17') every(1s) fill(linear)") - tdSql.checkRows(5) + tdSql.checkRows(3) tdSql.checkData(0, 0, 13) tdSql.checkData(1, 0, 14) tdSql.checkData(2, 0, 15) @@ -555,7 +555,7 @@ class TDTestCase: tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") - tdSql.checkRows(12) + tdSql.checkRows(11) tdSql.checkCols(2) tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') @@ -583,7 +583,7 @@ class TDTestCase: # multiple _irowts tdSql.query(f"select interp(c0),_irowts from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") - tdSql.checkRows(12) + tdSql.checkRows(11) tdSql.checkCols(2) tdSql.checkData(0, 1, '2020-02-01 00:00:05.000') @@ -599,7 +599,7 @@ class TDTestCase: tdSql.checkData(10, 1, '2020-02-01 00:00:15.000') tdSql.query(f"select _irowts, interp(c0), interp(c0), _irowts from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") - tdSql.checkRows(12) + tdSql.checkRows(11) tdSql.checkCols(4) cols = (0, 3) @@ -837,6 +837,47 @@ class TDTestCase: tdSql.checkData(0, 0, 15) tdSql.checkData(1, 0, 15) + # test fill linear + + ## | {. | | .} | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-11 00:00:05') every(1d) fill(linear)") + tdSql.checkRows(11) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 6) + tdSql.checkData(2, 0, 7) + tdSql.checkData(3, 0, 8) + tdSql.checkData(4, 0, 9) + tdSql.checkData(5, 0, 10) + tdSql.checkData(6, 0, 11) + tdSql.checkData(7, 0, 12) + tdSql.checkData(8, 0, 13) + tdSql.checkData(9, 0, 14) + tdSql.checkData(10, 0, 15) + + ## | . | {} | . | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-03 00:00:05', '2020-02-07 00:00:05') every(1d) fill(linear)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 7) + tdSql.checkData(1, 0, 8) + tdSql.checkData(2, 0, 9) + tdSql.checkData(3, 0, 10) + tdSql.checkData(4, 0, 11) + + ## | {. | } | . | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-01-31 00:00:05', '2020-02-05 00:00:05') every(1d) fill(linear)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 6) + tdSql.checkData(2, 0, 7) + tdSql.checkData(3, 0, 8) + tdSql.checkData(4, 0, 9) + + ## | . | { | .} | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(linear)") + tdSql.checkRows(2) + tdSql.checkData(0, 0, 14) + tdSql.checkData(1, 0, 15) + tdLog.printNoPrefix("==========step10:test multi-interp cases") tdSql.query(f"select interp(c0),interp(c1),interp(c2),interp(c3) from {dbname}.{tbname} range('2020-02-09 00:00:05', '2020-02-13 00:00:05') every(1d) fill(null)") tdSql.checkRows(5) From f19f6201366badc90e3ddbdb070b1331b5dd23fb Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 8 Nov 2022 18:25:38 +0800 Subject: [PATCH 04/95] interp fill linear do not output NULL value --- source/libs/executor/src/timewindowoperator.c | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 0950fcce2f..75d7010eb1 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2018,7 +2018,7 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo } static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, - SSDataBlock* pSrcBlock, int32_t index, bool beforeRange) { + SSDataBlock* pSrcBlock, int32_t index, bool beforeTs) { int32_t rows = pResBlock->info.rows; blockDataEnsureCapacity(pResBlock, rows + 1); // todo set the correct primary timestamp column @@ -2072,10 +2072,10 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp // before interp range, do not fill SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot); - if (colDataIsNull_s(pSrc, index) || (!beforeRange && end.key == INT64_MAX)) { + if (colDataIsNull_s(pSrc, index) || (!beforeTs && end.key == INT64_MAX)) { hasInterp = false; break; - } else if (start.key == INT64_MIN || (beforeRange && end.key == INT64_MAX)) { + } else if (start.key == INT64_MIN || (beforeTs && end.key == INT64_MAX)) { //hasInterp = true; return true; } @@ -2264,6 +2264,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { if (ts == pSliceInfo->current) { blockDataEnsureCapacity(pResBlock, pResBlock->info.rows + 1); + bool isNullRow = false; for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; @@ -2277,8 +2278,12 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); if (colDataIsNull_s(pSrc, i)) { - colDataAppendNULL(pDst, pResBlock->info.rows); - continue; + if (pSliceInfo->fillType != TSDB_FILL_LINEAR) { + colDataAppendNULL(pDst, pResBlock->info.rows); + continue; + } else { + isNullRow = true; + } } char* v = colDataGetData(pSrc, i); @@ -2286,7 +2291,9 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } } - pResBlock->info.rows += 1; + if (!isNullRow) { + pResBlock->info.rows += 1; + } doKeepPrevRows(pSliceInfo, pBlock, i); doKeepLinearInfo(pSliceInfo, pBlock, i); @@ -2342,6 +2349,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { // add current row if timestamp match if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) { blockDataEnsureCapacity(pResBlock, pResBlock->info.rows + 1); + bool isNullRow = false; for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; @@ -2355,8 +2363,12 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); if (colDataIsNull_s(pSrc, i)) { - colDataAppendNULL(pDst, pResBlock->info.rows); - continue; + if (pSliceInfo->fillType != TSDB_FILL_LINEAR) { + colDataAppendNULL(pDst, pResBlock->info.rows); + continue; + } else { + isNullRow = true; + } } char* v = colDataGetData(pSrc, i); @@ -2364,7 +2376,9 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } } - pResBlock->info.rows += 1; + if (!isNullRow) { + pResBlock->info.rows += 1; + } doKeepPrevRows(pSliceInfo, pBlock, i); pSliceInfo->current = From 7500f3e27eb52874b278703b6430c3dd72a3c20b Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 8 Nov 2022 18:26:07 +0800 Subject: [PATCH 05/95] add test case --- tests/system-test/2-query/interp.py | 256 +++++++++++++++++++++++++++- 1 file changed, 253 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index 4db074011d..fa5c35f782 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -18,6 +18,7 @@ class TDTestCase: def run(self): dbname = "db" tbname = "tb" + tbname1 = "tb1" stbname = "stb" ctbname1 = "ctb1" ctbname2 = "ctb2" @@ -878,7 +879,256 @@ class TDTestCase: tdSql.checkData(0, 0, 14) tdSql.checkData(1, 0, 15) - tdLog.printNoPrefix("==========step10:test multi-interp cases") + + tdLog.printNoPrefix("==========step10:test interp with null data") + tdSql.execute( + f'''create table if not exists {dbname}.{tbname1} + (ts timestamp, c0 int, c1 int) + ''' + ) + + + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:00', 0, NULL)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:05', NULL, NULL)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:10', 10, 10)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:15', NULL, NULL)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:20', 20, NULL)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:25', NULL, NULL)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:30', 30, 30)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:35', 35, NULL)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:40', 40, 40)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:45', NULL, 45)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:50', 50, NULL)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:55', NULL, NULL)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:01:00', 55, 60)") + + # check c0 + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:00') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:03') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:05') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:08') every(1s) fill(linear)") + tdSql.checkRows(9) + tdSql.checkData(0, 0, 0) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 2) + tdSql.checkData(3, 0, 3) + tdSql.checkData(4, 0, 4) + tdSql.checkData(5, 0, 5) + tdSql.checkData(6, 0, 6) + tdSql.checkData(7, 0, 7) + tdSql.checkData(8, 0, 8) + + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:01', '2020-02-02 00:00:03') every(1s) fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:03', '2020-02-02 00:00:08') every(1s) fill(linear)") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 4) + tdSql.checkData(2, 0, 5) + tdSql.checkData(3, 0, 6) + tdSql.checkData(4, 0, 7) + tdSql.checkData(5, 0, 8) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:05', '2020-02-02 00:00:10') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:05', '2020-02-02 00:00:15') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:05', '2020-02-02 00:00:18') every(1s) fill(linear)") + tdSql.checkRows(9) + tdSql.checkData(0, 0, 10) + tdSql.checkData(1, 0, 11) + tdSql.checkData(2, 0, 12) + tdSql.checkData(3, 0, 13) + tdSql.checkData(4, 0, 14) + tdSql.checkData(5, 0, 15) + tdSql.checkData(6, 0, 16) + tdSql.checkData(7, 0, 17) + tdSql.checkData(8, 0, 18) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:05', '2020-02-02 00:00:20') every(1s) fill(linear)") + tdSql.checkRows(11) + tdSql.checkData(0, 0, 10) + tdSql.checkData(1, 0, 11) + tdSql.checkData(2, 0, 12) + tdSql.checkData(3, 0, 13) + tdSql.checkData(4, 0, 14) + tdSql.checkData(5, 0, 15) + tdSql.checkData(6, 0, 16) + tdSql.checkData(7, 0, 17) + tdSql.checkData(8, 0, 18) + tdSql.checkData(9, 0, 19) + tdSql.checkData(10, 0, 20) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:09', '2020-02-02 00:00:11') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:10', '2020-02-02 00:00:15') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:12', '2020-02-02 00:00:13') every(1s) fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:12', '2020-02-02 00:00:15') every(1s) fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:12', '2020-02-02 00:00:18') every(1s) fill(linear)") + tdSql.checkRows(7) + tdSql.checkData(0, 0, 12) + tdSql.checkData(1, 0, 13) + tdSql.checkData(2, 0, 14) + tdSql.checkData(3, 0, 15) + tdSql.checkData(4, 0, 16) + tdSql.checkData(5, 0, 17) + tdSql.checkData(6, 0, 18) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:30', '2020-02-02 00:00:40') every(1s) fill(linear)") + tdSql.checkRows(11) + tdSql.checkData(0, 0, 30) + tdSql.checkData(10, 0, 40) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:25', '2020-02-02 00:00:45') every(1s) fill(linear)") + tdSql.checkRows(11) + tdSql.checkData(0, 0, 30) + tdSql.checkData(10, 0, 40) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:25', '2020-02-02 00:00:45') every(1s) fill(linear)") + tdSql.checkRows(11) + tdSql.checkData(0, 0, 30) + tdSql.checkData(10, 0, 40) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:20', '2020-02-02 00:00:40') every(1s) fill(linear)") + tdSql.checkRows(21) + tdSql.checkData(0, 0, 20) + tdSql.checkData(20, 0, 40) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:30', '2020-02-02 00:00:50') every(1s) fill(linear)") + tdSql.checkRows(21) + tdSql.checkData(0, 0, 30) + tdSql.checkData(20, 0, 50) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:20', '2020-02-02 00:00:50') every(1s) fill(linear)") + tdSql.checkRows(31) + tdSql.checkData(0, 0, 20) + tdSql.checkData(30, 0, 50) + + # check c1 + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:05') every(1s) fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:05') every(1s) fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:08') every(1s) fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:10') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:15') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:20') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:25') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:30') every(1s) fill(linear)") + tdSql.checkRows(21) + tdSql.checkData(0, 0, 10) + tdSql.checkData(1, 0, 11) + tdSql.checkData(2, 0, 12) + tdSql.checkData(3, 0, 13) + tdSql.checkData(4, 0, 14) + tdSql.checkData(5, 0, 15) + tdSql.checkData(6, 0, 16) + tdSql.checkData(7, 0, 17) + tdSql.checkData(8, 0, 18) + tdSql.checkData(9, 0, 19) + tdSql.checkData(10, 0, 20) + tdSql.checkData(11, 0, 21) + tdSql.checkData(12, 0, 22) + tdSql.checkData(13, 0, 23) + tdSql.checkData(14, 0, 24) + tdSql.checkData(15, 0, 25) + tdSql.checkData(16, 0, 26) + tdSql.checkData(17, 0, 27) + tdSql.checkData(18, 0, 28) + tdSql.checkData(19, 0, 29) + tdSql.checkData(20, 0, 30) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:35') every(1s) fill(linear)") + tdSql.checkRows(21) + tdSql.checkData(0, 0, 10) + tdSql.checkData(20, 0, 30) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:40') every(1s) fill(linear)") + tdSql.checkRows(31) + tdSql.checkData(0, 0, 10) + tdSql.checkData(30, 0, 40) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:45') every(1s) fill(linear)") + tdSql.checkRows(36) + tdSql.checkData(0, 0, 10) + tdSql.checkData(35, 0, 45) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:50') every(1s) fill(linear)") + tdSql.checkRows(36) + tdSql.checkData(0, 0, 10) + tdSql.checkData(35, 0, 45) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:55') every(1s) fill(linear)") + tdSql.checkRows(36) + tdSql.checkData(0, 0, 10) + tdSql.checkData(35, 0, 45) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(linear)") + tdSql.checkRows(51) + tdSql.checkData(0, 0, 10) + tdSql.checkData(50, 0, 60) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:40', '2020-02-02 00:00:45') every(1s) fill(linear)") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 40) + tdSql.checkData(5, 0, 45) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:35', '2020-02-02 00:00:50') every(1s) fill(linear)") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 40) + tdSql.checkData(5, 0, 45) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:35', '2020-02-02 00:00:55') every(1s) fill(linear)") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 40) + tdSql.checkData(5, 0, 45) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:30', '2020-02-02 00:01:00') every(1s) fill(linear)") + tdSql.checkRows(31) + tdSql.checkData(0, 0, 30) + tdSql.checkData(30, 0, 60) + + tdLog.printNoPrefix("==========step11:test multi-interp cases") tdSql.query(f"select interp(c0),interp(c1),interp(c2),interp(c3) from {dbname}.{tbname} range('2020-02-09 00:00:05', '2020-02-13 00:00:05') every(1d) fill(null)") tdSql.checkRows(5) tdSql.checkCols(4) @@ -932,7 +1182,7 @@ class TDTestCase: for i in range (tdSql.queryCols): tdSql.checkData(0, i, 13) - tdLog.printNoPrefix("==========step11:test error cases") + tdLog.printNoPrefix("==========step12:test error cases") tdSql.error(f"select interp(c0) from {dbname}.{tbname}") tdSql.error(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05')") @@ -955,7 +1205,7 @@ class TDTestCase: tdSql.error(f"select interp('abcd') from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") tdSql.error(f"select interp('中文字符') from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") - tdLog.printNoPrefix("==========step12:stable cases") + tdLog.printNoPrefix("==========step13:stable cases") #tdSql.query(f"select interp(c0) from {dbname}.{stbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(null)") #tdSql.checkRows(13) From b902ab0d6184f193bd384c6037b783f876281cf3 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 8 Nov 2022 18:48:22 +0800 Subject: [PATCH 06/95] refactor code --- source/libs/executor/src/timewindowoperator.c | 97 +++++++------------ 1 file changed, 36 insertions(+), 61 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 75d7010eb1..461a4fbe43 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2121,6 +2121,40 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp return hasInterp; } +static void addCurrentRowToResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, + SSDataBlock* pSrcBlock, int32_t index) { + blockDataEnsureCapacity(pResBlock, pResBlock->info.rows + 1); + for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) { + SExprInfo* pExprInfo = &pExprSup->pExprInfo[j]; + + int32_t dstSlot = pExprInfo->base.resSchema.slotId; + SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); + + if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { + colDataAppend(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false); + } else { + int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot); + + if (colDataIsNull_s(pSrc, index)) { + if (pSliceInfo->fillType != TSDB_FILL_LINEAR) { + colDataAppendNULL(pDst, pResBlock->info.rows); + continue; + } else { + return; + } + } + + char* v = colDataGetData(pSrc, index); + colDataAppend(pDst, pResBlock->info.rows, v, false); + } + } + + pResBlock->info.rows += 1; + return; +} + + static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) { if (pInfo->pPrevRow != NULL) { return TSDB_CODE_SUCCESS; @@ -2263,37 +2297,8 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (ts == pSliceInfo->current) { - blockDataEnsureCapacity(pResBlock, pResBlock->info.rows + 1); - bool isNullRow = false; - for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { - SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; + addCurrentRowToResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i); - int32_t dstSlot = pExprInfo->base.resSchema.slotId; - SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); - - if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { - colDataAppend(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false); - } else { - int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; - SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); - - if (colDataIsNull_s(pSrc, i)) { - if (pSliceInfo->fillType != TSDB_FILL_LINEAR) { - colDataAppendNULL(pDst, pResBlock->info.rows); - continue; - } else { - isNullRow = true; - } - } - - char* v = colDataGetData(pSrc, i); - colDataAppend(pDst, pResBlock->info.rows, v, false); - } - } - - if (!isNullRow) { - pResBlock->info.rows += 1; - } doKeepPrevRows(pSliceInfo, pBlock, i); doKeepLinearInfo(pSliceInfo, pBlock, i); @@ -2348,37 +2353,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { // add current row if timestamp match if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) { - blockDataEnsureCapacity(pResBlock, pResBlock->info.rows + 1); - bool isNullRow = false; - for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { - SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; - - int32_t dstSlot = pExprInfo->base.resSchema.slotId; - SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); - - if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { - colDataAppend(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false); - } else { - int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; - SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); - - if (colDataIsNull_s(pSrc, i)) { - if (pSliceInfo->fillType != TSDB_FILL_LINEAR) { - colDataAppendNULL(pDst, pResBlock->info.rows); - continue; - } else { - isNullRow = true; - } - } - - char* v = colDataGetData(pSrc, i); - colDataAppend(pDst, pResBlock->info.rows, v, false); - } - } - - if (!isNullRow) { - pResBlock->info.rows += 1; - } + addCurrentRowToResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i); doKeepPrevRows(pSliceInfo, pBlock, i); pSliceInfo->current = From f6de1a8634444e624803911fef5e7c643a252809 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Wed, 9 Nov 2022 18:42:01 +0800 Subject: [PATCH 07/95] enh crash_gen for 3.0 about [stream processing] , [data subscription] ,[delete data] --- tests/pytest/crash_gen.sh | 2 +- tests/pytest/crash_gen/crash_gen_main.py | 748 ++++++++++++++++++++++- tests/pytest/crash_gen/shared/db.py | 58 +- 3 files changed, 774 insertions(+), 34 deletions(-) diff --git a/tests/pytest/crash_gen.sh b/tests/pytest/crash_gen.sh index 539314dea4..cc2941a52a 100755 --- a/tests/pytest/crash_gen.sh +++ b/tests/pytest/crash_gen.sh @@ -45,7 +45,7 @@ fi # Now getting ready to execute Python # The following is the default of our standard dev env (Ubuntu 20.04), modify/adjust at your own risk -PYTHON_EXEC=python3.8 +PYTHON_EXEC=python3 # First we need to set up a path for Python to find our own TAOS modules, so that "import" can work. # export PYTHONPATH=$(pwd)/../../src/connector/python:$(pwd) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 600c64b8e6..757ddf4af8 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -37,6 +37,7 @@ import requests # from guppy import hpy import gc import taos +from taos.tmq import * from .shared.types import TdColumns, TdTags @@ -254,7 +255,7 @@ class WorkerThread: class ThreadCoordinator: - WORKER_THREAD_TIMEOUT = 120 # Normal: 120 + WORKER_THREAD_TIMEOUT = 1200 # Normal: 120 def __init__(self, pool: ThreadPool, dbManager: DbManager): self._curStep = -1 # first step is 0 @@ -674,9 +675,13 @@ class AnyState: # only "under normal circumstances", as we may override it with the -b option CAN_DROP_DB = 2 CAN_CREATE_FIXED_SUPER_TABLE = 3 + CAN_CREATE_STREAM = 3 # super table must exists + CAN_CREATE_TOPIC = 3 # super table must exists + CAN_CREATE_CONSUMERS = 3 CAN_DROP_FIXED_SUPER_TABLE = 4 CAN_ADD_DATA = 5 CAN_READ_DATA = 6 + CAN_DELETE_DATA = 6 def __init__(self): self._info = self.getInfo() @@ -727,12 +732,24 @@ class AnyState: return False return self._info[self.CAN_DROP_FIXED_SUPER_TABLE] + def canCreateTopic(self): + return self._info[self.CAN_CREATE_TOPIC] + + def canCreateConsumers(self): + return self._info[self.CAN_CREATE_CONSUMERS] + + def canCreateStream(self): + return self._info[self.CAN_CREATE_STREAM] + def canAddData(self): return self._info[self.CAN_ADD_DATA] def canReadData(self): return self._info[self.CAN_READ_DATA] + def canDeleteData(self): + return self._info[self.CAN_DELETE_DATA] + def assertAtMostOneSuccess(self, tasks, cls): sCnt = 0 for task in tasks: @@ -921,7 +938,7 @@ class StateMechine: except taos.error.ProgrammingError as err: Logging.error("Failed to initialized state machine, cannot find current state: {}".format(err)) traceback.print_stack() - raise # re-throw + pass # re-throw # TODO: seems no lnoger used, remove? def getCurrentState(self): @@ -974,14 +991,21 @@ class StateMechine: # did not do this when openning connection, and this is NOT the worker # thread, which does this on their own dbc.use(dbName) + if not dbc.hasTables(): # no tables + Logging.debug("[STT] DB_ONLY found, between {} and {}".format(ts, time.time())) return StateDbOnly() # For sure we have tables, which means we must have the super table. # TODO: are we sure? + sTable = self._db.getFixedSuperTable() - if sTable.hasRegTables(dbc): # no regular tables + + + if sTable.hasRegTables(dbc): # no regular tables + # print("debug=====*\n"*100) Logging.debug("[STT] SUPER_TABLE_ONLY found, between {} and {}".format(ts, time.time())) + return StateSuperTableOnly() else: # has actual tables Logging.debug("[STT] HAS_DATA found, between {} and {}".format(ts, time.time())) @@ -1109,6 +1133,7 @@ class Database: return "fs_table" def getFixedSuperTable(self) -> TdSuperTable: + return TdSuperTable(self.getFixedSuperTableName(), self.getName()) # We aim to create a starting time tick, such that, whenever we run our test here once @@ -1342,7 +1367,13 @@ class Task(): 0x2603, # Table does not exist, replaced by 2662 below 0x260d, # Tags number not matched 0x2662, # Table does not exist #TODO: what about 2603 above? - + 0x032C, # Object is creating + 0x032D, # Object is dropping + 0x03D3, # Conflict transaction not completed + 0x0707, # Query not ready , it always occur at replica 3 + 0x707, # Query not ready + 0x396, # Database in creating status + 0x386, # Database in droping status 1000 # REST catch-all error @@ -1638,9 +1669,12 @@ class TaskCreateDb(StateTransitionTask): # numReplica = Dice.throw(Settings.getConfig().max_replicas) + 1 # 1,2 ... N numReplica = Config.getConfig().num_replicas # fixed, always repStr = "replica {}".format(numReplica) - updatePostfix = "update 1" if Config.getConfig().verify_data else "" # allow update only when "verify data" is active + updatePostfix = "" if Config.getConfig().verify_data else "" # allow update only when "verify data" is active , 3.0 version default is update 1 + vg_nums = random.randint(1,8) + cache_model = Dice.choice(['none' , 'last_row' , 'last_value' , 'both']) + buffer = random.randint(3,128) dbName = self._db.getName() - self.execWtSql(wt, "create database {} {} {} ".format(dbName, repStr, updatePostfix ) ) + self.execWtSql(wt, "create database {} {} {} vgroups {} cachemodel '{}' buffer {} ".format(dbName, repStr, updatePostfix, vg_nums, cache_model,buffer ) ) if dbName == "db_0" and Config.getConfig().use_shadow_db: self.execWtSql(wt, "create database {} {} {} ".format("db_s", repStr, updatePostfix ) ) @@ -1654,9 +1688,237 @@ class TaskDropDb(StateTransitionTask): return state.canDropDb() def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + + # drop topics before drop db + + if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): + + self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) + self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),self._db.getFixedSuperTableName) + self.execWtSql(wt, "drop database {}".format(self._db.getName())) + Logging.debug("[OPS] database dropped at {}".format(time.time())) + +''' +# Streams will generator TD-20237 （it will crash taosd , start this task when this issue fixed ） + +class TaskCreateStream(StateTransitionTask): + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canCreateStream() + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + dbname = self._db.getName() + + sub_stream_name = dbname+ '_sub_stream' + sub_stream_tb_name = 'stream_tb_sub' + super_stream_name = dbname+ '_super_stream' + super_stream_tb_name = 'stream_tb_super' + if not self._db.exists(wt.getDbConn()): + Logging.debug("Skipping task, no DB yet") + return + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + + # create stream + + # CREATE STREAM avg_vol_s INTO avg_vol AS SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); + + stbname =sTable.getName() + sub_tables = sTable.getRegTables(wt.getDbConn()) + aggExpr = Dice.choice([ + 'count(*)', + 'avg(speed)', + # 'twa(speed)', # TODO: this one REQUIRES a where statement, not reasonable + 'sum(speed)', + 'stddev(speed)', + # SELECTOR functions + 'min(speed)', + 'max(speed)', + 'first(speed)', + 'last(speed)', + 'apercentile(speed, 10)', # TODO: TD-1316 + 'last_row(*)', # TODO: commented out per TD-3231, we should re-create + # Transformation Functions + 'twa(speed)' + + ]) # TODO: add more from 'top' + + if sub_tables: + + if sub_tables: # if not empty + sub_tbname = sub_tables[0] + # create stream with query above sub_table + stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(sub_stream_name,dbname,sub_stream_tb_name ,aggExpr,dbname,sub_tbname) + try: + self.execWtSql(wt, stream_sql) + Logging.debug("[OPS] stream is creating at {}".format(time.time())) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x03f0]: # stream already exists + # stream need drop before drop table + pass + + else: + pass + + else: + stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(super_stream_name,dbname,super_stream_tb_name,aggExpr, dbname,stbname) + + try: + self.execWtSql(wt, stream_sql) + Logging.debug("[OPS] stream is creating at {}".format(time.time())) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x03f0]: # stream already exists + # stream need drop before drop table + pass +''' + +class TaskCreateTopic(StateTransitionTask): + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canCreateTopic() + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + dbname = self._db.getName() + + sub_topic_name = dbname+ '_sub_topic' + super_topic_name = dbname+ '_super_topic' + stable_topic = dbname+ '_stable_topic' + db_topic = 'database_' + dbname+ '_topics' + if not self._db.exists(wt.getDbConn()): + Logging.debug("Skipping task, no DB yet") + return + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + + # create topic + + # create topic if not exists topic_ctb_column as select ts, c1, c2, c3 from stb1; + + stbname =sTable.getName() + sub_tables = sTable.getRegTables(wt.getDbConn()) + scalarExpr = Dice.choice([ '*','speed','color', + 'abs(speed)', + 'acos(speed)', + 'asin(speed)', + 'atan(speed)', + 'ceil(speed)', + 'cos(speed)', + 'cos(speed)', + 'floor(speed)', + 'log(speed,2)', + 'pow(speed,2)', + 'round(speed)', + 'sin(speed)', + 'sqrt(speed)', + 'char_length(color)', + 'concat(color,color)', + 'concat_ws(" ", color,color," ")', + 'length(color)', + 'lower(color)', + 'ltrim(color)', + 'substr(color , 2)', + 'upper(color)', + 'cast(speed as double)', + 'cast(ts as bigint)', + + ]) # TODO: add more from 'top' + if Dice.throw(3)==0: + + if sub_tables: + + if sub_tables: # if not empty + sub_tbname = sub_tables[0] + # create stream with query above sub_table + topic_sql = 'create topic {} as select {} FROM {}.{} ; '.format(sub_topic_name,scalarExpr,dbname,sub_tbname) + try: + self.execWtSql(wt, "use {}".format(dbname)) + self.execWtSql(wt, topic_sql) + Logging.debug("[OPS] topic is creating at {}".format(time.time())) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x03f0]: # topic already exists + # topic need drop before drop table + pass + + else: + pass + + else: + topic_sql = 'create topic {} as select {} FROM {}.{} '.format(super_topic_name,scalarExpr, dbname,stbname) + try: + self.execWtSql(wt, "use {}".format(dbname)) + self.execWtSql(wt, topic_sql) + Logging.debug("[OPS] subquery topic is creating at {}".format(time.time())) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x03f0]: # topic already exists + # topic need drop before drop table + pass + elif Dice.throw(3)==1: + topic_sql = 'create topic {} AS STABLE {}.{} '.format(stable_topic,dbname,stbname) + try: + self.execWtSql(wt, "use {}".format(dbname)) + self.execWtSql(wt, topic_sql) + Logging.debug("[OPS] stable topic is creating at {}".format(time.time())) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x03f0]: # topic already exists + # topic need drop before drop table + pass + elif Dice.throw(3)==2: + topic_sql = 'create topic {} AS DATABASE {} '.format(db_topic,dbname) + try: + self.execWtSql(wt, "use {}".format(dbname)) + self.execWtSql(wt, topic_sql) + Logging.debug("[OPS] db topic is creating at {}".format(time.time())) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x03f0]: # topic already exists + # topic need drop before drop table + pass + else: + pass + + +class TaskCreateConsumers(StateTransitionTask): + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canCreateConsumers() + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + dbname = self._db.getName() + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + + # create Consumers + if Dice.throw(50)==0: # because subscribe is cost so much time , Reduce frequency of this task + if sTable.hasTopics(wt.getDbConn()): + sTable.createConsumer(wt.getDbConn(),random.randint(1,10)) + + class TaskCreateSuperTable(StateTransitionTask): @classmethod def getEndState(cls): @@ -1673,7 +1935,7 @@ class TaskCreateSuperTable(StateTransitionTask): sTable = self._db.getFixedSuperTable() # type: TdSuperTable # wt.execSql("use db") # should always be in place - + sTable.create(wt.getDbConn(), {'ts': TdDataType.TIMESTAMP, 'speed': TdDataType.INT, 'color': TdDataType.BINARY16}, { 'b': TdDataType.BINARY200, 'f': TdDataType.FLOAT}, @@ -1688,15 +1950,42 @@ class TdSuperTable: def __init__(self, stName, dbName): self._stName = stName self._dbName = dbName + self._consumerLists = {} + self._ConsumerInsts = [] def getName(self): return self._stName + def drop(self, dbc, skipCheck = False): dbName = self._dbName if self.exists(dbc) : # if myself exists - fullTableName = dbName + '.' + self._stName - dbc.execute("DROP TABLE {}".format(fullTableName)) + fullTableName = dbName + '.' + self._stName + if self.hasStreams(dbc): + self.dropStreams(dbc) + self.dropStreamTables(dbc) + if self.hasTopics(dbc): + self.dropTopics(dbName,None) + self.dropTopics(dbName,self._stName) + try: + dbc.execute("DROP TABLE {}".format(fullTableName)) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [1011,0x3F3,0x03f3,0x2662]: # table doesn't exist + pass + # # stream need drop before drop table + # for stream in self.getStreamName(): + # drop_stream_sql = 'drop stream {}'.format(stream) + # try: + # dbc.execute(drop_stream_sql) + # except taos.error.ProgrammingError as err: + # # correcting for strange error number scheme + # errno3 = Helper.convertErrno(err.errno) + # if errno3 in [1011,0x3F3,0x03f3,0x2662,0x03f1]: # stream not exists + # pass + # dbc.execute("DROP TABLE {}".format(fullTableName)) + # pass + else: if not skipCheck: raise CrashGenError("Cannot drop non-existant super table: {}".format(self._stName)) @@ -1711,10 +2000,24 @@ class TdSuperTable: dbName = self._dbName dbc.execute("USE " + dbName) - fullTableName = dbName + '.' + self._stName + fullTableName = dbName + '.' + self._stName + if dbc.existsSuperTable(self._stName): if dropIfExists: + if self.hasStreams(dbc): + self.dropStreams(dbc) + self.dropStreamTables(dbc) + + # drop topics before drop stables + if self.hasTopics(dbc): + self.dropTopics(dbc,self._dbName,None) + self.dropTopics(dbc,self._dbName,self._stName ) + + dbc.execute("DROP TABLE {}".format(fullTableName)) + + pass + # dbc.execute("DROP TABLE {}".format(fullTableName)) else: # error raise CrashGenError("Cannot create super table, already exists: {}".format(self._stName)) @@ -1728,12 +2031,52 @@ class TdSuperTable: ) else: sql += " TAGS (dummy int) " - dbc.execute(sql) + dbc.execute(sql) + + def createConsumer(self, dbc: DbConn , Consumer_nums): + + def generateConsumer(current_topic_list): + conf = TaosTmqConf() + conf.set("group.id", "tg2") + conf.set("td.connect.user", "root") + conf.set("td.connect.pass", "taosdata") + conf.set("enable.auto.commit", "true") + def tmq_commit_cb_print(tmq, resp, offset, param=None): + print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}") + conf.set_auto_commit_cb(tmq_commit_cb_print, None) + consumer = conf.new_consumer() + topic_list = TaosTmqList() + for topic in current_topic_list: + topic_list.append(topic) + consumer.subscribe(topic_list) + time.sleep(5) # consumer work only 5 sec ,and then it will exit + try: + consumer.unsubscribe() + except TmqError as e : + pass + return + + # mulit Consumer + current_topic_list = self.getTopicLists(dbc) + for i in range(Consumer_nums): + consumer_inst = threading.Thread(target=generateConsumer, args=(current_topic_list,)) + self._ConsumerInsts.append(consumer_inst) + + for ConsumerInst in self._ConsumerInsts: + ConsumerInst.start() + for ConsumerInst in self._ConsumerInsts: + ConsumerInst.join() + + def getTopicLists(self, dbc: DbConn): + dbc.query("show topics ") + topics = dbc.getQueryResult() + topicLists = [v[0] for v in topics] + return topicLists def getRegTables(self, dbc: DbConn): dbName = self._dbName try: - dbc.query("select TBNAME from {}.{}".format(dbName, self._stName)) # TODO: analyze result set later + dbc.query("select distinct TBNAME from {}.{}".format(dbName, self._stName)) # TODO: analyze result set later except taos.error.ProgrammingError as err: errno2 = Helper.convertErrno(err.errno) Logging.debug("[=] Failed to get tables from super table: errno=0x{:X}, msg: {}".format(errno2, err)) @@ -1743,7 +2086,73 @@ class TdSuperTable: return [v[0] for v in qr] # list transformation, ref: https://stackoverflow.com/questions/643823/python-list-transformation def hasRegTables(self, dbc: DbConn): - return dbc.query("SELECT * FROM {}.{}".format(self._dbName, self._stName)) > 0 + + if dbc.existsSuperTable(self._stName): + + return dbc.query("SELECT * FROM {}.{}".format(self._dbName, self._stName)) > 0 + else: + return False + + def hasStreamTables(self,dbc: DbConn): + + return dbc.query("show {}.stables like 'stream_tb%'".format(self._dbName)) > 0 + + def hasStreams(self,dbc: DbConn): + return dbc.query("show streams") > 0 + + def hasTopics(self,dbc: DbConn): + + return dbc.query("show topics") > 0 + + def dropTopics(self,dbc: DbConn , dbname=None,stb_name=None): + dbc.query("show topics ") + topics = dbc.getQueryResult() + + if dbname !=None and stb_name == None : + + for topic in topics: + if dbname in topic[0] and topic[0].startswith("database"): + try: + dbc.execute('drop topic {}'.format(topic[0])) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x03EB]: # Topic subscribed cannot be dropped + pass + # for subsript in subscriptions: + + else: + pass + + pass + return True + elif dbname !=None and stb_name!= None: + for topic in topics: + if topic[0].startswith(self._dbName) and topic[0].endswith('topic'): + dbc.execute('drop topic {}'.format(topic[0])) + return True + else: + return True + pass + + def dropStreams(self,dbc:DbConn): + dbc.query("show streams ") + Streams = dbc.getQueryResult() + for Stream in Streams: + if Stream[0].startswith(self._dbName): + dbc.execute('drop stream {}'.format(Stream[0])) + + return not dbc.query("show streams ") > 0 + + def dropStreamTables(self, dbc: DbConn): + dbc.query("show {}.stables like 'stream_tb%'".format(self._dbName)) + + StreamTables = dbc.getQueryResult() + + for StreamTable in StreamTables: + if self.dropStreams(dbc): + dbc.execute('drop table {}.{}'.format(self._dbName,StreamTable[0])) + + return not dbc.query("show {}.stables like 'stream_tb%'".format(self._dbName)) def ensureRegTable(self, task: Optional[Task], dbc: DbConn, regTableName: str): ''' @@ -1838,10 +2247,46 @@ class TdSuperTable: # Run the query against the regular table first doAggr = (Dice.throw(2) == 0) # 1 in 2 chance if not doAggr: # don't do aggregate query, just simple one + commonExpr = Dice.choice([ + '*', + 'abs(speed)', + 'acos(speed)', + 'asin(speed)', + 'atan(speed)', + 'ceil(speed)', + 'cos(speed)', + 'cos(speed)', + 'floor(speed)', + 'log(speed,2)', + 'pow(speed,2)', + 'round(speed)', + 'sin(speed)', + 'sqrt(speed)', + 'char_length(color)', + 'concat(color,color)', + 'concat_ws(" ", color,color," ")', + 'length(color)', + 'lower(color)', + 'ltrim(color)', + 'substr(color , 2)', + 'upper(color)', + 'cast(speed as double)', + 'cast(ts as bigint)', + # 'TO_ISO8601(color)', + # 'TO_UNIXTIMESTAMP(ts)', + 'now()', + 'timediff(ts,now)', + 'timezone()', + 'TIMETRUNCATE(ts,1s)', + 'TIMEZONE()', + 'TODAY()', + 'distinct(color)' + ] + ) ret.append(SqlQuery( # reg table - "select {} from {}.{}".format('*', self._dbName, rTbName))) + "select {} from {}.{}".format(commonExpr, self._dbName, rTbName))) ret.append(SqlQuery( # super table - "select {} from {}.{}".format('*', self._dbName, self.getName()))) + "select {} from {}.{}".format(commonExpr, self._dbName, self.getName()))) else: # Aggregate query aggExpr = Dice.choice([ 'count(*)', @@ -1857,17 +2302,34 @@ class TdSuperTable: 'top(speed, 50)', # TODO: not supported? 'bottom(speed, 50)', # TODO: not supported? 'apercentile(speed, 10)', # TODO: TD-1316 - # 'last_row(speed)', # TODO: commented out per TD-3231, we should re-create + 'last_row(*)', # TODO: commented out per TD-3231, we should re-create # Transformation Functions # 'diff(speed)', # TODO: no supported?! - 'spread(speed)' + 'spread(speed)', + 'elapsed(ts)', + 'mode(speed)', + 'bottom(speed,1)', + 'top(speed,1)', + 'tail(speed,1)', + 'unique(color)', + 'csum(speed)', + 'DERIVATIVE(speed,1s,1)', + 'diff(speed,1)', + 'irate(speed)', + 'mavg(speed,3)', + 'sample(speed,5)', + 'STATECOUNT(speed,"LT",1)', + 'STATEDURATION(speed,"LT",1)', + 'twa(speed)' + ]) # TODO: add more from 'top' # if aggExpr not in ['stddev(speed)']: # STDDEV not valid for super tables?! (Done in TD-1049) sql = "select {} from {}.{}".format(aggExpr, self._dbName, self.getName()) if Dice.throw(3) == 0: # 1 in X chance - sql = sql + ' GROUP BY color' + partion_expr = Dice.choice(['color','tbname']) + sql = sql + ' partition BY ' + partion_expr + ' order by ' + partion_expr Progress.emit(Progress.QUERY_GROUP_BY) # Logging.info("Executing GROUP-BY query: " + sql) ret.append(SqlQuery(sql)) @@ -1965,15 +2427,12 @@ class TaskDropSuperTable(StateTransitionTask): for i in tblSeq: regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) try: + self.execWtSql(wt, "drop table {}.{}". format(self._db.getName(), regTableName)) # nRows always 0, like MySQL except taos.error.ProgrammingError as err: - # correcting for strange error number scheme - errno2 = Helper.convertErrno(err.errno) - if (errno2 in [0x362]): # mnode invalid table name - isSuccess = False - Logging.debug("[DB] Acceptable error when dropping a table") - continue # try to delete next regular table + pass + if (not tickOutput): tickOutput = True # Print only one time @@ -1984,7 +2443,28 @@ class TaskDropSuperTable(StateTransitionTask): # Drop the super table itself tblName = self._db.getFixedSuperTableName() - self.execWtSql(wt, "drop table {}.{}".format(self._db.getName(), tblName)) + + # drop streams before drop stables + if self._db.getFixedSuperTable().hasStreams(wt.getDbConn()): + self._db.getFixedSuperTable().dropStreams(wt.getDbConn()) + self._db.getFixedSuperTable().dropStreamTables(wt.getDbConn()) + + # drop topics before drop stables + if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): + self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) + self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),tblName) + + try: + self.execWtSql(wt, "drop table {}.{}".format(self._db.getName(), tblName)) + except taos.error.ProgrammingError as err: + # correcting for strange error number scheme + errno2 = Helper.convertErrno(err.errno) + if (errno2 in [0x362]): # mnode invalid table name + isSuccess = False + Logging.debug("[DB] Acceptable error when dropping a table") + elif errno2 in [1011,0x3F3,0x03f3]: # table doesn't exist + + pass class TaskAlterTags(StateTransitionTask): @@ -2234,6 +2714,220 @@ class TaskAddData(StateTransitionTask): self.activeTable.discard(i) # not raising an error, unlike remove +class TaskDeleteData(StateTransitionTask): + # Track which table is being actively worked on + activeTable: Set[int] = set() + + # We use these two files to record operations to DB, useful for power-off tests + fAddLogReady = None # type: Optional[io.TextIOWrapper] + fAddLogDone = None # type: Optional[io.TextIOWrapper] + + @classmethod + def prepToRecordOps(cls): + if Config.getConfig().record_ops: + if (cls.fAddLogReady is None): + Logging.info( + "Recording in a file operations to be performed...") + cls.fAddLogReady = open("add_log_ready.txt", "w") + if (cls.fAddLogDone is None): + Logging.info("Recording in a file operations completed...") + cls.fAddLogDone = open("add_log_done.txt", "w") + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canDeleteData() + + def _lockTableIfNeeded(self, fullTableName, extraMsg = ''): + if Config.getConfig().verify_data: + # Logging.info("Locking table: {}".format(fullTableName)) + self.lockTable(fullTableName) + # Logging.info("Table locked {}: {}".format(extraMsg, fullTableName)) + # print("_w" + str(nextInt % 100), end="", flush=True) # Trace what was written + else: + # Logging.info("Skipping locking table") + pass + + def _unlockTableIfNeeded(self, fullTableName): + if Config.getConfig().verify_data: + # Logging.info("Unlocking table: {}".format(fullTableName)) + self.unlockTable(fullTableName) + # Logging.info("Table unlocked: {}".format(fullTableName)) + else: + pass + # Logging.info("Skipping unlocking table") + + def _deleteData(self, db: Database, dbc, regTableName, te: TaskExecutor): # implied: NOT in batches + numRecords = self.LARGE_NUMBER_OF_RECORDS if Config.getConfig().larger_data else self.SMALL_NUMBER_OF_RECORDS + del_Records = int(numRecords/5) + if Dice.throw(2) == 0: + for j in range(del_Records): # number of records per table + intToWrite = db.getNextInt() + nextTick = db.getNextTick() + # nextColor = db.getNextColor() + if Config.getConfig().record_ops: + self.prepToRecordOps() + if self.fAddLogReady is None: + raise CrashGenError("Unexpected empty fAddLogReady") + self.fAddLogReady.write("Ready to delete {} to {}\n".format(intToWrite, regTableName)) + self.fAddLogReady.flush() + os.fsync(self.fAddLogReady.fileno()) + + # TODO: too ugly trying to lock the table reliably, refactor... + fullTableName = db.getName() + '.' + regTableName + self._lockTableIfNeeded(fullTableName) # so that we are verify read-back. TODO: deal with exceptions before unlock + + try: + sql = "delete from {} where ts = '{}' ;".format( # removed: tags ('{}', {}) + fullTableName, + # ds.getFixedSuperTableName(), + # ds.getNextBinary(), ds.getNextFloat(), + nextTick) + + # print(sql) + # Logging.info("Adding data: {}".format(sql)) + dbc.execute(sql) + # Logging.info("Data added: {}".format(sql)) + intWrote = intToWrite + + # Quick hack, attach an update statement here. TODO: create an "update" task + if (not Config.getConfig().use_shadow_db) and Dice.throw(5) == 0: # 1 in N chance, plus not using shaddow DB + intToUpdate = db.getNextInt() # Updated， but should not succeed + # nextColor = db.getNextColor() + sql = "delete from {} where ts = '{}' ;".format( # "INSERt" means "update" here + fullTableName, + nextTick) + # sql = "UPDATE {} set speed={}, color='{}' WHERE ts='{}'".format( + # fullTableName, db.getNextInt(), db.getNextColor(), nextTick) + dbc.execute(sql) + intWrote = intToUpdate # We updated, seems TDengine non-cluster accepts this. + + except: # Any exception at all + self._unlockTableIfNeeded(fullTableName) + raise + + # Now read it back and verify, we might encounter an error if table is dropped + if Config.getConfig().verify_data: # only if command line asks for it + try: + dbc.query("SELECT * from {}.{} WHERE ts='{}'". + format(db.getName(), regTableName, nextTick)) + result = dbc.getQueryResult() + if len(result)==0: + # means data has been delete + print("D1",end="") # DF means delete failed + else: + print("DF",end="") # DF means delete failed + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + # if errno == CrashGenError.INVALID_EMPTY_RESULT: # empty result + # print("D1",end="") # D1 means delete data success and only 1 record + + if errno in [0x218, 0x362,0x2662]: # table doesn't exist + # do nothing + pass + else: + # Re-throw otherwise + raise + finally: + self._unlockTableIfNeeded(fullTableName) # Quite ugly, refactor lock/unlock + # Done with read-back verification, unlock the table now + # Successfully wrote the data into the DB, let's record it somehow + te.recordDataMark(intWrote) + else: + + # delete all datas and verify datas ,expected table is empty + if Config.getConfig().record_ops: + self.prepToRecordOps() + if self.fAddLogReady is None: + raise CrashGenError("Unexpected empty fAddLogReady") + self.fAddLogReady.write("Ready to delete {} to {}\n".format(intToWrite, regTableName)) + self.fAddLogReady.flush() + os.fsync(self.fAddLogReady.fileno()) + + # TODO: too ugly trying to lock the table reliably, refactor... + fullTableName = db.getName() + '.' + regTableName + self._lockTableIfNeeded(fullTableName) # so that we are verify read-back. TODO: deal with exceptions before unlock + + try: + sql = "delete from {} ;".format( # removed: tags ('{}', {}) + fullTableName) + # Logging.info("Adding data: {}".format(sql)) + dbc.execute(sql) + # Logging.info("Data added: {}".format(sql)) + + # Quick hack, attach an update statement here. TODO: create an "update" task + if (not Config.getConfig().use_shadow_db) and Dice.throw(5) == 0: # 1 in N chance, plus not using shaddow DB + sql = "delete from {} ;".format( # "INSERt" means "update" here + fullTableName) + dbc.execute(sql) + + except: # Any exception at all + self._unlockTableIfNeeded(fullTableName) + raise + + # Now read it back and verify, we might encounter an error if table is dropped + if Config.getConfig().verify_data: # only if command line asks for it + try: + dbc.query("SELECT * from {}.{} WHERE ts='{}'". + format(db.getName(), regTableName, nextTick)) + result = dbc.getQueryResult() + if len(result)==0: + # means data has been delete + print("DA",end="") + else: + print("DF",end="") # DF means delete failed + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + # if errno == CrashGenError.INVALID_EMPTY_RESULT: # empty result + # print("Da",end="") # Da means delete data success and for all datas + + if errno in [0x218, 0x362,0x2662]: # table doesn't exist + # do nothing + pass + else: + # Re-throw otherwise + raise + finally: + self._unlockTableIfNeeded(fullTableName) # Quite ugly, refactor lock/unlock + # Done with read-back verification, unlock the table now + + if Config.getConfig().record_ops: + if self.fAddLogDone is None: + raise CrashGenError("Unexpected empty fAddLogDone") + self.fAddLogDone.write("Wrote {} to {}\n".format(intWrote, regTableName)) + self.fAddLogDone.flush() + os.fsync(self.fAddLogDone.fileno()) + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + # ds = self._dbManager # Quite DANGEROUS here, may result in multi-thread client access + db = self._db + dbc = wt.getDbConn() + numTables = self.LARGE_NUMBER_OF_TABLES if Config.getConfig().larger_data else self.SMALL_NUMBER_OF_TABLES + numRecords = self.LARGE_NUMBER_OF_RECORDS if Config.getConfig().larger_data else self.SMALL_NUMBER_OF_RECORDS + tblSeq = list(range(numTables )) + random.shuffle(tblSeq) # now we have random sequence + for i in tblSeq: + if (i in self.activeTable): # wow already active + # print("x", end="", flush=True) # concurrent insertion + Progress.emit(Progress.CONCURRENT_INSERTION) + else: + self.activeTable.add(i) # marking it active + + dbName = db.getName() + sTable = db.getFixedSuperTable() + regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) + fullTableName = dbName + '.' + regTableName + # self._lockTable(fullTableName) # "create table" below. Stop it if the table is "locked" + sTable.ensureRegTable(self, wt.getDbConn(), regTableName) # Ensure the table exists + # self._unlockTable(fullTableName) + + self._deleteData(db, dbc, regTableName, te) + + self.activeTable.discard(i) # not raising an error, unlike remove + class ThreadStacks: # stack info for all threads def __init__(self): @@ -2259,7 +2953,8 @@ class ThreadStacks: # stack info for all threads # Now print print("\n<----- Thread Info for LWP/ID: {} (most recent call last) <-----".format(shortTid)) lastSqlForThread = DbConn.fetchSqlForThread(shortTid) - print("Last SQL statement attempted from thread {} is: {}".format(shortTid, lastSqlForThread)) + time_cost = DbConn.get_time_cost() + print("Last SQL statement attempted from thread {} ({:.4f} sec ago) is: {}".format(shortTid, time_cost ,lastSqlForThread)) stackFrame = 0 for frame in stack: # was using: reversed(stack) # print(frame) @@ -2631,4 +3326,3 @@ class Container(): return self._verifyValidProperty(name) self._cargo[name] = value - diff --git a/tests/pytest/crash_gen/shared/db.py b/tests/pytest/crash_gen/shared/db.py index 60c830f4f7..35ec9a5da6 100644 --- a/tests/pytest/crash_gen/shared/db.py +++ b/tests/pytest/crash_gen/shared/db.py @@ -26,9 +26,12 @@ class DbConn: TYPE_NATIVE = "native-c" TYPE_REST = "rest-api" TYPE_INVALID = "invalid" + + # class variables lastSqlFromThreads : dict[int, str] = {} # stored by thread id, obtained from threading.current_thread().ident%10000 + spendThreads : dict[int, float] = {} # stored by thread id, obtained from threading.current_thread().ident%10000 @classmethod def saveSqlForCurrentThread(cls, sql: str): @@ -37,15 +40,36 @@ class DbConn: run into a dead-lock situation, we can pick out the deadlocked thread, and use that information to find what what SQL statement is stuck. ''' + th = threading.current_thread() shortTid = th.native_id % 10000 #type: ignore cls.lastSqlFromThreads[shortTid] = sql # Save this for later @classmethod - def fetchSqlForThread(cls, shortTid : int) -> str : + def fetchSqlForThread(cls, shortTid : int) -> str : + + print("=======================") if shortTid not in cls.lastSqlFromThreads: raise CrashGenError("No last-attempted-SQL found for thread id: {}".format(shortTid)) - return cls.lastSqlFromThreads[shortTid] + return cls.lastSqlFromThreads[shortTid] + + + @classmethod + def sql_exec_spend(cls, cost: float): + ''' + Let us save the last SQL statement on a per-thread basis, so that when later we + run into a dead-lock situation, we can pick out the deadlocked thread, and use + that information to find what what SQL statement is stuck. + ''' + th = threading.current_thread() + shortTid = th.native_id % 10000 #type: ignore + cls.spendThreads[shortTid] = cost # Save this for later + + @classmethod + def get_time_cost(cls) ->float: + th = threading.current_thread() + shortTid = th.native_id % 10000 #type: ignore + return cls.spendThreads.get(shortTid) @classmethod def create(cls, connType, dbTarget): @@ -61,6 +85,7 @@ class DbConn: def createNative(cls, dbTarget) -> DbConn: return cls.create(cls.TYPE_NATIVE, dbTarget) + @classmethod def createRest(cls, dbTarget) -> DbConn: return cls.create(cls.TYPE_REST, dbTarget) @@ -75,6 +100,7 @@ class DbConn: return "[DbConn: type={}, target={}]".format(self._type, self._dbTarget) def getLastSql(self): + return self._lastSql def open(self): @@ -184,13 +210,19 @@ class DbConnRest(DbConn): def _doSql(self, sql): self._lastSql = sql # remember this, last SQL attempted self.saveSqlForCurrentThread(sql) # Save in global structure too. #TODO: combine with above - try: + time_cost = -1 + time_start = time.time() + try: r = requests.post(self._url, data = sql, - auth = HTTPBasicAuth('root', 'taosdata')) + auth = HTTPBasicAuth('root', 'taosdata')) except: print("REST API Failure (TODO: more info here)") + self.sql_exec_spend(-2) raise + finally: + time_cost = time.time()- time_start + self.sql_exec_spend(time_cost) rj = r.json() # Sanity check for the "Json Result" if ('status' not in rj): @@ -223,6 +255,8 @@ class DbConnRest(DbConn): "[SQL-REST] Execution Result, nRows = {}, SQL = {}".format(nRows, sql)) return nRows + + def query(self, sql): # return rows affected return self.execute(sql) @@ -336,6 +370,7 @@ class MyTDSql: raise return self.affectedRows + class DbTarget: def __init__(self, cfgPath, hostAddr, port): self.cfgPath = cfgPath @@ -355,6 +390,7 @@ class DbConnNative(DbConn): # _connInfoDisplayed = False # TODO: find another way to display this totalConnections = 0 # Not private totalRequests = 0 + time_cost = -1 def __init__(self, dbTarget): super().__init__(dbTarget) @@ -413,8 +449,19 @@ class DbConnNative(DbConn): "Cannot exec SQL unless db connection is open", CrashGenError.DB_CONNECTION_NOT_OPEN) Logging.debug("[SQL] Executing SQL: {}".format(sql)) self._lastSql = sql + time_cost = -1 + nRows = 0 + time_start = time.time() self.saveSqlForCurrentThread(sql) # Save in global structure too. #TODO: combine with above - nRows = self._tdSql.execute(sql) + try: + nRows= self._tdSql.execute(sql) + except Exception as e: + self.sql_exec_spend(-2) + finally: + time_cost = time.time() - time_start + self.sql_exec_spend(time_cost) + + cls = self.__class__ cls.totalRequests += 1 Logging.debug( @@ -494,4 +541,3 @@ class DbManager(): self._dbConn.close() self._dbConn = None Logging.debug("DbManager closed DB connection...") - From 30b1c576e134277b1af26a4639d9fb9b0de01b26 Mon Sep 17 00:00:00 2001 From: wenzhouwww Date: Wed, 9 Nov 2022 18:45:21 +0800 Subject: [PATCH 08/95] Update crash_gen_main.py --- tests/pytest/crash_gen/crash_gen_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 757ddf4af8..bb8de2ab4a 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -255,7 +255,7 @@ class WorkerThread: class ThreadCoordinator: - WORKER_THREAD_TIMEOUT = 1200 # Normal: 120 + WORKER_THREAD_TIMEOUT = 120 # Normal: 120 def __init__(self, pool: ThreadPool, dbManager: DbManager): self._curStep = -1 # first step is 0 From e1ca6a5e9c81fe7dc611b5780b0f670d33739eb9 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Wed, 9 Nov 2022 18:54:21 +0800 Subject: [PATCH 09/95] update --- tests/pytest/crash_gen/crash_gen_main.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index bb8de2ab4a..31b013d065 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1905,18 +1905,23 @@ class TaskCreateConsumers(StateTransitionTask): @classmethod def canBeginFrom(cls, state: AnyState): - return state.canCreateConsumers() + return state.canCreateConsumers() def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - dbname = self._db.getName() - - sTable = self._db.getFixedSuperTable() # type: TdSuperTable - # wt.execSql("use db") # should always be in place - # create Consumers - if Dice.throw(50)==0: # because subscribe is cost so much time , Reduce frequency of this task - if sTable.hasTopics(wt.getDbConn()): - sTable.createConsumer(wt.getDbConn(),random.randint(1,10)) + if Config.getConfig().connector_type == 'native': + dbname = self._db.getName() + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + + # create Consumers + if Dice.throw(50)==0: # because subscribe is cost so much time , Reduce frequency of this task + if sTable.hasTopics(wt.getDbConn()): + sTable.createConsumer(wt.getDbConn(),random.randint(1,10)) + else: + print(" restful not support tmq consumers") + return class TaskCreateSuperTable(StateTransitionTask): From f4aedc9ec2b4032aa191b88f04d925a94efa4dcd Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 10 Nov 2022 09:31:18 +0800 Subject: [PATCH 10/95] update tmq subscript --- tests/pytest/crash_gen/crash_gen_main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 31b013d065..18c3957630 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1374,6 +1374,7 @@ class Task(): 0x707, # Query not ready 0x396, # Database in creating status 0x386, # Database in droping status + 0x03E1, # failed on tmq_subscribe ,topic not exist 1000 # REST catch-all error @@ -2053,7 +2054,10 @@ class TdSuperTable: topic_list = TaosTmqList() for topic in current_topic_list: topic_list.append(topic) - consumer.subscribe(topic_list) + try: + consumer.subscribe(topic_list) + except TmqError as e : + pass time.sleep(5) # consumer work only 5 sec ,and then it will exit try: consumer.unsubscribe() From 119c4c7cab403af8755e136b81f566b6c5e099d6 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 10 Nov 2022 10:38:53 +0800 Subject: [PATCH 11/95] update --- tests/pytest/crash_gen/crash_gen_main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 18c3957630..2a9966eaf8 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1367,6 +1367,7 @@ class Task(): 0x2603, # Table does not exist, replaced by 2662 below 0x260d, # Tags number not matched 0x2662, # Table does not exist #TODO: what about 2603 above? + 0x2600, # database not specified, SQL: show stables , database droped , and show tables 0x032C, # Object is creating 0x032D, # Object is dropping 0x03D3, # Conflict transaction not completed @@ -1377,6 +1378,7 @@ class Task(): 0x03E1, # failed on tmq_subscribe ,topic not exist + 1000 # REST catch-all error ]: return True # These are the ALWAYS-ACCEPTABLE ones From e48d057f7048c28659b2cb26be91965b73987f16 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 10 Nov 2022 18:36:45 +0800 Subject: [PATCH 12/95] update --- tests/pytest/crash_gen/crash_gen_main.py | 212 ++++++++++++++++++----- tests/pytest/crash_gen/shared/db.py | 23 ++- 2 files changed, 192 insertions(+), 43 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 2a9966eaf8..ce0a306a9b 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -420,10 +420,12 @@ class ThreadCoordinator: except threading.BrokenBarrierError as err: self._execStats.registerFailure("Aborted due to worker thread timeout") Logging.error("\n") + Logging.error("Main loop aborted, caused by worker thread(s) time-out of {} seconds".format( ThreadCoordinator.WORKER_THREAD_TIMEOUT)) Logging.error("TAOS related threads blocked at (stack frames top-to-bottom):") ts = ThreadStacks() + ts.record_current_time(time.time()) # record thread exit time at current moment ts.print(filterInternal=True) workerTimeout = True @@ -678,7 +680,11 @@ class AnyState: CAN_CREATE_STREAM = 3 # super table must exists CAN_CREATE_TOPIC = 3 # super table must exists CAN_CREATE_CONSUMERS = 3 + CAN_CREATE_SMA = 3 + CAN_DROP_SMA = 3 CAN_DROP_FIXED_SUPER_TABLE = 4 + CAN_DROP_TOPIC = 4 + CAN_DROP_STREAM = 4 CAN_ADD_DATA = 5 CAN_READ_DATA = 6 CAN_DELETE_DATA = 6 @@ -734,13 +740,25 @@ class AnyState: def canCreateTopic(self): return self._info[self.CAN_CREATE_TOPIC] + + def canDropTopic(self): + return self._info[self.CAN_DROP_TOPIC] def canCreateConsumers(self): return self._info[self.CAN_CREATE_CONSUMERS] + + def canCreateSma(self): + return self._info[self.CAN_CREATE_SMA] + + def canDropSma(self): + return self._info[self.CAN_DROP_SMA] - def canCreateStream(self): + def canCreateStreams(self): return self._info[self.CAN_CREATE_STREAM] + def canDropStream(self): + return self._info[self.CAN_DROP_STREAM] + def canAddData(self): return self._info[self.CAN_ADD_DATA] @@ -919,7 +937,7 @@ class StateHasData(AnyState): ): # only if we didn't create one # we shouldn't have dropped it self.assertNoTask(tasks, TaskDropDb) - if (not self.hasTask(tasks, TaskCreateSuperTable) + if not( self.hasTask(tasks, TaskCreateSuperTable) ): # if we didn't create the table # we should not have a task that drops it self.assertNoTask(tasks, TaskDropSuperTable) @@ -1376,6 +1394,7 @@ class Task(): 0x396, # Database in creating status 0x386, # Database in droping status 0x03E1, # failed on tmq_subscribe ,topic not exist + 0x03ed , # Topic must be dropped first, SQL: drop database db_0 @@ -1694,19 +1713,20 @@ class TaskDropDb(StateTransitionTask): # drop topics before drop db - if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): + # if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): - self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) - self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),self._db.getFixedSuperTableName) + # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) + # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),self._db.getFixedSuperTableName) - self.execWtSql(wt, "drop database {}".format(self._db.getName())) + self.queryWtSql(wt, "drop database {}".format(self._db.getName())) # drop database maybe failed ,because topic exists Logging.debug("[OPS] database dropped at {}".format(time.time())) -''' + # Streams will generator TD-20237 （it will crash taosd , start this task when this issue fixed ） + class TaskCreateStream(StateTransitionTask): @classmethod @@ -1715,7 +1735,7 @@ class TaskCreateStream(StateTransitionTask): @classmethod def canBeginFrom(cls, state: AnyState): - return state.canCreateStream() + return state.canCreateStreams() def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): dbname = self._db.getName() @@ -1784,7 +1804,6 @@ class TaskCreateStream(StateTransitionTask): if errno in [0x03f0]: # stream already exists # stream need drop before drop table pass -''' class TaskCreateTopic(StateTransitionTask): @@ -1856,7 +1875,7 @@ class TaskCreateTopic(StateTransitionTask): Logging.debug("[OPS] topic is creating at {}".format(time.time())) except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) - if errno in [0x03f0]: # topic already exists + if errno in [0x03f0 ]: # topic already exists # topic need drop before drop table pass @@ -1878,7 +1897,7 @@ class TaskCreateTopic(StateTransitionTask): topic_sql = 'create topic {} AS STABLE {}.{} '.format(stable_topic,dbname,stbname) try: self.execWtSql(wt, "use {}".format(dbname)) - self.execWtSql(wt, topic_sql) + self.queryWtSql(wt, topic_sql) Logging.debug("[OPS] stable topic is creating at {}".format(time.time())) except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) @@ -1898,7 +1917,105 @@ class TaskCreateTopic(StateTransitionTask): pass else: pass - + +class TaskDropTopics(StateTransitionTask): + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canDropTopic() + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + dbname = self._db.getName() + + + if not self._db.exists(wt.getDbConn()): + Logging.debug("Skipping task, no DB yet") + return + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + tblName = sTable.getName() + if sTable.hasTopics(wt.getDbConn()): + sTable.dropTopics(wt.getDbConn(),dbname,None) # drop topics of database + sTable.dropTopics(wt.getDbConn(),dbname,tblName) # drop topics of stable + +class TaskCreateSma(StateTransitionTask): + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canCreateSma() + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + # dbname = self._db.getName() + + if not self._db.exists(wt.getDbConn()): + Logging.debug("Skipping task, no DB yet") + return + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + # tblName = sTable.getName() + if sTable.hasStreams(wt.getDbConn()): + sTable.dropStreams(wt.getDbConn()) # drop stream of database + # sTable.dropStreamTables(wt.getDbConn()) # drop streamtables of stable + +class TaskDropStreams(StateTransitionTask): + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canDropStream() + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + # dbname = self._db.getName() + + + if not self._db.exists(wt.getDbConn()): + Logging.debug("Skipping task, no DB yet") + return + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + # tblName = sTable.getName() + if sTable.hasStreams(wt.getDbConn()): + sTable.dropStreams(wt.getDbConn()) # drop stream of database + # sTable.dropStreamTables(wt.getDbConn()) # drop streamtables of stable + +class TaskDropStreamTables(StateTransitionTask): + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canDropStream() + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + # dbname = self._db.getName() + + + if not self._db.exists(wt.getDbConn()): + Logging.debug("Skipping task, no DB yet") + return + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + # tblName = sTable.getName() + if sTable.hasStreamTables(wt.getDbConn()): + # sTable.dropStreams(wt.getDbConn()) + sTable.dropStreamTables(wt.getDbConn()) # drop stream tables class TaskCreateConsumers(StateTransitionTask): @@ -1919,9 +2036,10 @@ class TaskCreateConsumers(StateTransitionTask): # wt.execSql("use db") # should always be in place # create Consumers - if Dice.throw(50)==0: # because subscribe is cost so much time , Reduce frequency of this task - if sTable.hasTopics(wt.getDbConn()): - sTable.createConsumer(wt.getDbConn(),random.randint(1,10)) + # if Dice.throw(50)==0: # because subscribe is cost so much time , Reduce frequency of this task + if sTable.hasTopics(wt.getDbConn()): + sTable.createConsumer(wt.getDbConn(),random.randint(1,10)) + pass else: print(" restful not support tmq consumers") return @@ -1969,17 +2087,17 @@ class TdSuperTable: dbName = self._dbName if self.exists(dbc) : # if myself exists fullTableName = dbName + '.' + self._stName - if self.hasStreams(dbc): - self.dropStreams(dbc) - self.dropStreamTables(dbc) - if self.hasTopics(dbc): - self.dropTopics(dbName,None) - self.dropTopics(dbName,self._stName) + # if self.hasStreams(dbc): + # self.dropStreams(dbc) + # self.dropStreamTables(dbc) + # if self.hasTopics(dbc): + # self.dropTopics(dbName,None) + # self.dropTopics(dbName,self._stName) try: dbc.execute("DROP TABLE {}".format(fullTableName)) except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) - if errno in [1011,0x3F3,0x03f3,0x2662]: # table doesn't exist + if errno in [1011,0x3F3,0x03f3,0x2662]: # table doesn't exist # Stream must be dropped first, SQL: DROP TABLE db_0.fs_table pass # # stream need drop before drop table # for stream in self.getStreamName(): @@ -2012,17 +2130,21 @@ class TdSuperTable: if dbc.existsSuperTable(self._stName): if dropIfExists: - if self.hasStreams(dbc): - self.dropStreams(dbc) - self.dropStreamTables(dbc) + # if self.hasStreams(dbc): + # self.dropStreams(dbc) + # self.dropStreamTables(dbc) - # drop topics before drop stables - if self.hasTopics(dbc): - self.dropTopics(dbc,self._dbName,None) - self.dropTopics(dbc,self._dbName,self._stName ) + # # drop topics before drop stables + # if self.hasTopics(dbc): + # self.dropTopics(dbc,self._dbName,None) + # self.dropTopics(dbc,self._dbName,self._stName ) - - dbc.execute("DROP TABLE {}".format(fullTableName)) + try: + dbc.execute("DROP TABLE {}".format(fullTableName)) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [1011,0x3F3,0x03f3,0x2662]: # table doesn't exist # Stream must be dropped first, SQL: DROP TABLE db_0.fs_table + pass pass # dbc.execute("DROP TABLE {}".format(fullTableName)) @@ -2125,6 +2247,7 @@ class TdSuperTable: if dbname in topic[0] and topic[0].startswith("database"): try: dbc.execute('drop topic {}'.format(topic[0])) + Logging.debug("[OPS] topic {} is droping at {}".format(topic,time.time())) except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) if errno in [0x03EB]: # Topic subscribed cannot be dropped @@ -2140,6 +2263,7 @@ class TdSuperTable: for topic in topics: if topic[0].startswith(self._dbName) and topic[0].endswith('topic'): dbc.execute('drop topic {}'.format(topic[0])) + Logging.debug("[OPS] topic {} is droping at {}".format(topic,time.time())) return True else: return True @@ -2455,15 +2579,16 @@ class TaskDropSuperTable(StateTransitionTask): # Drop the super table itself tblName = self._db.getFixedSuperTableName() - # drop streams before drop stables - if self._db.getFixedSuperTable().hasStreams(wt.getDbConn()): - self._db.getFixedSuperTable().dropStreams(wt.getDbConn()) - self._db.getFixedSuperTable().dropStreamTables(wt.getDbConn()) + # # drop streams before drop stables + # if self._db.getFixedSuperTable().hasStreams(wt.getDbConn()): + # self._db.getFixedSuperTable().dropStreams(wt.getDbConn()) + # self._db.getFixedSuperTable().dropStreamTables(wt.getDbConn()) + + # # drop topics before drop stables + # if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): + # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) + # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),tblName) - # drop topics before drop stables - if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): - self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) - self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),tblName) try: self.execWtSql(wt, "drop table {}.{}".format(self._db.getName(), tblName)) @@ -2949,6 +3074,9 @@ class ThreadStacks: # stack info for all threads shortTid = th.native_id % 10000 #type: ignore self._allStacks[shortTid] = stack # Was using th.native_id + def record_current_time(self,current_time): + self.current_time = current_time + def print(self, filteredEndName = None, filterInternal = False): for shortTid, stack in self._allStacks.items(): # for each thread, stack frames top to bottom lastFrame = stack[-1] @@ -2963,9 +3091,11 @@ class ThreadStacks: # stack info for all threads continue # ignore # Now print print("\n<----- Thread Info for LWP/ID: {} (most recent call last) <-----".format(shortTid)) + lastSqlForThread = DbConn.fetchSqlForThread(shortTid) - time_cost = DbConn.get_time_cost() - print("Last SQL statement attempted from thread {} ({:.4f} sec ago) is: {}".format(shortTid, time_cost ,lastSqlForThread)) + last_sql_commit_time = DbConn.get_save_sql_time(shortTid) + # time_cost = DbConn.get_time_cost() + print("Last SQL statement attempted from thread {} ({:.4f} sec ago) is: {}".format(shortTid, self.current_time-last_sql_commit_time ,lastSqlForThread)) stackFrame = 0 for frame in stack: # was using: reversed(stack) # print(frame) diff --git a/tests/pytest/crash_gen/shared/db.py b/tests/pytest/crash_gen/shared/db.py index 35ec9a5da6..05711efbc6 100644 --- a/tests/pytest/crash_gen/shared/db.py +++ b/tests/pytest/crash_gen/shared/db.py @@ -32,7 +32,7 @@ class DbConn: # class variables lastSqlFromThreads : dict[int, str] = {} # stored by thread id, obtained from threading.current_thread().ident%10000 spendThreads : dict[int, float] = {} # stored by thread id, obtained from threading.current_thread().ident%10000 - + current_time : dict[int, float] = {} # save current time @classmethod def saveSqlForCurrentThread(cls, sql: str): ''' @@ -44,6 +44,7 @@ class DbConn: th = threading.current_thread() shortTid = th.native_id % 10000 #type: ignore cls.lastSqlFromThreads[shortTid] = sql # Save this for later + cls.record_save_sql_time() @classmethod def fetchSqlForThread(cls, shortTid : int) -> str : @@ -53,6 +54,25 @@ class DbConn: raise CrashGenError("No last-attempted-SQL found for thread id: {}".format(shortTid)) return cls.lastSqlFromThreads[shortTid] + @classmethod + def get_save_sql_time(cls, shortTid : int): + ''' + Let us save the last SQL statement on a per-thread basis, so that when later we + run into a dead-lock situation, we can pick out the deadlocked thread, and use + that information to find what what SQL statement is stuck. + ''' + return cls.current_time[shortTid] + + @classmethod + def record_save_sql_time(cls): + ''' + Let us save the last SQL statement on a per-thread basis, so that when later we + run into a dead-lock situation, we can pick out the deadlocked thread, and use + that information to find what what SQL statement is stuck. + ''' + th = threading.current_thread() + shortTid = th.native_id % 10000 #type: ignore + cls.current_time[shortTid] = float(time.time()) # Save this for later @classmethod def sql_exec_spend(cls, cost: float): @@ -460,7 +480,6 @@ class DbConnNative(DbConn): finally: time_cost = time.time() - time_start self.sql_exec_spend(time_cost) - cls = self.__class__ cls.totalRequests += 1 From 0f8bb6faa7b67e360362efa55af32b51d0775ca0 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 10 Nov 2022 20:16:04 +0800 Subject: [PATCH 13/95] update enh crash_gen tasks --- tests/pytest/crash_gen/crash_gen_main.py | 63 +++++++++++------------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index ce0a306a9b..6780f12d96 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -549,7 +549,12 @@ class ThreadCoordinator: # pick a task type for current state db = self.pickDatabase() - taskType = db.getStateMachine().pickTaskType() # dynamic name of class + if Dice.throw(2)==1: + taskType = db.getStateMachine().pickTaskType() # dynamic name of class + else: + taskType = db.getStateMachine().balance_pickTaskType() # and an method can get balance task types + pass + return taskType(self._execStats, db) # create a task from it def resetExecutedTasks(self): @@ -680,8 +685,6 @@ class AnyState: CAN_CREATE_STREAM = 3 # super table must exists CAN_CREATE_TOPIC = 3 # super table must exists CAN_CREATE_CONSUMERS = 3 - CAN_CREATE_SMA = 3 - CAN_DROP_SMA = 3 CAN_DROP_FIXED_SUPER_TABLE = 4 CAN_DROP_TOPIC = 4 CAN_DROP_STREAM = 4 @@ -746,12 +749,6 @@ class AnyState: def canCreateConsumers(self): return self._info[self.CAN_CREATE_CONSUMERS] - - def canCreateSma(self): - return self._info[self.CAN_CREATE_SMA] - - def canDropSma(self): - return self._info[self.CAN_DROP_SMA] def canCreateStreams(self): return self._info[self.CAN_CREATE_STREAM] @@ -1093,6 +1090,28 @@ class StateMechine: # Logging.debug(" (weighted random:{}/{}) ".format(i, len(taskTypes))) return taskTypes[i] + def balance_pickTaskType(self): + # all the task types we can choose from at curent state + BasicTypes = self.getTaskTypes() + weightsTypes = BasicTypes.copy() + + # this matrixs can balance the Frequency of different types of tasks + weight_matrixs = {'TaskDropDb': 5 , 'TaskDropTopics': 20 , 'TaskDropStreams':10 , 'TaskDropStreamTables':10 , + 'TaskReadData':50 , 'TaskDropSuperTable':5 , 'TaskAlterTags':3 , 'TaskAddData':10, + 'TaskDeleteData':10 , 'TaskCreateDb':10 , 'TaskCreateStream': 3, 'TaskCreateTopic' :3, + 'TaskCreateConsumers':10, 'TaskCreateSuperTable': 10 } # task type : weghts matrixs + + for task , weights in weight_matrixs.items(): + + for basicType in BasicTypes: + if basicType.__name__ == task: + for _ in range(weights): + weightsTypes.append(basicType) + + task = random.sample(weightsTypes,1) + return task[0] + + # ref: # https://eli.thegreenplace.net/2010/01/22/weighted-random-generation-in-python/ def _weighted_choice_sub(self, weights) -> int: @@ -1395,6 +1414,8 @@ class Task(): 0x386, # Database in droping status 0x03E1, # failed on tmq_subscribe ,topic not exist 0x03ed , # Topic must be dropped first, SQL: drop database db_0 + 0x0203 , # Invalid value + @@ -1942,30 +1963,6 @@ class TaskDropTopics(StateTransitionTask): if sTable.hasTopics(wt.getDbConn()): sTable.dropTopics(wt.getDbConn(),dbname,None) # drop topics of database sTable.dropTopics(wt.getDbConn(),dbname,tblName) # drop topics of stable - -class TaskCreateSma(StateTransitionTask): - - @classmethod - def getEndState(cls): - return StateHasData() - - @classmethod - def canBeginFrom(cls, state: AnyState): - return state.canCreateSma() - - def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - # dbname = self._db.getName() - - if not self._db.exists(wt.getDbConn()): - Logging.debug("Skipping task, no DB yet") - return - - sTable = self._db.getFixedSuperTable() # type: TdSuperTable - # wt.execSql("use db") # should always be in place - # tblName = sTable.getName() - if sTable.hasStreams(wt.getDbConn()): - sTable.dropStreams(wt.getDbConn()) # drop stream of database - # sTable.dropStreamTables(wt.getDbConn()) # drop streamtables of stable class TaskDropStreams(StateTransitionTask): From 6175929113777d50ded7b7d282424e99e05a9981 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 11 Nov 2022 13:50:37 +0800 Subject: [PATCH 14/95] update --- tests/pytest/crash_gen/crash_gen_main.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 6780f12d96..4fa19f543d 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -953,7 +953,7 @@ class StateMechine: except taos.error.ProgrammingError as err: Logging.error("Failed to initialized state machine, cannot find current state: {}".format(err)) traceback.print_stack() - pass # re-throw + raise # re-throw # TODO: seems no lnoger used, remove? def getCurrentState(self): @@ -1095,13 +1095,13 @@ class StateMechine: BasicTypes = self.getTaskTypes() weightsTypes = BasicTypes.copy() - # this matrixs can balance the Frequency of different types of tasks - weight_matrixs = {'TaskDropDb': 5 , 'TaskDropTopics': 20 , 'TaskDropStreams':10 , 'TaskDropStreamTables':10 , + # this matrixs can balance the Frequency of TaskTypes + balance_TaskType_matrixs = {'TaskDropDb': 5 , 'TaskDropTopics': 20 , 'TaskDropStreams':10 , 'TaskDropStreamTables':10 , 'TaskReadData':50 , 'TaskDropSuperTable':5 , 'TaskAlterTags':3 , 'TaskAddData':10, 'TaskDeleteData':10 , 'TaskCreateDb':10 , 'TaskCreateStream': 3, 'TaskCreateTopic' :3, - 'TaskCreateConsumers':10, 'TaskCreateSuperTable': 10 } # task type : weghts matrixs + 'TaskCreateConsumers':10, 'TaskCreateSuperTable': 10 } # TaskType : balance_matrixs of task - for task , weights in weight_matrixs.items(): + for task , weights in balance_TaskType_matrixs.items(): for basicType in BasicTypes: if basicType.__name__ == task: @@ -1738,8 +1738,12 @@ class TaskDropDb(StateTransitionTask): # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),self._db.getFixedSuperTableName) - - self.queryWtSql(wt, "drop database {}".format(self._db.getName())) # drop database maybe failed ,because topic exists + try: + self.queryWtSql(wt, "drop database {}".format(self._db.getName())) # drop database maybe failed ,because topic exists + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x0203]: # drop maybe failed + pass Logging.debug("[OPS] database dropped at {}".format(time.time())) From b47052b7afecd4bec7c7e46a6e68633c4307e6a9 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Tue, 15 Nov 2022 09:12:42 +0800 Subject: [PATCH 15/95] update --- tests/pytest/crash_gen/crash_gen_main.py | 244 +++++------------------ 1 file changed, 48 insertions(+), 196 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 4fa19f543d..90d27c3b8d 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1415,6 +1415,7 @@ class Task(): 0x03E1, # failed on tmq_subscribe ,topic not exist 0x03ed , # Topic must be dropped first, SQL: drop database db_0 0x0203 , # Invalid value + 0x03f0 , # Stream already exist , topic already exists @@ -1732,12 +1733,6 @@ class TaskDropDb(StateTransitionTask): def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - # drop topics before drop db - - # if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): - - # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) - # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),self._db.getFixedSuperTableName) try: self.queryWtSql(wt, "drop database {}".format(self._db.getName())) # drop database maybe failed ,because topic exists except taos.error.ProgrammingError as err: @@ -1748,10 +1743,6 @@ class TaskDropDb(StateTransitionTask): Logging.debug("[OPS] database dropped at {}".format(time.time())) - -# Streams will generator TD-20237 （it will crash taosd , start this task when this issue fixed ） - - class TaskCreateStream(StateTransitionTask): @classmethod @@ -1775,60 +1766,25 @@ class TaskCreateStream(StateTransitionTask): sTable = self._db.getFixedSuperTable() # type: TdSuperTable # wt.execSql("use db") # should always be in place - - # create stream - - # CREATE STREAM avg_vol_s INTO avg_vol AS SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); - stbname =sTable.getName() sub_tables = sTable.getRegTables(wt.getDbConn()) aggExpr = Dice.choice([ - 'count(*)', - 'avg(speed)', - # 'twa(speed)', # TODO: this one REQUIRES a where statement, not reasonable - 'sum(speed)', - 'stddev(speed)', - # SELECTOR functions - 'min(speed)', - 'max(speed)', - 'first(speed)', - 'last(speed)', - 'apercentile(speed, 10)', # TODO: TD-1316 - 'last_row(*)', # TODO: commented out per TD-3231, we should re-create - # Transformation Functions - 'twa(speed)' - - ]) # TODO: add more from 'top' + 'count(*)', 'avg(speed)', 'sum(speed)', 'stddev(speed)','min(speed)', 'max(speed)', 'first(speed)', 'last(speed)', + 'apercentile(speed, 10)', 'last_row(*)', 'twa(speed)']) + + stream_sql = '' # set default value if sub_tables: - - if sub_tables: # if not empty - sub_tbname = sub_tables[0] - # create stream with query above sub_table - stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(sub_stream_name,dbname,sub_stream_tb_name ,aggExpr,dbname,sub_tbname) - try: - self.execWtSql(wt, stream_sql) - Logging.debug("[OPS] stream is creating at {}".format(time.time())) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [0x03f0]: # stream already exists - # stream need drop before drop table - pass - - else: - pass - + sub_tbname = sub_tables[0] + # create stream with query above sub_table + stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.\ + format(sub_stream_name,dbname,sub_stream_tb_name ,aggExpr,dbname,sub_tbname) else: - stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(super_stream_name,dbname,super_stream_tb_name,aggExpr, dbname,stbname) + stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.\ + format(super_stream_name,dbname,super_stream_tb_name,aggExpr, dbname,stbname) + self.execWtSql(wt, stream_sql) + Logging.debug("[OPS] stream is creating at {}".format(time.time())) - try: - self.execWtSql(wt, stream_sql) - Logging.debug("[OPS] stream is creating at {}".format(time.time())) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [0x03f0]: # stream already exists - # stream need drop before drop table - pass class TaskCreateTopic(StateTransitionTask): @@ -1853,96 +1809,37 @@ class TaskCreateTopic(StateTransitionTask): sTable = self._db.getFixedSuperTable() # type: TdSuperTable # wt.execSql("use db") # should always be in place - - # create topic - # create topic if not exists topic_ctb_column as select ts, c1, c2, c3 from stb1; stbname =sTable.getName() sub_tables = sTable.getRegTables(wt.getDbConn()) - scalarExpr = Dice.choice([ '*','speed','color', - 'abs(speed)', - 'acos(speed)', - 'asin(speed)', - 'atan(speed)', - 'ceil(speed)', - 'cos(speed)', - 'cos(speed)', - 'floor(speed)', - 'log(speed,2)', - 'pow(speed,2)', - 'round(speed)', - 'sin(speed)', - 'sqrt(speed)', - 'char_length(color)', - 'concat(color,color)', - 'concat_ws(" ", color,color," ")', - 'length(color)', - 'lower(color)', - 'ltrim(color)', - 'substr(color , 2)', - 'upper(color)', - 'cast(speed as double)', - 'cast(ts as bigint)', - - ]) # TODO: add more from 'top' - if Dice.throw(3)==0: - if sub_tables: - - if sub_tables: # if not empty - sub_tbname = sub_tables[0] - # create stream with query above sub_table - topic_sql = 'create topic {} as select {} FROM {}.{} ; '.format(sub_topic_name,scalarExpr,dbname,sub_tbname) - try: - self.execWtSql(wt, "use {}".format(dbname)) - self.execWtSql(wt, topic_sql) - Logging.debug("[OPS] topic is creating at {}".format(time.time())) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [0x03f0 ]: # topic already exists - # topic need drop before drop table - pass - - else: - pass - - else: + scalarExpr = Dice.choice([ '*','speed','color','abs(speed)','acos(speed)','asin(speed)','atan(speed)','ceil(speed)','cos(speed)','cos(speed)', + 'floor(speed)','log(speed,2)','pow(speed,2)','round(speed)','sin(speed)','sqrt(speed)','char_length(color)','concat(color,color)', + 'concat_ws(" ", color,color," ")','length(color)', 'lower(color)', 'ltrim(color)','substr(color , 2)','upper(color)','cast(speed as double)', + 'cast(ts as bigint)']) + topic_sql = '' # set default value + if Dice.throw(3)==0: # create topic : source data from sub query + if sub_tables: # if not empty + sub_tbname = sub_tables[0] + # create topic : source data from sub query of sub stable + topic_sql = 'create topic {} as select {} FROM {}.{} ; '.format(sub_topic_name,scalarExpr,dbname,sub_tbname) + + else: # create topic : source data from sub query of stable topic_sql = 'create topic {} as select {} FROM {}.{} '.format(super_topic_name,scalarExpr, dbname,stbname) - try: - self.execWtSql(wt, "use {}".format(dbname)) - self.execWtSql(wt, topic_sql) - Logging.debug("[OPS] subquery topic is creating at {}".format(time.time())) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [0x03f0]: # topic already exists - # topic need drop before drop table - pass - elif Dice.throw(3)==1: + elif Dice.throw(3)==1: # create topic : source data from super table topic_sql = 'create topic {} AS STABLE {}.{} '.format(stable_topic,dbname,stbname) - try: - self.execWtSql(wt, "use {}".format(dbname)) - self.queryWtSql(wt, topic_sql) - Logging.debug("[OPS] stable topic is creating at {}".format(time.time())) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [0x03f0]: # topic already exists - # topic need drop before drop table - pass - elif Dice.throw(3)==2: - topic_sql = 'create topic {} AS DATABASE {} '.format(db_topic,dbname) - try: - self.execWtSql(wt, "use {}".format(dbname)) - self.execWtSql(wt, topic_sql) - Logging.debug("[OPS] db topic is creating at {}".format(time.time())) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [0x03f0]: # topic already exists - # topic need drop before drop table - pass + + elif Dice.throw(3)==2: # create topic : source data from whole database + topic_sql = 'create topic {} AS DATABASE {} '.format(db_topic,dbname) else: pass + # exec create topics + self.execWtSql(wt, "use {}".format(dbname)) + self.execWtSql(wt, topic_sql) + Logging.debug("[OPS] db topic is creating at {}".format(time.time())) + class TaskDropTopics(StateTransitionTask): @classmethod @@ -1991,7 +1888,6 @@ class TaskDropStreams(StateTransitionTask): # tblName = sTable.getName() if sTable.hasStreams(wt.getDbConn()): sTable.dropStreams(wt.getDbConn()) # drop stream of database - # sTable.dropStreamTables(wt.getDbConn()) # drop streamtables of stable class TaskDropStreamTables(StateTransitionTask): @@ -2012,10 +1908,9 @@ class TaskDropStreamTables(StateTransitionTask): return sTable = self._db.getFixedSuperTable() # type: TdSuperTable - # wt.execSql("use db") # should always be in place + wt.execSql("use db") # should always be in place # tblName = sTable.getName() if sTable.hasStreamTables(wt.getDbConn()): - # sTable.dropStreams(wt.getDbConn()) sTable.dropStreamTables(wt.getDbConn()) # drop stream tables class TaskCreateConsumers(StateTransitionTask): @@ -2031,13 +1926,9 @@ class TaskCreateConsumers(StateTransitionTask): def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): if Config.getConfig().connector_type == 'native': - dbname = self._db.getName() - + sTable = self._db.getFixedSuperTable() # type: TdSuperTable # wt.execSql("use db") # should always be in place - - # create Consumers - # if Dice.throw(50)==0: # because subscribe is cost so much time , Reduce frequency of this task if sTable.hasTopics(wt.getDbConn()): sTable.createConsumer(wt.getDbConn(),random.randint(1,10)) pass @@ -2088,30 +1979,13 @@ class TdSuperTable: dbName = self._dbName if self.exists(dbc) : # if myself exists fullTableName = dbName + '.' + self._stName - # if self.hasStreams(dbc): - # self.dropStreams(dbc) - # self.dropStreamTables(dbc) - # if self.hasTopics(dbc): - # self.dropTopics(dbName,None) - # self.dropTopics(dbName,self._stName) + try: dbc.execute("DROP TABLE {}".format(fullTableName)) except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) if errno in [1011,0x3F3,0x03f3,0x2662]: # table doesn't exist # Stream must be dropped first, SQL: DROP TABLE db_0.fs_table pass - # # stream need drop before drop table - # for stream in self.getStreamName(): - # drop_stream_sql = 'drop stream {}'.format(stream) - # try: - # dbc.execute(drop_stream_sql) - # except taos.error.ProgrammingError as err: - # # correcting for strange error number scheme - # errno3 = Helper.convertErrno(err.errno) - # if errno3 in [1011,0x3F3,0x03f3,0x2662,0x03f1]: # stream not exists - # pass - # dbc.execute("DROP TABLE {}".format(fullTableName)) - # pass else: if not skipCheck: @@ -2131,15 +2005,6 @@ class TdSuperTable: if dbc.existsSuperTable(self._stName): if dropIfExists: - # if self.hasStreams(dbc): - # self.dropStreams(dbc) - # self.dropStreamTables(dbc) - - # # drop topics before drop stables - # if self.hasTopics(dbc): - # self.dropTopics(dbc,self._dbName,None) - # self.dropTopics(dbc,self._dbName,self._stName ) - try: dbc.execute("DROP TABLE {}".format(fullTableName)) except taos.error.ProgrammingError as err: @@ -2164,7 +2029,7 @@ class TdSuperTable: sql += " TAGS (dummy int) " dbc.execute(sql) - def createConsumer(self, dbc: DbConn , Consumer_nums): + def createConsumer(self, dbc,Consumer_nums): def generateConsumer(current_topic_list): conf = TaosTmqConf() @@ -2183,7 +2048,14 @@ class TdSuperTable: consumer.subscribe(topic_list) except TmqError as e : pass - time.sleep(5) # consumer work only 5 sec ,and then it will exit + + # consumer work only 30 sec + time_start = time.time() + while 1: + res = consumer.poll(1000) + if time.time() - time_start >5 : + break + # time.sleep(10) try: consumer.unsubscribe() except TmqError as e : @@ -2579,29 +2451,9 @@ class TaskDropSuperTable(StateTransitionTask): # Drop the super table itself tblName = self._db.getFixedSuperTableName() - - # # drop streams before drop stables - # if self._db.getFixedSuperTable().hasStreams(wt.getDbConn()): - # self._db.getFixedSuperTable().dropStreams(wt.getDbConn()) - # self._db.getFixedSuperTable().dropStreamTables(wt.getDbConn()) + self.execWtSql(wt, "drop table {}.{}".format(self._db.getName(), tblName)) + - # # drop topics before drop stables - # if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): - # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) - # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),tblName) - - - try: - self.execWtSql(wt, "drop table {}.{}".format(self._db.getName(), tblName)) - except taos.error.ProgrammingError as err: - # correcting for strange error number scheme - errno2 = Helper.convertErrno(err.errno) - if (errno2 in [0x362]): # mnode invalid table name - isSuccess = False - Logging.debug("[DB] Acceptable error when dropping a table") - elif errno2 in [1011,0x3F3,0x03f3]: # table doesn't exist - - pass class TaskAlterTags(StateTransitionTask): From 8b82fdf0e8b48747e0d54994d4cab30db4e445bc Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 16 Nov 2022 16:00:37 +0800 Subject: [PATCH 16/95] fix(query): correct calc sort page size --- include/util/tdef.h | 3 +++ source/libs/executor/inc/tsort.h | 3 ++- source/libs/executor/src/scanoperator.c | 3 ++- source/libs/executor/src/sortoperator.c | 3 ++- source/libs/executor/src/tsort.c | 17 +++++++---------- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 556ad6b680..48dedd3e3e 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -489,6 +489,9 @@ enum { #define MAX_META_MSG_IN_BATCH 1048576 #define MAX_META_BATCH_RSP_SIZE (1 * 1048576 * 1024) +// sort page size by default +#define DEFAULT_PAGESIZE 4096 + #ifdef __cplusplus } #endif diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h index 5591299d30..51440a7f59 100644 --- a/source/libs/executor/inc/tsort.h +++ b/source/libs/executor/inc/tsort.h @@ -163,9 +163,10 @@ SSortExecInfo tsortGetSortExecInfo(SSortHandle* pHandle); /** * get proper sort buffer pages according to the row size * @param rowSize + * @param numOfCols columns count that be put into page * @return */ -int32_t getProperSortPageSize(size_t rowSize); +int32_t getProperSortPageSize(size_t rowSize, uint32_t numOfCols); #ifdef __cplusplus } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 77d2aeba28..145f236040 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -4822,7 +4822,8 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN initLimitInfo(pTableScanNode->scan.node.pLimit, pTableScanNode->scan.node.pSlimit, &pInfo->limitInfo); int32_t rowSize = pInfo->pResBlock->info.rowSize; - pInfo->bufPageSize = getProperSortPageSize(rowSize); + uint32_t numOfCols = taosArrayGetSize(pInfo->pResBlock->pDataBlock); + pInfo->bufPageSize = getProperSortPageSize(rowSize, numOfCols); setOperatorInfo(pOperator, "TableMergeScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index fc53623d44..add580ce7c 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -760,7 +760,8 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size pInfo->groupSort = pMergePhyNode->groupSort; pInfo->pSortInfo = createSortInfo(pMergePhyNode->pMergeKeys); pInfo->pInputBlock = pInputBlock; - pInfo->bufPageSize = getProperSortPageSize(rowSize); + size_t numOfCols = taosArrayGetSize(pInfo->binfo.pRes->pDataBlock); + pInfo->bufPageSize = getProperSortPageSize(rowSize, numOfCols); pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1); // one additional is reserved for merged result. setOperatorInfo(pOperator, "MultiwayMergeOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE, false, OP_NOT_OPENED, pInfo, pTaskInfo); diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 71fad2e27c..9c10b51b1f 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -584,15 +584,11 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { return 0; } -// TODO consider the page meta size -int32_t getProperSortPageSize(size_t rowSize) { - uint32_t defaultPageSize = 4096; - - uint32_t pgSize = 0; - if (rowSize * 4 > defaultPageSize) { - pgSize = rowSize * 4; - } else { - pgSize = defaultPageSize; +// get sort page size +int32_t getProperSortPageSize(size_t rowSize, uint32_t numOfCols) { + uint32_t pgSize = rowSize * 4 + blockDataGetSerialMetaSize(numOfCols); + if (pgSize < DEFAULT_PAGESIZE) { + return DEFAULT_PAGESIZE; } return pgSize; @@ -612,7 +608,8 @@ static int32_t createInitialSources(SSortHandle* pHandle) { } if (pHandle->pDataBlock == NULL) { - pHandle->pageSize = getProperSortPageSize(blockDataGetRowSize(pBlock)); + uint32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + pHandle->pageSize = getProperSortPageSize(blockDataGetRowSize(pBlock), numOfCols); // todo, number of pages are set according to the total available sort buffer pHandle->numOfPages = 1024; From 28c692b52f56f3a4448e2cbe00dabce30c988fe1 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 16 Nov 2022 16:10:22 +0800 Subject: [PATCH 17/95] fix(query): fixed build error --- source/libs/executor/src/scanoperator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 145f236040..108fbb7a82 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -4822,8 +4822,8 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN initLimitInfo(pTableScanNode->scan.node.pLimit, pTableScanNode->scan.node.pSlimit, &pInfo->limitInfo); int32_t rowSize = pInfo->pResBlock->info.rowSize; - uint32_t numOfCols = taosArrayGetSize(pInfo->pResBlock->pDataBlock); - pInfo->bufPageSize = getProperSortPageSize(rowSize, numOfCols); + uint32_t nCols = taosArrayGetSize(pInfo->pResBlock->pDataBlock); + pInfo->bufPageSize = getProperSortPageSize(rowSize, nCols); setOperatorInfo(pOperator, "TableMergeScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); From cb0f1f537e4d81c6b66574cc9b0ddeb004711da2 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 16 Nov 2022 17:03:04 +0800 Subject: [PATCH 18/95] enh: fill value supports implicit type conversion --- source/libs/parser/src/parTranslater.c | 22 +++++++++++----------- tests/script/tsim/parser/fill_stb.sim | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 9883c81819..c82634b400 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2762,17 +2762,17 @@ static bool needFill(SNode* pNode) { return hasFillFunc; } -static bool mismatchFillDataType(SDataType origDt, SDataType fillDt) { - if (TSDB_DATA_TYPE_NULL == fillDt.type) { - return false; +static int32_t convertFillValue(STranslateContext* pCxt, SDataType dt, SNodeList* pValues, int32_t index) { + SListCell* pCell = nodesListGetCell(pValues, index); + if (dataTypeEqual(&dt, &((SExprNode*)pCell->pNode)->resType)) { + return TSDB_CODE_SUCCESS; } - if (IS_NUMERIC_TYPE(origDt.type) && !IS_NUMERIC_TYPE(fillDt.type)) { - return true; + SNode* pCaseFunc = NULL; + int32_t code = createCastFunc(pCxt, pCell->pNode, dt, &pCaseFunc); + if (TSDB_CODE_SUCCESS == code) { + pCell->pNode = pCaseFunc; } - if (IS_VAR_DATA_TYPE(origDt.type) && !IS_VAR_DATA_TYPE(fillDt.type)) { - return true; - } - return false; + return code; } static int32_t checkFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeList* pProjectionList) { @@ -2788,8 +2788,8 @@ static int32_t checkFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeL if (fillNo >= LIST_LENGTH(pFillValues->pNodeList)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled values number mismatch"); } - if (mismatchFillDataType(((SExprNode*)pProject)->resType, - ((SExprNode*)nodesListGetNode(pFillValues->pNodeList, fillNo))->resType)) { + if (TSDB_CODE_SUCCESS != + convertFillValue(pCxt, ((SExprNode*)pProject)->resType, pFillValues->pNodeList, fillNo)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled data type mismatch"); } ++fillNo; diff --git a/tests/script/tsim/parser/fill_stb.sim b/tests/script/tsim/parser/fill_stb.sim index 66787b3606..b6cc2f38bd 100644 --- a/tests/script/tsim/parser/fill_stb.sim +++ b/tests/script/tsim/parser/fill_stb.sim @@ -354,7 +354,7 @@ endi ## NULL fill print fill(NULL) -print select _wstart, max(c1), min(c2), avg(c3), sum(c4), count(c5), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 4 partition by t1 interval(5m) fill(value, NULL) limit 5 +print select _wstart, max(c1), min(c2), avg(c3), sum(c4), count(c5), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 4 partition by t1 interval(5m) fill(value, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) limit 5 sql select _wstart, max(c1), min(c2), avg(c3), sum(c4), count(c5), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 4 partition by t1 interval(5m) fill(NULL) limit 5 if $rows != 25 then return -1 From 6515c9c7c524e59556288dbfe527ecf68b5a0842 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 17 Nov 2022 10:56:24 +0800 Subject: [PATCH 19/95] fix: real path optimization in case of invalid read --- source/dnode/vnode/src/tsdb/tsdbOpen.c | 2 +- source/os/src/osDir.c | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index bb20a9b012..f71b5b6706 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -49,7 +49,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee pTsdb->path = (char *)&pTsdb[1]; snprintf(pTsdb->path, TD_PATH_MAX, "%s%s%s", pVnode->path, TD_DIRSEP, dir); - taosRealPath(pTsdb->path, NULL, slen); + // taosRealPath(pTsdb->path, NULL, slen); pTsdb->pVnode = pVnode; taosThreadRwlockInit(&pTsdb->rwLock, NULL); if (!pKeepCfg) { diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 5e85e40594..421901184b 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -336,12 +336,14 @@ int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen) { #else if (realpath(dirname, tmp) != NULL) { #endif - if (realPath == NULL) { - strncpy(dirname, tmp, maxlen); - } else { - strncpy(realPath, tmp, maxlen); + if (strlen(tmp) < maxlen) { + if (realPath == NULL) { + strncpy(dirname, tmp, maxlen); + } else { + strncpy(realPath, tmp, maxlen); + } + return 0; } - return 0; } return -1; From 031ceda752a2a6e531e6e621454ffbb59e25fc55 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 17 Nov 2022 14:58:53 +0800 Subject: [PATCH 20/95] createOneDataBlock rowsize must reset to zero --- source/common/src/tdatablock.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index f7b1196248..3c2a5377e3 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1403,6 +1403,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { pBlock->info = pDataBlock->info; pBlock->info.rows = 0; pBlock->info.capacity = 0; + pBlock->info.rowSize = 0; size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { From dc0d3745a7df4e3606fa11f75c240130fa16a893 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 17 Nov 2022 15:48:42 +0800 Subject: [PATCH 21/95] enh: fill value supports implicit type conversion --- include/common/tvariant.h | 3 +- source/common/src/tvariant.c | 13 ++++-- source/libs/function/src/builtinsimpl.c | 43 +++++++++++-------- source/libs/nodes/src/nodesUtilFuncs.c | 2 + source/libs/parser/src/parTranslater.c | 2 +- tests/script/tsim/parser/fill.sim | 18 ++++---- tests/script/tsim/parser/fill_stb.sim | 8 ++-- tests/script/tsim/parser/fill_us.sim | 18 ++++---- .../script/tsim/stream/fillIntervalValue.sim | 2 +- 9 files changed, 62 insertions(+), 47 deletions(-) diff --git a/include/common/tvariant.h b/include/common/tvariant.h index 0507934e6a..130945cce5 100644 --- a/include/common/tvariant.h +++ b/include/common/tvariant.h @@ -30,6 +30,7 @@ typedef struct SVariant { int64_t i; uint64_t u; double d; + float f; char *pz; TdUcs4 *ucs4; SArray *arr; // only for 'in' query to hold value list, not value for a field @@ -47,7 +48,7 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc); int32_t taosVariantCompare(const SVariant *p1, const SVariant *p2); -char *taosVariantGet(SVariant *pVar, int32_t type); +char *taosVariantGet(SVariant *pVar, int32_t type); #ifdef __cplusplus } diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 8d4c17a821..65e9a767f5 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -109,7 +109,7 @@ void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uin } case TSDB_DATA_TYPE_FLOAT: { pVar->nLen = tDataTypes[type].bytes; - pVar->d = GET_FLOAT_VAL(pz); + pVar->f = GET_FLOAT_VAL(pz); break; } case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length @@ -223,12 +223,18 @@ int32_t taosVariantCompare(const SVariant *p1, const SVariant *p2) { } else { return p1->nLen > p2->nLen ? 1 : -1; } - } else if (p1->nType == TSDB_DATA_TYPE_FLOAT || p1->nType == TSDB_DATA_TYPE_DOUBLE) { + } else if (p1->nType == TSDB_DATA_TYPE_DOUBLE) { if (p1->d == p2->d) { return 0; } else { return p1->d > p2->d ? 1 : -1; } + } else if (p1->nType == TSDB_DATA_TYPE_FLOAT) { + if (p1->f == p2->f) { + return 0; + } else { + return p1->f > p2->f ? 1 : -1; + } } else if (IS_UNSIGNED_NUMERIC_TYPE(p1->nType)) { if (p1->u == p2->u) { return 0; @@ -259,8 +265,9 @@ char *taosVariantGet(SVariant *pVar, int32_t type) { case TSDB_DATA_TYPE_UBIGINT: return (char *)&pVar->u; case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_FLOAT: return (char *)&pVar->d; + case TSDB_DATA_TYPE_FLOAT: + return (char *)&pVar->f; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_JSON: return (char *)pVar->pz; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 10860d291f..f69a76623c 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -48,8 +48,8 @@ typedef struct SSumRes { double dsum; }; int16_t type; - int64_t prevTs; // used for csum only - bool isPrevTsSet; //used for csum only + int64_t prevTs; // used for csum only + bool isPrevTsSet; // used for csum only } SSumRes; @@ -2217,8 +2217,8 @@ bool leastSQRFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInf SLeastSQRInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo); - pInfo->startVal = IS_FLOAT_TYPE(pCtx->param[1].param.nType) ? pCtx->param[1].param.d : (double)pCtx->param[1].param.i; - pInfo->stepVal = IS_FLOAT_TYPE(pCtx->param[2].param.nType) ? pCtx->param[2].param.d : (double)pCtx->param[2].param.i; + GET_TYPED_DATA(pInfo->startVal, double, pCtx->param[1].param.nType, pCtx->param[1].param.i); + GET_TYPED_DATA(pInfo->stepVal, double, pCtx->param[2].param.nType, pCtx->param[2].param.i); return true; } @@ -2562,8 +2562,8 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SVariant* pVal = &pCtx->param[1].param; - double v = - (IS_SIGNED_NUMERIC_TYPE(pVal->nType) ? pVal->i : (IS_UNSIGNED_NUMERIC_TYPE(pVal->nType) ? pVal->u : pVal->d)); + double v = 0; + GET_TYPED_DATA(v, double, pVal->nType, pVal->i); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SPercentileInfo* ppInfo = (SPercentileInfo*)GET_ROWCELL_INTERBUF(pResInfo); @@ -2622,8 +2622,8 @@ bool apercentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResult SAPercentileInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo); SVariant* pVal = &pCtx->param[1].param; - pInfo->percent = - (IS_SIGNED_NUMERIC_TYPE(pVal->nType) ? pVal->i : (IS_UNSIGNED_NUMERIC_TYPE(pVal->nType) ? pVal->u : pVal->d)); + pInfo->percent = 0; + GET_TYPED_DATA(pInfo->percent, double, pVal->nType, pVal->i); if (pCtx->numOfParams == 2) { pInfo->algo = APERCT_ALGO_DEFAULT; @@ -3717,6 +3717,12 @@ static int32_t topBotResComparFn(const void* p1, const void* p2, const void* par } return (val1->v.u > val2->v.u) ? 1 : -1; + } else if (TSDB_DATA_TYPE_FLOAT == type) { + if (val1->v.f == val2->v.f) { + return 0; + } + + return (val1->v.f > val2->v.f) ? 1 : -1; } if (val1->v.d == val2->v.d) { @@ -3757,10 +3763,12 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData } else { // replace the minimum value in the result if ((isTopQuery && ((IS_SIGNED_NUMERIC_TYPE(type) && val.i > pItems[0].v.i) || (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u > pItems[0].v.u) || - (IS_FLOAT_TYPE(type) && val.d > pItems[0].v.d))) || + (TSDB_DATA_TYPE_FLOAT == type && val.f > pItems[0].v.f) || + (TSDB_DATA_TYPE_DOUBLE == type && val.d > pItems[0].v.d))) || (!isTopQuery && ((IS_SIGNED_NUMERIC_TYPE(type) && val.i < pItems[0].v.i) || (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u < pItems[0].v.u) || - (IS_FLOAT_TYPE(type) && val.d < pItems[0].v.d)))) { + (TSDB_DATA_TYPE_FLOAT == type && val.f < pItems[0].v.f) || + (TSDB_DATA_TYPE_DOUBLE == type && val.d < pItems[0].v.d)))) { // replace the old data and the coresponding tuple data STopBotResItem* pItem = &pItems[0]; pItem->v = val; @@ -3923,12 +3931,7 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) { STopBotResItem* pItem = &pRes->pItems[i]; - if (type == TSDB_DATA_TYPE_FLOAT) { - float v = pItem->v.d; - colDataAppend(pCol, currentRow, (const char*)&v, false); - } else { - colDataAppend(pCol, currentRow, (const char*)&pItem->v.i, false); - } + colDataAppend(pCol, currentRow, (const char*)&pItem->v.i, false); #ifdef BUF_PAGE_DEBUG qDebug("page_finalize i:%d,item:%p,pageId:%d, offset:%d\n", i, pItem, pItem->tuplePos.pageId, pItem->tuplePos.offset); @@ -3959,10 +3962,12 @@ void addResult(SqlFunctionCtx* pCtx, STopBotResItem* pSourceItem, int16_t type, } else { // replace the minimum value in the result if ((isTopQuery && ((IS_SIGNED_NUMERIC_TYPE(type) && pSourceItem->v.i > pItems[0].v.i) || (IS_UNSIGNED_NUMERIC_TYPE(type) && pSourceItem->v.u > pItems[0].v.u) || - (IS_FLOAT_TYPE(type) && pSourceItem->v.d > pItems[0].v.d))) || + (TSDB_DATA_TYPE_FLOAT == type && pSourceItem->v.f > pItems[0].v.f) || + (TSDB_DATA_TYPE_DOUBLE == type && pSourceItem->v.d > pItems[0].v.d))) || (!isTopQuery && ((IS_SIGNED_NUMERIC_TYPE(type) && pSourceItem->v.i < pItems[0].v.i) || (IS_UNSIGNED_NUMERIC_TYPE(type) && pSourceItem->v.u < pItems[0].v.u) || - (IS_FLOAT_TYPE(type) && pSourceItem->v.d < pItems[0].v.d)))) { + (TSDB_DATA_TYPE_FLOAT == type && pSourceItem->v.f < pItems[0].v.f) || + (TSDB_DATA_TYPE_DOUBLE == type && pSourceItem->v.d < pItems[0].v.d)))) { // replace the old data and the coresponding tuple data STopBotResItem* pItem = &pItems[0]; pItem->v = pSourceItem->v; @@ -6011,7 +6016,7 @@ int32_t twaFinalize(struct SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } else { if (pInfo->win.ekey == pInfo->win.skey) { pInfo->dOutput = pInfo->p.val; - } else if (pInfo->win.ekey == INT64_MAX || pInfo->win.skey == INT64_MIN) { //no data in timewindow + } else if (pInfo->win.ekey == INT64_MAX || pInfo->win.skey == INT64_MIN) { // no data in timewindow pInfo->dOutput = 0; } else { pInfo->dOutput = pInfo->dOutput / (pInfo->win.ekey - pInfo->win.skey); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index cc1bae6a3c..8c1a85b101 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -2036,6 +2036,8 @@ void nodesValueNodeToVariant(const SValueNode* pNode, SVariant* pVal) { pVal->u = pNode->datum.u; break; case TSDB_DATA_TYPE_FLOAT: + pVal->f = pNode->datum.d; + break; case TSDB_DATA_TYPE_DOUBLE: pVal->d = pNode->datum.d; break; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c82634b400..54f450e971 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2770,7 +2770,7 @@ static int32_t convertFillValue(STranslateContext* pCxt, SDataType dt, SNodeList SNode* pCaseFunc = NULL; int32_t code = createCastFunc(pCxt, pCell->pNode, dt, &pCaseFunc); if (TSDB_CODE_SUCCESS == code) { - pCell->pNode = pCaseFunc; + code = scalarCalculateConstants(pCaseFunc, &pCell->pNode); } return code; } diff --git a/tests/script/tsim/parser/fill.sim b/tests/script/tsim/parser/fill.sim index ea0311ebde..4f0fca7c49 100644 --- a/tests/script/tsim/parser/fill.sim +++ b/tests/script/tsim/parser/fill.sim @@ -330,7 +330,7 @@ if $data11 != -1 then endi # fill_char_values_to_arithmetic_fields -sql_error select sum(c1), avg(c2), max(c3), min(c4), avg(c4), count(c6), last(c7), last(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c') +sql select sum(c1), avg(c2), max(c3), min(c4), avg(c4), count(c6), last(c7), last(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c') # fill_multiple_columns sql_error select sum(c1), avg(c2), min(c3), max(c4), count(c6), first(c7), last(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 99, 99, 99, 99, 99, abc, abc) @@ -355,25 +355,25 @@ endi # fill_into_nonarithmetic_fieds print select _wstart, first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 20000000, 20000000, 20000000) -sql_error select _wstart, first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 20000000, 20000000, 20000000) +sql select _wstart, first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 20000000, 20000000, 20000000) -sql_error select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1, 1, 1) -sql_error select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1.1, 1.1, 1.1) -sql_error select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1e1, 1e1, 1e1) +sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1, 1, 1) +sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1.1, 1.1, 1.1) +sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1e1, 1e1, 1e1) sql select first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '1e', '1e1') # fill quoted values into bool column will throw error unless the value is 'true' or 'false' Note:2018-10-24 # fill values into binary or nchar columns will be set to NULL automatically Note:2018-10-24 sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '1e', '1e1','1e1') -sql_error select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, true, true, true) +sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, true, true, true) sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'true', 'true','true') # fill nonarithmetic values into arithmetic fields sql_error select count(*) where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, abc); -sql_error select count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'true'); +sql select count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'true'); print select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '1e1'); -sql_error select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '1e1'); +sql select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '1e1'); sql select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1e1); if $rows != 9 then @@ -383,7 +383,7 @@ if $data01 != 1 then return -1 endi -sql_error select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '10'); +sql select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '10'); ## linear fill # feature currently switched off 2018/09/29 diff --git a/tests/script/tsim/parser/fill_stb.sim b/tests/script/tsim/parser/fill_stb.sim index b6cc2f38bd..e6a1d53ec7 100644 --- a/tests/script/tsim/parser/fill_stb.sim +++ b/tests/script/tsim/parser/fill_stb.sim @@ -170,7 +170,7 @@ endi sql_error select max(c1), max(c2), max(c3), max(c4), max(c5) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill (6, 6, 6, 6, 6) # fill_char_values_to_arithmetic_fields -sql_error select sum(c1), avg(c2), max(c3), min(c4), avg(c4), count(c6), last(c7), last(c8) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c') +sql select sum(c1), avg(c2), max(c3), min(c4), avg(c4), count(c6), last(c7), last(c8) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c') # fill_multiple_columns sql_error select sum(c1), avg(c2), min(c3), max(c4), count(c6), first(c7), last(c8) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 99, 99, 99, 99, 99, abc, abc) @@ -240,10 +240,10 @@ sql select first(c7), first(c8), first(c9) from $stb where ts >= $ts0 and ts <= sql select first(c7), first(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'true', 'true','true') # fill nonarithmetic values into arithmetic fields -sql_error select count(*) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'abc'); -sql_error select count(*) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'true'); +sql select count(*) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'abc'); +sql select count(*) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'true'); -sql_error select _wstart, count(*) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '2e1'); +sql select _wstart, count(*) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '2e1'); sql select _wstart, count(*) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 20); if $rows != $val then diff --git a/tests/script/tsim/parser/fill_us.sim b/tests/script/tsim/parser/fill_us.sim index f760ba3577..d7b4941c27 100644 --- a/tests/script/tsim/parser/fill_us.sim +++ b/tests/script/tsim/parser/fill_us.sim @@ -332,7 +332,7 @@ if $data11 != -1 then endi # fill_char_values_to_arithmetic_fields -sql_error select sum(c1), avg(c2), max(c3), min(c4), avg(c4), count(c6), last(c7), last(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c') +sql select sum(c1), avg(c2), max(c3), min(c4), avg(c4), count(c6), last(c7), last(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c') # fill_multiple_columns sql_error select _wstart, sum(c1), avg(c2), min(c3), max(c4), count(c6), first(c7), last(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 99, 99, 99, 99, 99, abc, abc) @@ -358,24 +358,24 @@ endi # fill_into_nonarithmetic_fieds -sql_error select _wstart, first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 20000000, 20000000, 20000000) +sql select _wstart, first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 20000000, 20000000, 20000000) -sql_error select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1, 1, 1) -sql_error select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1.1, 1.1, 1.1) -sql_error select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1e1, 1e1, 1e1) +sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1, 1, 1) +sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1.1, 1.1, 1.1) +sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1e1, 1e1, 1e1) sql select first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '1e', '1e1') # fill quoted values into bool column will throw error unless the value is 'true' or 'false' Note:2018-10-24 # fill values into binary or nchar columns will be set to null automatically Note:2018-10-24 sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '1e', '1e1','1e1') -sql_error select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, true, true, true) +sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, true, true, true) sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'true', 'true','true') # fill nonarithmetic values into arithmetic fields sql_error select count(*) where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, abc); -sql_error select count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'true'); +sql select count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'true'); -sql_error select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '1e1'); +sql select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '1e1'); sql select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1e1); if $rows != 9 then @@ -385,7 +385,7 @@ if $data01 != 1 then return -1 endi -sql_error select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '10'); +sql select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '10'); ## linear fill # feature currently switched off 2018/09/29 diff --git a/tests/script/tsim/stream/fillIntervalValue.sim b/tests/script/tsim/stream/fillIntervalValue.sim index 49e68ae9f2..89590d1be0 100644 --- a/tests/script/tsim/stream/fillIntervalValue.sim +++ b/tests/script/tsim/stream/fillIntervalValue.sim @@ -4,7 +4,7 @@ looptest: system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -#==system sh/exec.sh -n dnode1 -s start -v + sleep 200 sql connect From 40af2903c1a1e13c4e381b8f48d40543755c3dc7 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 17 Nov 2022 16:06:12 +0800 Subject: [PATCH 22/95] fix: scalar mnode for in set more than 10 and print only 1000 groups to log in filter mode --- source/libs/scalar/src/filter.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 4befe29e1d..cd8606bc8e 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1704,7 +1704,8 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) } qDebug("GROUP Num:%u", info->groupNum); - for (uint32_t i = 0; i < info->groupNum; ++i) { + uint32_t maxDbgGrpNum = MIN(info->groupNum, 1000); + for (uint32_t i = 0; i < maxDbgGrpNum; ++i) { SFilterGroup *group = &info->groups[i]; qDebug("Group%d : unit num[%u]", i, group->unitNum); @@ -3916,6 +3917,10 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { } else { SColumnNode *refNode = (SColumnNode *)node->pLeft; SNodeListNode *listNode = (SNodeListNode *)node->pRight; + if (LIST_LENGTH(listNode->pNodeList) > 10) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } int32_t type = vectorGetConvertType(refNode->node.resType.type, listNode->dataType.type); if (0 != type && type != refNode->node.resType.type) { stat->scalarMode = true; From 7cda5c2752317ada2e0fa80ca6c36bc74d39aac4 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 17 Nov 2022 16:24:56 +0800 Subject: [PATCH 23/95] enh: fill value supports implicit type conversion --- source/libs/function/src/builtinsimpl.c | 8 ++++---- source/libs/planner/src/planOptimizer.c | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index f69a76623c..2bd3555d1a 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2217,8 +2217,8 @@ bool leastSQRFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInf SLeastSQRInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo); - GET_TYPED_DATA(pInfo->startVal, double, pCtx->param[1].param.nType, pCtx->param[1].param.i); - GET_TYPED_DATA(pInfo->stepVal, double, pCtx->param[2].param.nType, pCtx->param[2].param.i); + GET_TYPED_DATA(pInfo->startVal, double, pCtx->param[1].param.nType, &pCtx->param[1].param.i); + GET_TYPED_DATA(pInfo->stepVal, double, pCtx->param[2].param.nType, &pCtx->param[2].param.i); return true; } @@ -2563,7 +2563,7 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SVariant* pVal = &pCtx->param[1].param; double v = 0; - GET_TYPED_DATA(v, double, pVal->nType, pVal->i); + GET_TYPED_DATA(v, double, pVal->nType, &pVal->i); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SPercentileInfo* ppInfo = (SPercentileInfo*)GET_ROWCELL_INTERBUF(pResInfo); @@ -2623,7 +2623,7 @@ bool apercentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResult SVariant* pVal = &pCtx->param[1].param; pInfo->percent = 0; - GET_TYPED_DATA(pInfo->percent, double, pVal->nType, pVal->i); + GET_TYPED_DATA(pInfo->percent, double, pVal->nType, &pVal->i); if (pCtx->numOfParams == 2) { pInfo->algo = APERCT_ALGO_DEFAULT; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 40b70d7d3e..45fa67faef 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -348,7 +348,9 @@ static int32_t scanPathOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub int32_t code = scanPathOptMatch(pCxt, pLogicSubplan->pNode, &info); if (TSDB_CODE_SUCCESS == code && info.pScan) { scanPathOptSetScanWin(info.pScan); - scanPathOptSetScanOrder(info.scanOrder, info.pScan); + if (!pCxt->pPlanCxt->streamQuery) { + scanPathOptSetScanOrder(info.scanOrder, info.pScan); + } } if (TSDB_CODE_SUCCESS == code && (NULL != info.pDsoFuncs || NULL != info.pSdrFuncs)) { info.pScan->dataRequired = scanPathOptGetDataRequired(info.pSdrFuncs); From 0d7787387b8ced763fb92b315a3da7690e8caa48 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Thu, 17 Nov 2022 16:58:04 +0800 Subject: [PATCH 24/95] test:add test case for boundary check --- tests/pytest/util/sql.py | 11 +- tests/system-test/1-insert/boundary.py | 186 ++++++++++++++++++++++ tests/system-test/1-insert/update_data.py | 2 +- 3 files changed, 197 insertions(+), 2 deletions(-) create mode 100644 tests/system-test/1-insert/boundary.py diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index b320cf5995..9cfd1d368e 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -73,8 +73,15 @@ class TDSql: expectErrNotOccured = True try: self.cursor.execute(sql) - except BaseException: + except BaseException as e: expectErrNotOccured = False + caller = inspect.getframeinfo(inspect.stack()[1][0]) + self.error_info = repr(e) + # print(error_info) + # self.error_info = error_info[error_info.index('(')+1:-1].split(",")[0].replace("'","") + # self.error_info = (','.join(error_info.split(",")[:-1]).split("(",1)[1:][0]).replace("'","") + # print("!!!!!!!!!!!!!!",self.error_info) + if expectErrNotOccured: caller = inspect.getframeinfo(inspect.stack()[1][0]) tdLog.exit("%s(%d) failed: sql:%s, expect error not occured" % (caller.filename, caller.lineno, sql)) @@ -83,6 +90,8 @@ class TDSql: self.queryCols = 0 self.queryResult = None tdLog.info("sql:%s, expect error occured" % (sql)) + return self.error_info + def query(self, sql, row_tag=None,queryTimes=10): self.sql = sql diff --git a/tests/system-test/1-insert/boundary.py b/tests/system-test/1-insert/boundary.py new file mode 100644 index 0000000000..35b7ca2669 --- /dev/null +++ b/tests/system-test/1-insert/boundary.py @@ -0,0 +1,186 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +import math +from random import randint +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +from util.boundary import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.boundary = DataBoundary() + self.dbname_length_boundary = self.boundary.DBNAME_MAX_LENGTH + self.tbname_length_boundary = self.boundary.TBNAME_MAX_LENGTH + self.stbname_length_boundary = self.boundary.STBNAME_MAX_LENGTH + self.colname_length_boundary = self.boundary.COL_KEY_MAX_LENGTH + self.tagname_length_boundary = self.boundary.TAG_KEY_MAX_LENGTH + self.username_length_boundary = 23 + self.password_length_boundary = 15 + def dbname_length_check(self): + dbname_length = randint(1,self.dbname_length_boundary-1) + for dbname in [tdCom.get_long_name(self.dbname_length_boundary),tdCom.get_long_name(dbname_length)]: + tdSql.execute(f'create database if not exists {dbname}') + tdSql.query(f'select name from information_schema.ins_databases where name = "{dbname}"') + tdSql.checkEqual(tdSql.queryResult[0][0],dbname) + tdSql.execute(f'drop database if exists {dbname}') + dbname = tdCom.get_long_name(self.dbname_length_boundary+1) + tdSql.error(f'create database if not exists {dbname}') + if "Invalid identifier name" in tdSql.error_info: + tdLog.info("error info is true!") + else: + tdLog.exit("error info is not true") + + def tbname_length_check(self): + tdSql.prepare() + tdSql.execute('use db') + tbname_length = randint(1,self.tbname_length_boundary-1) + tdSql.execute(f'create table stb (ts timestamp,c0 int) tags(t0 int)') + for tbname in [tdCom.get_long_name(self.tbname_length_boundary),tdCom.get_long_name(tbname_length)]: + tdSql.execute(f'create table {tbname} using stb tags(1)') + tdSql.query(f'select table_name from information_schema.ins_tables where table_name = "{tbname}"') + tdSql.checkEqual(tdSql.queryResult[0][0],tbname) + tdSql.execute(f'drop table {tbname}') + tbname = tdCom.get_long_name(self.tbname_length_boundary+1) + tdSql.error(f'create table {tbname} using stb tags(1)') + if "Invalid identifier name" in tdSql.error_info: + tdLog.info("error info is true!") + else: + tdLog.exit("error info is not true") + stbname_length = randint(1,self.stbname_length_boundary-1) + for stbname in [tdCom.get_long_name(self.stbname_length_boundary),tdCom.get_long_name(stbname_length)]: + tdSql.execute(f'create table {stbname} (ts timestamp,c0 int) tags(t0 int)') + tdSql.query(f'select stable_name from information_schema.ins_stables where stable_name = "{stbname}"') + tdSql.checkEqual(tdSql.queryResult[0][0],stbname) + tdSql.execute(f'drop table {stbname}') + stbname = tdCom.get_long_name(self.stbname_length_boundary+1) + tdSql.error(f'create table {stbname} (ts timestamp,c0 int) tags(t0 int)') + print(tdSql.error_info) + if "Invalid identifier name" in tdSql.error_info: + tdLog.info("error info is true!") + else: + tdLog.exit("error info is not true") + tdSql.execute('drop database db') + + def colname_length_check(self): + tdSql.prepare() + tdSql.execute('use db') + column_name_length = randint(1,self.colname_length_boundary-1) + for colname in [tdCom.get_long_name(column_name_length),tdCom.get_long_name(self.colname_length_boundary)]: + stbname = tdCom.get_long_name(3) + ntbname = tdCom.get_long_name(4) + tdSql.execute(f'create table {stbname} (ts timestamp,{colname} int) tags(t0 int)') + tdSql.query(f'describe {stbname}') + tdSql.checkEqual(tdSql.queryResult[1][0],colname) + tdSql.execute(f'create table {ntbname} (ts timestamp,{colname} int)') + tdSql.query(f'describe {ntbname}') + tdSql.checkEqual(tdSql.queryResult[1][0],colname) + colname = tdCom.get_long_name(self.colname_length_boundary+1) + tdSql.error(f'create table stb (ts timestamp,{colname} int) tags(t0 int)') + if "Invalid identifier name" in tdSql.error_info: + tdLog.info("error info is true!") + else: + tdLog.exit("error info is not true") + tdSql.execute('drop database db') + + def tagname_length_check(self): + tdSql.prepare() + tdSql.execute('use db') + tag_name_length = randint(1,self.tagname_length_boundary-1) + for tagname in (tdCom.get_long_name(tag_name_length),tdCom.get_long_name(self.tagname_length_boundary)): + stbname = tdCom.get_long_name(3) + tdSql.execute(f'create table {stbname} (ts timestamp,c0 int) tags({tagname} int)') + tdSql.query(f'describe {stbname}') + tdSql.checkEqual(tdSql.queryResult[-1][0],tagname) + tagname = tdCom.get_long_name(self.tagname_length_boundary+1) + tdSql.error(f'create table {stbname} (ts timestamp,c0 int) tags({tagname} int)') + if "Invalid identifier name" in tdSql.error_info: + tdLog.info("error info is true!") + else: + tdLog.exit("error info is not true") + tdSql.execute('drop database db') + + def username_length_check(self): + username_length = randint(1,self.username_length_boundary-1) + for username in [tdCom.get_long_name(username_length),tdCom.get_long_name(self.username_length_boundary)]: + tdSql.execute(f'create user {username} pass "123"') + tdSql.query('show users') + for user in tdSql.queryResult: + if user[0].lower() != 'root': + tdSql.checkEqual(user[0],username) + tdSql.execute(f'drop user {username}') + username = tdCom.get_long_name(self.username_length_boundary+1) + tdSql.error(f'create user {username} pass "123"') + if "Name or password too long" in tdSql.error_info: + tdLog.info("error info is true!") + else: + tdLog.exit("error info is not true") + + def password_length_check(self): + password_length = randint(1,self.password_length_boundary-1) + for password in [tdCom.get_long_name(password_length),tdCom.get_long_name(self.password_length_boundary)]: + username = tdCom.get_long_name(3) + tdSql.execute(f'create user {username} pass "{password}"') + password = tdCom.get_long_name(self.password_length_boundary+1) + tdSql.error(f'create user {username} pass "{password}"') + if "Name or password too long" in tdSql.error_info: + tdLog.info("error info is true!") + else: + tdLog.exit("error info is not true") + def sql_length_check(self): + insert_rows = 1021 + tdSql.prepare() + tdSql.execute('use db') + tdSql.execute('create table ntb (ts timestamp,c0 binary(1013))') + values_sql = '' + value = tdCom.get_long_name(1013) + for num in range(insert_rows): + values_sql += f' (now+{num}s,"{value}")' + + value = tdCom.get_long_name(65) + values_sql += f"(now-1s,'{value}')" + tdSql.execute(f'insert into ntb values{values_sql}') + tdSql.query('select * from ntb') + tdSql.checkRows(insert_rows+1) + + tdSql.execute('create table ntb1 (ts timestamp,c0 binary(1013))') + tdSql.error(f'insert into ntb1 values{values_sql};') + print(tdSql.error_info) + if "SQL statement too long" in tdSql.error_info: + tdLog.info("error info is true!") + else: + tdLog.exit("error info is not true") + + tdSql.execute('drop database db') + def run(self): + self.dbname_length_check() + self.tbname_length_check() + self.colname_length_check() + self.tagname_length_check() + self.username_length_check() + # self.password_length_check() + self.sql_length_check() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index 6fcd987ea4..6a4d532ac6 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(),logSql) + tdSql.init(conn.cursor()) self.setsql = TDSetSql() self.dbname = 'db_test' self.ntbname = 'ntb' From 3cc78e127e16a219dc495832d12d64e097563a69 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Thu, 17 Nov 2022 17:08:26 +0800 Subject: [PATCH 25/95] update --- tests/system-test/1-insert/boundary.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/system-test/1-insert/boundary.py b/tests/system-test/1-insert/boundary.py index 35b7ca2669..d3742ef5f9 100644 --- a/tests/system-test/1-insert/boundary.py +++ b/tests/system-test/1-insert/boundary.py @@ -33,7 +33,7 @@ class TDTestCase: self.colname_length_boundary = self.boundary.COL_KEY_MAX_LENGTH self.tagname_length_boundary = self.boundary.TAG_KEY_MAX_LENGTH self.username_length_boundary = 23 - self.password_length_boundary = 15 + self.password_length_boundary = 128 def dbname_length_check(self): dbname_length = randint(1,self.dbname_length_boundary-1) for dbname in [tdCom.get_long_name(self.dbname_length_boundary),tdCom.get_long_name(dbname_length)]: @@ -153,13 +153,11 @@ class TDTestCase: value = tdCom.get_long_name(1013) for num in range(insert_rows): values_sql += f' (now+{num}s,"{value}")' - value = tdCom.get_long_name(65) values_sql += f"(now-1s,'{value}')" tdSql.execute(f'insert into ntb values{values_sql}') tdSql.query('select * from ntb') tdSql.checkRows(insert_rows+1) - tdSql.execute('create table ntb1 (ts timestamp,c0 binary(1013))') tdSql.error(f'insert into ntb1 values{values_sql};') print(tdSql.error_info) @@ -167,7 +165,6 @@ class TDTestCase: tdLog.info("error info is true!") else: tdLog.exit("error info is not true") - tdSql.execute('drop database db') def run(self): self.dbname_length_check() @@ -175,7 +172,7 @@ class TDTestCase: self.colname_length_check() self.tagname_length_check() self.username_length_check() - # self.password_length_check() + self.password_length_check() self.sql_length_check() def stop(self): From f69c5b500c9b8b3809214ccfad12de5a47d2654a Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Thu, 17 Nov 2022 17:10:41 +0800 Subject: [PATCH 26/95] add into ci --- tests/parallel_test/cases.task | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 4c9715270d..1531c4840b 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -424,6 +424,7 @@ ,,,system-test,python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py ,,,system-test,python3 ./test.py -f 1-insert/alter_stable.py ,,,system-test,python3 ./test.py -f 1-insert/alter_table.py +,,,system-test,python3 ./test.py -f 1-insert/boundary.py ,,,system-test,python3 ./test.py -f 1-insert/insertWithMoreVgroup.py ,,,system-test,python3 ./test.py -f 1-insert/table_comment.py ,,,system-test,python3 ./test.py -f 1-insert/time_range_wise.py From c8f069c6478fea167e8f77f8c7b3403b283cb4d3 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 18 Nov 2022 09:43:40 +0800 Subject: [PATCH 27/95] change interp linear behavior --- source/libs/executor/inc/tfill.h | 2 + source/libs/executor/src/timewindowoperator.c | 61 +++++++++++-------- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index 7e0866f545..2d8df81dbd 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -36,6 +36,8 @@ typedef struct SFillColInfo { typedef struct SFillLinearInfo { SPoint start; SPoint end; + bool isStartSet; + bool isEndSet; int16_t type; int32_t bytes; } SFillLinearInfo; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 991a509d49..7582107c93 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1991,27 +1991,36 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId); SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, i); - // null data should not be kept since it can not be used to perform interpolation - if (!colDataIsNull_s(pColInfoData, rowIndex)) { - if (pLinearInfo->start.key == INT64_MIN) { + // null value is represented by using key = INT64_MIN for now. + // TODO: optimize to ignore null values for linear interpolation. + if (!pLinearInfo->isStartSet) { + if (!colDataIsNull_s(pColInfoData, rowIndex)) { pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); memcpy(pLinearInfo->start.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); - } else if (pLinearInfo->end.key == INT64_MAX) { + } + pLinearInfo->isStartSet = true; + } else if (!pLinearInfo->isEndSet) { + if (!colDataIsNull_s(pColInfoData, rowIndex)) { + pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); + memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); + } + pLinearInfo->isEndSet = true; + } else { + pLinearInfo->start.key = pLinearInfo->end.key; + memcpy(pLinearInfo->start.val, pLinearInfo->end.val, pLinearInfo->bytes); + + if (!colDataIsNull_s(pColInfoData, rowIndex)) { pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); } else { - pLinearInfo->start.key = pLinearInfo->end.key; - pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); - memcpy(pLinearInfo->start.val, pLinearInfo->end.val, pLinearInfo->bytes); - memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); + pLinearInfo->end.key = INT64_MIN; } } } } -static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, - SSDataBlock* pSrcBlock, int32_t index, bool beforeTs) { +static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock) { int32_t rows = pResBlock->info.rows; blockDataEnsureCapacity(pResBlock, rows + 1); // todo set the correct primary timestamp column @@ -2061,18 +2070,18 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp SPoint start = pLinearInfo->start; SPoint end = pLinearInfo->end; SPoint current = {.key = pSliceInfo->current}; - current.val = taosMemoryCalloc(pLinearInfo->bytes, 1); - // before interp range, do not fill - SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot); - if (colDataIsNull_s(pSrc, index) || (!beforeTs && end.key == INT64_MAX)) { + if (!pLinearInfo->isStartSet || !pLinearInfo->isEndSet) { hasInterp = false; break; - } else if (start.key == INT64_MIN || (beforeTs && end.key == INT64_MAX)) { - //hasInterp = true; - return true; } + if (start.key == INT64_MIN || end.key == INT64_MIN) { + colDataAppendNULL(pDst, rows); + break; + } + + current.val = taosMemoryCalloc(pLinearInfo->bytes, 1); taosGetLinearInterpolationVal(¤t, pLinearInfo->type, &start, &end, pLinearInfo->type); colDataAppend(pDst, rows, (char*)current.val, false); @@ -2130,12 +2139,8 @@ static void addCurrentRowToResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot); if (colDataIsNull_s(pSrc, index)) { - if (pSliceInfo->fillType != TSDB_FILL_LINEAR) { - colDataAppendNULL(pDst, pResBlock->info.rows); - continue; - } else { - return; - } + colDataAppendNULL(pDst, pResBlock->info.rows); + continue; } char* v = colDataGetData(pSrc, index); @@ -2218,9 +2223,11 @@ static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB SFillLinearInfo linearInfo = {0}; linearInfo.start.key = INT64_MIN; - linearInfo.end.key = INT64_MAX; + linearInfo.end.key = INT64_MIN; linearInfo.start.val = taosMemoryCalloc(1, pColInfo->info.bytes); linearInfo.end.val = taosMemoryCalloc(1, pColInfo->info.bytes); + linearInfo.isStartSet = false; + linearInfo.isEndSet = false; linearInfo.type = pColInfo->info.type; linearInfo.bytes = pColInfo->info.bytes; taosArrayPush(pInfo->pLinearInfo, &linearInfo); @@ -2312,7 +2319,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i, false) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { + if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { break; } else { pSliceInfo->current = @@ -2336,7 +2343,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doKeepLinearInfo(pSliceInfo, pBlock, i); while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { - if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i, true) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { + if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { break; } else { pSliceInfo->current = @@ -2365,7 +2372,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { // except for fill(next), fill(linear) while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT && pSliceInfo->fillType != TSDB_FILL_LINEAR) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, NULL, 0, false); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); } From 2055879debe9513c779047d2669c0d555e1bf55b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 18 Nov 2022 09:47:26 +0800 Subject: [PATCH 28/95] refactor: do some internal refactor. --- source/libs/executor/inc/executorimpl.h | 33 +- source/libs/executor/src/executor.c | 22 +- source/libs/executor/src/executorimpl.c | 16 +- source/libs/executor/src/scanoperator.c | 329 ++++++------------ source/libs/executor/src/timewindowoperator.c | 4 +- 5 files changed, 143 insertions(+), 261 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 0e2635459d..03dbca8668 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -315,26 +315,31 @@ typedef struct STableMetaCacheInfo { uint64_t cacheHit; } STableMetaCacheInfo; -typedef struct STableScanInfo { +typedef struct STableScanBase { STsdbReader* dataReader; - SReadHandle readHandle; - SLimitInfo limitInfo; SFileBlockLoadRecorder readRecorder; + SQueryTableDataCond cond; + SAggOptrPushDownInfo pdInfo; + SColMatchInfo matchInfo; + SReadHandle readHandle; + SExprSupp pseudoSup; + STableMetaCacheInfo metaCache; + int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan +} STableScanBase; + +typedef struct STableScanInfo { + STableScanBase base; + SLimitInfo limitInfo; SScanInfo scanInfo; int32_t scanTimes; SSDataBlock* pResBlock; - SColMatchInfo matchInfo; - SExprSupp pseudoSup; - SQueryTableDataCond cond; int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan int32_t dataBlockLoadFlag; SSampleExecInfo sample; // sample execution info int32_t currentGroupId; int32_t currentTable; int8_t scanMode; - SAggOptrPushDownInfo pdInfo; int8_t assignBlockUid; - STableMetaCacheInfo metaCache; } STableScanInfo; typedef struct STableMergeScanInfo { @@ -344,8 +349,7 @@ typedef struct STableMergeScanInfo { bool hasGroupId; uint64_t groupId; SArray* queryConds; // array of queryTableDataCond - STsdbReader* pReader; - SReadHandle readHandle; + STableScanBase base; int32_t bufPageSize; uint32_t sortBufSize; // max buffer size for in-memory sort SArray* pSortInfo; @@ -354,20 +358,11 @@ typedef struct STableMergeScanInfo { int64_t startTs; // sort start time SArray* sortSourceParams; SLimitInfo limitInfo; - SFileBlockLoadRecorder readRecorder; int64_t numOfRows; SScanInfo scanInfo; int32_t scanTimes; - SqlFunctionCtx* pCtx; // which belongs to the direct upstream operator operator query context - SResultRowInfo* pResultRowInfo; - int32_t* rowEntryInfoOffset; - SExprInfo* pExpr; SSDataBlock* pResBlock; - SColMatchInfo matchInfo; int32_t numOfOutput; - SExprSupp pseudoSup; - SQueryTableDataCond cond; - int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan int32_t dataBlockLoadFlag; // if the upstream is an interval operator, the interval info is also kept here to get the time diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index e561b6e124..7dad9245d5 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1026,8 +1026,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT SStreamScanInfo* pInfo = pOperator->info; if (pOffset->type == TMQ_OFFSET__LOG) { STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - tsdbReaderClose(pTSInfo->dataReader); - pTSInfo->dataReader = NULL; + tsdbReaderClose(pTSInfo->base.dataReader); + pTSInfo->base.dataReader = NULL; #if 0 if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus) && pInfo->tqReader->pWalReader->curVersion != pOffset->version) { @@ -1079,23 +1079,23 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT // TODO after dropping table, table may not found ASSERT(found); - if (pTableScanInfo->dataReader == NULL) { + if (pTableScanInfo->base.dataReader == NULL) { STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0); int32_t num = tableListGetSize(pTaskInfo->pTableInfoList); - if (tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &pTableScanInfo->cond, pList, num, - &pTableScanInfo->dataReader, NULL) < 0 || - pTableScanInfo->dataReader == NULL) { + if (tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &pTableScanInfo->base.cond, pList, num, + &pTableScanInfo->base.dataReader, NULL) < 0 || + pTableScanInfo->base.dataReader == NULL) { ASSERT(0); } } STableKeyInfo tki = {.uid = uid}; - tsdbSetTableList(pTableScanInfo->dataReader, &tki, 1); - int64_t oldSkey = pTableScanInfo->cond.twindows.skey; - pTableScanInfo->cond.twindows.skey = ts + 1; - tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); - pTableScanInfo->cond.twindows.skey = oldSkey; + tsdbSetTableList(pTableScanInfo->base.dataReader, &tki, 1); + int64_t oldSkey = pTableScanInfo->base.cond.twindows.skey; + pTableScanInfo->base.cond.twindows.skey = ts + 1; + tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pTableScanInfo->base.cond.twindows.skey = oldSkey; pTableScanInfo->scanTimes = 0; qDebug("tsdb reader offset seek to uid %" PRId64 " ts %" PRId64 ", table cur set to %d , all table num %d", uid, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index cf890ac3d7..c36175cd6a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1725,13 +1725,13 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan return TSDB_CODE_SUCCESS; } else if (type == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { STableScanInfo* pTableScanInfo = pOperator->info; - *order = pTableScanInfo->cond.order; - *scanFlag = pTableScanInfo->scanFlag; + *order = pTableScanInfo->base.cond.order; + *scanFlag = pTableScanInfo->base.scanFlag; return TSDB_CODE_SUCCESS; } else if (type == QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN) { STableMergeScanInfo* pTableScanInfo = pOperator->info; - *order = pTableScanInfo->cond.order; - *scanFlag = pTableScanInfo->scanFlag; + *order = pTableScanInfo->base.cond.order; + *scanFlag = pTableScanInfo->base.scanFlag; return TSDB_CODE_SUCCESS; } else { if (pOperator->pDownstream == NULL || pOperator->pDownstream[0] == NULL) { @@ -2378,8 +2378,8 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { STableScanInfo* pTableScanInfo = downstream->info; - pTableScanInfo->pdInfo.pExprSup = &pOperator->exprSupp; - pTableScanInfo->pdInfo.pAggSup = &pInfo->aggSup; + pTableScanInfo->base.pdInfo.pExprSup = &pOperator->exprSupp; + pTableScanInfo->base.pdInfo.pAggSup = &pInfo->aggSup; } code = appendDownstream(pOperator, &downstream, 1); @@ -2744,7 +2744,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } STableScanInfo* pScanInfo = pOperator->info; - pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; + pTaskInfo->cost.pRecoder = &pScanInfo->base.readRecorder; } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; @@ -2769,7 +2769,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } STableScanInfo* pScanInfo = pOperator->info; - pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; + pTaskInfo->cost.pRecoder = &pScanInfo->base.readRecorder; } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { pOperator = createExchangeOperatorInfo(pHandle ? pHandle->pMsgCb->clientRpc : NULL, (SExchangePhysiNode*)pPhyNode, pTaskInfo); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 2a3bfdd948..ee1114f330 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -233,25 +233,25 @@ static SResultRow* getTableGroupOutputBuf(SOperatorInfo* pOperator, uint64_t gro STableScanInfo* pTableScanInfo = pOperator->info; - SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pTableScanInfo->pdInfo.pAggSup->pResultRowHashTable, buf, + SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pTableScanInfo->base.pdInfo.pAggSup->pResultRowHashTable, buf, GET_RES_WINDOW_KEY_LEN(sizeof(groupId))); if (p1 == NULL) { return NULL; } - *pPage = getBufPage(pTableScanInfo->pdInfo.pAggSup->pResultBuf, p1->pageId); + *pPage = getBufPage(pTableScanInfo->base.pdInfo.pAggSup->pResultBuf, p1->pageId); return (SResultRow*)((char*)(*pPage) + p1->offset); } static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* pBlockInfo, uint32_t* status) { STableScanInfo* pTableScanInfo = pOperator->info; - if (pTableScanInfo->pdInfo.pExprSup == NULL) { + if (pTableScanInfo->base.pdInfo.pExprSup == NULL) { return TSDB_CODE_SUCCESS; } - SExprSupp* pSup1 = pTableScanInfo->pdInfo.pExprSup; + SExprSupp* pSup1 = pTableScanInfo->base.pdInfo.pExprSup; SFilePage* pPage = NULL; SResultRow* pRow = getTableGroupOutputBuf(pOperator, pBlockInfo->groupId, &pPage); @@ -264,7 +264,7 @@ static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* for (int32_t i = 0; i < pSup1->numOfExprs; ++i) { int32_t functionId = pSup1->pCtx[i].functionId; - SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, i, pTableScanInfo->pdInfo.pExprSup->rowEntryInfoOffset); + SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, i, pTableScanInfo->base.pdInfo.pExprSup->rowEntryInfoOffset); int32_t reqStatus = fmFuncDynDataRequired(functionId, pEntry, &pBlockInfo->window); if (reqStatus != FUNC_DATA_REQUIRED_NOT_LOAD) { @@ -274,7 +274,7 @@ static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* } // release buffer pages - releaseBufPage(pTableScanInfo->pdInfo.pAggSup->pResultBuf, pPage); + releaseBufPage(pTableScanInfo->base.pdInfo.pAggSup->pResultBuf, pPage); if (notLoadBlock) { *status = FUNC_DATA_REQUIRED_NOT_LOAD; @@ -293,7 +293,7 @@ static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsA return keep; } -static bool doLoadBlockSMA(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { +static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { bool allColumnsHaveAgg = true; SColumnDataAgg** pColAgg = NULL; @@ -330,7 +330,7 @@ static bool doLoadBlockSMA(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, return true; } -static void doSetTagColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo, +static void doSetTagColumnData(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo, int32_t rows) { if (pTableScanInfo->pseudoSup.numOfExprs > 0) { SExprSupp* pSup = &pTableScanInfo->pseudoSup; @@ -374,7 +374,11 @@ void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo } } -static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, + + + + +static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STableScanInfo* pInfo = pOperator->info; @@ -492,7 +496,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca return TSDB_CODE_SUCCESS; } -static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunctionCtx* pCtx, int32_t numOfOutput) { +static void prepareForDescendingScan(STableScanBase* pTableScanInfo, SqlFunctionCtx* pCtx, int32_t numOfOutput) { SET_REVERSE_SCAN_FLAG(pTableScanInfo); switchCtxOrder(pCtx, numOfOutput); @@ -697,7 +701,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { int64_t st = taosGetTimestampUs(); - while (tsdbNextDataBlock(pTableScanInfo->dataReader)) { + while (tsdbNextDataBlock(pTableScanInfo->base.dataReader)) { if (isTaskKilled(pTaskInfo)) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -712,7 +716,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { SDataBlockInfo* pBInfo = &pBlock->info; int32_t rows = 0; - tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &rows, &pBInfo->uid, &pBInfo->window); + tsdbRetrieveDataBlockInfo(pTableScanInfo->base.dataReader, &rows, &pBInfo->uid, &pBInfo->window); blockDataEnsureCapacity(pBlock, rows); // todo remove it latter pBInfo->rows = rows; @@ -721,7 +725,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { pBlock->info.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.uid); uint32_t status = 0; - int32_t code = loadDataBlock(pOperator, pTableScanInfo, pBlock, &status); + int32_t code = loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status); // int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pOperator->pTaskInfo->env, code); @@ -732,10 +736,10 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { continue; } - pOperator->resultInfo.totalRows = pTableScanInfo->readRecorder.totalRows; - pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; + pOperator->resultInfo.totalRows = pTableScanInfo->base.readRecorder.totalRows; + pTableScanInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; - pOperator->cost.totalCost = pTableScanInfo->readRecorder.elapsedTime; + pOperator->cost.totalCost = pTableScanInfo->base.readRecorder.elapsedTime; // todo refactor /*pTableScanInfo->lastStatus.uid = pBlock->info.uid;*/ @@ -755,7 +759,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; // The read handle is not initialized yet, since no qualified tables exists - if (pTableScanInfo->dataReader == NULL || pOperator->status == OP_EXEC_DONE) { + if (pTableScanInfo->base.dataReader == NULL || pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -774,15 +778,15 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { qDebug("start to repeat ascending order scan data blocks due to query func required, %s", GET_TASKID(pTaskInfo)); // do prepare for the next round table scan operation - tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); + tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); } } int32_t total = pTableScanInfo->scanInfo.numOfAsc + pTableScanInfo->scanInfo.numOfDesc; if (pTableScanInfo->scanTimes < total) { - if (pTableScanInfo->cond.order == TSDB_ORDER_ASC) { - prepareForDescendingScan(pTableScanInfo, pOperator->exprSupp.pCtx, 0); - tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); + if (pTableScanInfo->base.cond.order == TSDB_ORDER_ASC) { + prepareForDescendingScan(&pTableScanInfo->base, pOperator->exprSupp.pCtx, 0); + tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); } @@ -799,7 +803,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { pTableScanInfo->scanFlag = REPEAT_SCAN; qDebug("%s start to repeat descending order scan data blocks", GET_TASKID(pTaskInfo)); - tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); + tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); } } } @@ -828,11 +832,11 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { } STableKeyInfo* pTableInfo = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->currentTable); - tsdbSetTableList(pInfo->dataReader, pTableInfo, 1); + tsdbSetTableList(pInfo->base.dataReader, pTableInfo, 1); qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d %s", pTableInfo->uid, numOfTables, pInfo->currentTable, pTaskInfo->id.str); - tsdbReaderReset(pInfo->dataReader, &pInfo->cond); + tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; } } else { // scan table group by group sequentially @@ -845,10 +849,10 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { int32_t num = 0; STableKeyInfo* pList = NULL; tableListGetGroupList(pTaskInfo->pTableInfoList, pInfo->currentGroupId, &pList, &num); - ASSERT(pInfo->dataReader == NULL); + ASSERT(pInfo->base.dataReader == NULL); - int32_t code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, pList, num, - (STsdbReader**)&pInfo->dataReader, GET_TASKID(pTaskInfo)); + int32_t code = tsdbReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, + (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -874,8 +878,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableKeyInfo* pList = NULL; tableListGetGroupList(pTaskInfo->pTableInfoList, pInfo->currentGroupId, &pList, &num); - tsdbSetTableList(pInfo->dataReader, pList, num); - tsdbReaderReset(pInfo->dataReader, &pInfo->cond); + tsdbSetTableList(pInfo->base.dataReader, pList, num); + tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; result = doGroupedTableScan(pOperator); @@ -891,7 +895,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { SFileBlockLoadRecorder* pRecorder = taosMemoryCalloc(1, sizeof(SFileBlockLoadRecorder)); STableScanInfo* pTableScanInfo = pOptr->info; - *pRecorder = pTableScanInfo->readRecorder; + *pRecorder = pTableScanInfo->base.readRecorder; *pOptrExplain = pRecorder; *len = sizeof(SFileBlockLoadRecorder); return 0; @@ -900,17 +904,17 @@ static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptr static void destroyTableScanOperatorInfo(void* param) { STableScanInfo* pTableScanInfo = (STableScanInfo*)param; blockDataDestroy(pTableScanInfo->pResBlock); - cleanupQueryTableDataCond(&pTableScanInfo->cond); + cleanupQueryTableDataCond(&pTableScanInfo->base.cond); - tsdbReaderClose(pTableScanInfo->dataReader); - pTableScanInfo->dataReader = NULL; + tsdbReaderClose(pTableScanInfo->base.dataReader); + pTableScanInfo->base.dataReader = NULL; - if (pTableScanInfo->matchInfo.pList != NULL) { - taosArrayDestroy(pTableScanInfo->matchInfo.pList); + if (pTableScanInfo->base.matchInfo.pList != NULL) { + taosArrayDestroy(pTableScanInfo->base.matchInfo.pList); } - taosLRUCacheCleanup(pTableScanInfo->metaCache.pTableMetaEntryCache); - cleanupExprSupp(&pTableScanInfo->pseudoSup); + taosLRUCacheCleanup(pTableScanInfo->base.metaCache.pTableMetaEntryCache); + cleanupExprSupp(&pTableScanInfo->base.pseudoSup); taosMemoryFreeClear(param); } @@ -927,26 +931,26 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, int32_t numOfCols = 0; int32_t code = - extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo); + extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->base.matchInfo); if (code != TSDB_CODE_SUCCESS) { goto _error; } initLimitInfo(pScanNode->node.pLimit, pScanNode->node.pSlimit, &pInfo->limitInfo); - code = initQueryTableDataCond(&pInfo->cond, pTableScanNode); + code = initQueryTableDataCond(&pInfo->base.cond, pTableScanNode); if (code != TSDB_CODE_SUCCESS) { goto _error; } if (pScanNode->pScanPseudoCols != NULL) { - SExprSupp* pSup = &pInfo->pseudoSup; + SExprSupp* pSup = &pInfo->base.pseudoSup; pSup->pExprInfo = createExprInfo(pScanNode->pScanPseudoCols, NULL, &pSup->numOfExprs); pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset); } pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; - pInfo->pdInfo.interval = extractIntervalInfo(pTableScanNode); - pInfo->readHandle = *readHandle; + pInfo->base.pdInfo.interval = extractIntervalInfo(pTableScanNode); + pInfo->base.readHandle = *readHandle; pInfo->sample.sampleRatio = pTableScanNode->ratio; pInfo->sample.seed = taosGetTimestampSec(); @@ -969,13 +973,13 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; - pInfo->metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5); - if (pInfo->metaCache.pTableMetaEntryCache == NULL) { + pInfo->base.metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5); + if (pInfo->base.metaCache.pTableMetaEntryCache == NULL) { code = terrno; goto _error; } - taosLRUCacheSetStrictCapacity(pInfo->metaCache.pTableMetaEntryCache, false); + taosLRUCacheSetStrictCapacity(pInfo->base.metaCache.pTableMetaEntryCache, false); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo, getTableScannerExecInfo); @@ -997,7 +1001,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - pInfo->dataReader = pReadHandle; + pInfo->base.dataReader = pReadHandle; // pInfo->prevGroupId = -1; setOperatorInfo(pOperator, "TableSeqScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, false, OP_NOT_OPENED, @@ -1204,11 +1208,11 @@ static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t grou } void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin) { - pTableScanInfo->cond.twindows = *pWin; + pTableScanInfo->base.cond.twindows = *pWin; pTableScanInfo->scanTimes = 0; pTableScanInfo->currentGroupId = -1; - tsdbReaderClose(pTableScanInfo->dataReader); - pTableScanInfo->dataReader = NULL; + tsdbReaderClose(pTableScanInfo->base.dataReader); + pTableScanInfo->base.dataReader = NULL; } static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbUid, TSKEY startTs, TSKEY endTs, @@ -1216,7 +1220,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU STableKeyInfo tblInfo = {.uid = tbUid, .groupId = 0}; STableScanInfo* pTableScanInfo = pTableScanOp->info; - SQueryTableDataCond cond = pTableScanInfo->cond; + SQueryTableDataCond cond = pTableScanInfo->base.cond; cond.startVersion = -1; cond.endVersion = maxVersion; @@ -1228,7 +1232,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU blockDataCleanup(pBlock); STsdbReader* pReader = NULL; - int32_t code = tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &cond, &tblInfo, 1, (STsdbReader**)&pReader, + int32_t code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, (STsdbReader**)&pReader, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { terrno = code; @@ -1247,8 +1251,8 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU blockDataEnsureCapacity(pBlock, rows); pBlock->info.rows = rows; - relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true); - doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, rows); + relocateColumnData(pBlock, pTableScanInfo->base.matchInfo.pList, pCols, true); + doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, rows); pBlock->info.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBInfo->uid); } @@ -1375,8 +1379,8 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 *pRowIndex = 0; pInfo->updateWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - tsdbReaderClose(pTableScanInfo->dataReader); - pTableScanInfo->dataReader = NULL; + tsdbReaderClose(pTableScanInfo->base.dataReader); + pTableScanInfo->base.dataReader = NULL; return NULL; } @@ -1806,8 +1810,8 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { } else { if (!pTaskInfo->streamInfo.returned) { STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - tsdbReaderClose(pTSInfo->dataReader); - pTSInfo->dataReader = NULL; + tsdbReaderClose(pTSInfo->base.dataReader); + pTSInfo->base.dataReader = NULL; tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer); qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1); if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1) < 0) { @@ -1973,22 +1977,22 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE1 || pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE2) { STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - memcpy(&pTSInfo->cond, &pTaskInfo->streamInfo.tableCond, sizeof(SQueryTableDataCond)); + memcpy(&pTSInfo->base.cond, &pTaskInfo->streamInfo.tableCond, sizeof(SQueryTableDataCond)); if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE1) { - pTSInfo->cond.startVersion = 0; - pTSInfo->cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer1; - qDebug("stream recover step 1, from %" PRId64 " to %" PRId64, pTSInfo->cond.startVersion, - pTSInfo->cond.endVersion); + pTSInfo->base.cond.startVersion = 0; + pTSInfo->base.cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer1; + qDebug("stream recover step 1, from %" PRId64 " to %" PRId64, pTSInfo->base.cond.startVersion, + pTSInfo->base.cond.endVersion); } else { - pTSInfo->cond.startVersion = pTaskInfo->streamInfo.fillHistoryVer1 + 1; - pTSInfo->cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer2; - qDebug("stream recover step 2, from %" PRId64 " to %" PRId64, pTSInfo->cond.startVersion, - pTSInfo->cond.endVersion); + pTSInfo->base.cond.startVersion = pTaskInfo->streamInfo.fillHistoryVer1 + 1; + pTSInfo->base.cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer2; + qDebug("stream recover step 2, from %" PRId64 " to %" PRId64, pTSInfo->base.cond.startVersion, + pTSInfo->base.cond.endVersion); } /*resetTableScanInfo(pTSInfo, pWin);*/ - tsdbReaderClose(pTSInfo->dataReader); - pTSInfo->dataReader = NULL; + tsdbReaderClose(pTSInfo->base.dataReader); + pTSInfo->base.dataReader = NULL; pTSInfo->scanTimes = 0; pTSInfo->currentGroupId = -1; @@ -2009,11 +2013,11 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { } pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - tsdbReaderClose(pTSInfo->dataReader); - pTSInfo->dataReader = NULL; + tsdbReaderClose(pTSInfo->base.dataReader); + pTSInfo->base.dataReader = NULL; - pTSInfo->cond.startVersion = -1; - pTSInfo->cond.endVersion = -1; + pTSInfo->base.cond.startVersion = -1; + pTSInfo->base.cond.endVersion = -1; return NULL; } @@ -2118,8 +2122,8 @@ FETCH_NEXT_BLOCK: SSDataBlock* pSDB = doRangeScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); if (pSDB) { STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - uint64_t version = getReaderMaxVersion(pTableScanInfo->dataReader); - updateInfoSetScanRange(pInfo->pUpdateInfo, &pTableScanInfo->cond.twindows, pInfo->groupId, version); + uint64_t version = getReaderMaxVersion(pTableScanInfo->base.dataReader); + updateInfoSetScanRange(pInfo->pUpdateInfo, &pTableScanInfo->base.cond.twindows, pInfo->groupId, version); pSDB->info.type = pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE ? STREAM_NORMAL : STREAM_PULL_DATA; checkUpdateData(pInfo, true, pSDB, false); // printDataBlock(pSDB, "stream scan update"); @@ -2513,7 +2517,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys SOperatorInfo* pTableScanOp = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo); STableScanInfo* pTSInfo = (STableScanInfo*)pTableScanOp->info; if (pHandle->version > 0) { - pTSInfo->cond.endVersion = pHandle->version; + pTSInfo->base.cond.endVersion = pHandle->version; } STableKeyInfo* pList = NULL; @@ -2522,8 +2526,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys if (pHandle->initTableReader) { pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER; - pTSInfo->dataReader = NULL; - code = tsdbReaderOpen(pHandle->vnode, &pTSInfo->cond, pList, num, &pTSInfo->dataReader, NULL); + pTSInfo->base.dataReader = NULL; + code = tsdbReaderOpen(pHandle->vnode, &pTSInfo->base.cond, pList, num, &pTSInfo->base.dataReader, NULL); if (code != 0) { terrno = code; destroyTableScanOperatorInfo(pTableScanOp); @@ -2559,7 +2563,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys goto _error; } taosArrayDestroy(tableIdList); - memcpy(&pTaskInfo->streamInfo.tableCond, &pTSInfo->cond, sizeof(SQueryTableDataCond)); + memcpy(&pTaskInfo->streamInfo.tableCond, &pTSInfo->base.cond, sizeof(SQueryTableDataCond)); } else { taosArrayDestroy(pColIds); } @@ -4374,123 +4378,6 @@ _error: return NULL; } -// todo refactor -static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo, - SSDataBlock* pBlock, uint32_t* status) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - STableMergeScanInfo* pInfo = pOperator->info; - - SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder; - - pCost->totalBlocks += 1; - pCost->totalRows += pBlock->info.rows; - - *status = pInfo->dataBlockLoadFlag; - if (pOperator->exprSupp.pFilterInfo != NULL || - overlapWithTimeWindow(&pTableScanInfo->interval, &pBlock->info, pTableScanInfo->cond.order)) { - (*status) = FUNC_DATA_REQUIRED_DATA_LOAD; - } - - SDataBlockInfo* pBlockInfo = &pBlock->info; - taosMemoryFreeClear(pBlock->pBlockAgg); - - if (*status == FUNC_DATA_REQUIRED_FILTEROUT) { - qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), - pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - pCost->filterOutBlocks += 1; - return TSDB_CODE_SUCCESS; - } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { - qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), - pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - pCost->skipBlocks += 1; - - // clear all data in pBlock that are set when handing the previous block - for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { - SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i); - pcol->pData = NULL; - } - - return TSDB_CODE_SUCCESS; - } else if (*status == FUNC_DATA_REQUIRED_SMA_LOAD) { - pCost->loadBlockStatis += 1; - - bool allColumnsHaveAgg = true; - SColumnDataAgg** pColAgg = NULL; - - if (allColumnsHaveAgg == true) { - int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - - // todo create this buffer during creating operator - if (pBlock->pBlockAgg == NULL) { - pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, POINTER_BYTES); - } - - for (int32_t i = 0; i < numOfCols; ++i) { - SColMatchItem* pColMatchInfo = taosArrayGet(pTableScanInfo->matchInfo.pList, i); - if (!pColMatchInfo->needOutput) { - continue; - } - pBlock->pBlockAgg[pColMatchInfo->dstSlotId] = pColAgg[i]; - } - - return TSDB_CODE_SUCCESS; - } else { // failed to load the block sma data, data block statistics does not exist, load data block instead - *status = FUNC_DATA_REQUIRED_DATA_LOAD; - } - } - - ASSERT(*status == FUNC_DATA_REQUIRED_DATA_LOAD); - - // todo filter data block according to the block sma data firstly -#if 0 - if (!doFilterByBlockSMA(pBlock->pBlockStatis, pTableScanInfo->pCtx, pBlockInfo->rows)) { - pCost->filterOutBlocks += 1; - qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, - pBlockInfo->window.ekey, pBlockInfo->rows); - (*status) = FUNC_DATA_REQUIRED_FILTEROUT; - return TSDB_CODE_SUCCESS; - } -#endif - - pCost->totalCheckedRows += pBlock->info.rows; - pCost->loadBlocks += 1; - - STsdbReader* reader = pTableScanInfo->pReader; - SArray* pCols = tsdbRetrieveDataBlock(reader, NULL); - if (pCols == NULL) { - return terrno; - } - - relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true); - - // currently only the tbname pseudo column - SExprSupp* pSup = &pTableScanInfo->pseudoSup; - - int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, - pBlock->info.rows, GET_TASKID(pTaskInfo), NULL); - if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, code); - } - - if (pOperator->exprSupp.pFilterInfo != NULL) { - int64_t st = taosGetTimestampMs(); - doFilter(pBlock, pOperator->exprSupp.pFilterInfo, &pTableScanInfo->matchInfo); - - double el = (taosGetTimestampUs() - st) / 1000.0; - pTableScanInfo->readRecorder.filterTime += el; - - if (pBlock->info.rows == 0) { - pCost->filterOutBlocks += 1; - qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d, elapsed time:%.2f ms", - GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, el); - } else { - qDebug("%s data block filter applied, elapsed time:%.2f ms", GET_TASKID(pTaskInfo), el); - } - } - - return TSDB_CODE_SUCCESS; -} - static SSDataBlock* getTableDataBlockImpl(void* param) { STableMergeScanSortSourceParam* source = param; SOperatorInfo* pOperator = source->pOperator; @@ -4506,14 +4393,14 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { int64_t st = taosGetTimestampUs(); void* p = tableListGetInfo(pInfo->tableListInfo, readIdx + pInfo->tableStartIndex); - SReadHandle* pHandle = &pInfo->readHandle; + SReadHandle* pHandle = &pInfo->base.readHandle; - int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, &pInfo->pReader, GET_TASKID(pTaskInfo)); + int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, &pInfo->base.dataReader, GET_TASKID(pTaskInfo)); if (code != 0) { T_LONG_JMP(pTaskInfo->env, code); } - STsdbReader* reader = pInfo->pReader; + STsdbReader* reader = pInfo->base.dataReader; while (tsdbNextDataBlock(reader)) { if (isTaskKilled(pTaskInfo)) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); @@ -4539,7 +4426,8 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } uint32_t status = 0; - code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, pBlock, &status); + loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status); +// code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, pBlock, &status); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -4552,15 +4440,15 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { pBlock->info.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.uid); pOperator->resultInfo.totalRows += pBlock->info.rows; - pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; + pTableScanInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; - tsdbReaderClose(pInfo->pReader); - pInfo->pReader = NULL; + tsdbReaderClose(pInfo->base.dataReader); + pInfo->base.dataReader = NULL; return pBlock; } - tsdbReaderClose(pInfo->pReader); - pInfo->pReader = NULL; + tsdbReaderClose(pInfo->base.dataReader); + pInfo->base.dataReader = NULL; return NULL; } @@ -4612,7 +4500,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { int32_t tableStartIdx = pInfo->tableStartIndex; int32_t tableEndIdx = pInfo->tableEndIndex; - pInfo->pReader = NULL; + pInfo->base.dataReader = NULL; // todo the total available buffer should be determined by total capacity of buffer of this task. // the additional one is reserved for merge result @@ -4635,7 +4523,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { taosArrayPush(pInfo->sortSourceParams, ¶m); SQueryTableDataCond cond; - dumpSQueryTableCond(&pInfo->cond, &cond); + dumpSQueryTableCond(&pInfo->base.cond, &cond); taosArrayPush(pInfo->queryConds, &cond); } @@ -4763,7 +4651,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { void destroyTableMergeScanOperatorInfo(void* param) { STableMergeScanInfo* pTableScanInfo = (STableMergeScanInfo*)param; - cleanupQueryTableDataCond(&pTableScanInfo->cond); + cleanupQueryTableDataCond(&pTableScanInfo->base.cond); int32_t numOfTable = taosArrayGetSize(pTableScanInfo->queryConds); @@ -4774,8 +4662,8 @@ void destroyTableMergeScanOperatorInfo(void* param) { taosArrayDestroy(pTableScanInfo->sortSourceParams); - tsdbReaderClose(pTableScanInfo->pReader); - pTableScanInfo->pReader = NULL; + tsdbReaderClose(pTableScanInfo->base.dataReader); + pTableScanInfo->base.dataReader = NULL; for (int i = 0; i < taosArrayGetSize(pTableScanInfo->queryConds); i++) { SQueryTableDataCond* pCond = taosArrayGet(pTableScanInfo->queryConds, i); @@ -4783,17 +4671,16 @@ void destroyTableMergeScanOperatorInfo(void* param) { } taosArrayDestroy(pTableScanInfo->queryConds); - if (pTableScanInfo->matchInfo.pList != NULL) { - taosArrayDestroy(pTableScanInfo->matchInfo.pList); + if (pTableScanInfo->base.matchInfo.pList != NULL) { + taosArrayDestroy(pTableScanInfo->base.matchInfo.pList); } pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock); pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock); taosArrayDestroy(pTableScanInfo->pSortInfo); - cleanupExprSupp(&pTableScanInfo->pseudoSup); + cleanupExprSupp(&pTableScanInfo->base.pseudoSup); - taosMemoryFreeClear(pTableScanInfo->rowEntryInfoOffset); taosMemoryFreeClear(param); } @@ -4802,7 +4689,7 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla // TODO: merge these two info into one struct STableMergeScanExecInfo* execInfo = taosMemoryCalloc(1, sizeof(STableMergeScanExecInfo)); STableMergeScanInfo* pInfo = pOptr->info; - execInfo->blockRecorder = pInfo->readRecorder; + execInfo->blockRecorder = pInfo->base.readRecorder; execInfo->sortExecInfo = pInfo->sortExecInfo; *pOptrExplain = execInfo; @@ -4823,26 +4710,26 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN int32_t numOfCols = 0; int32_t code = extractColMatchInfo(pTableScanNode->scan.pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, - &pInfo->matchInfo); + &pInfo->base.matchInfo); if (code != TSDB_CODE_SUCCESS) { goto _error; } - code = initQueryTableDataCond(&pInfo->cond, pTableScanNode); + code = initQueryTableDataCond(&pInfo->base.cond, pTableScanNode); if (code != TSDB_CODE_SUCCESS) { - taosArrayDestroy(pInfo->matchInfo.pList); + taosArrayDestroy(pInfo->base.matchInfo.pList); goto _error; } if (pTableScanNode->scan.pScanPseudoCols != NULL) { - SExprSupp* pSup = &pInfo->pseudoSup; + SExprSupp* pSup = &pInfo->base.pseudoSup; pSup->pExprInfo = createExprInfo(pTableScanNode->scan.pScanPseudoCols, NULL, &pSup->numOfExprs); pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset); } pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; - pInfo->readHandle = *readHandle; + pInfo->base.readHandle = *readHandle; pInfo->interval = extractIntervalInfo(pTableScanNode); pInfo->sample.sampleRatio = pTableScanNode->ratio; pInfo->sample.seed = taosGetTimestampSec(); @@ -4854,7 +4741,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN } pInfo->tableListInfo = pTableListInfo; - pInfo->scanFlag = MAIN_SCAN; + pInfo->base.scanFlag = MAIN_SCAN; initResultSizeInfo(&pOperator->resultInfo, 1024); pInfo->pResBlock = createResDataBlock(pDescNode); @@ -4862,7 +4749,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam)); - pInfo->pSortInfo = generateSortByTsInfo(pInfo->matchInfo.pList, pInfo->cond.order); + pInfo->pSortInfo = generateSortByTsInfo(pInfo->base.matchInfo.pList, pInfo->base.cond.order); pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false); initLimitInfo(pTableScanNode->scan.node.pLimit, pTableScanNode->scan.node.pSlimit, &pInfo->limitInfo); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index fe29b8eda9..bf1d146b13 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2549,8 +2549,8 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode pInfo->current = pInfo->win.skey; STableScanInfo* pScanInfo = (STableScanInfo*)downstream->info; - pScanInfo->cond.twindows = pInfo->win; - pScanInfo->cond.type = TIMEWINDOW_RANGE_EXTERNAL; + pScanInfo->base.cond.twindows = pInfo->win; + pScanInfo->base.cond.type = TIMEWINDOW_RANGE_EXTERNAL; setOperatorInfo(pOperator, "TimeSliceOperator", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, false, OP_NOT_OPENED, pInfo, pTaskInfo); From 7edde3a780c824029a7f1118b0a156cf2acf9eae Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 18 Nov 2022 10:04:28 +0800 Subject: [PATCH 29/95] test: add log and asan case --- source/libs/sync/src/syncMain.c | 20 ++++++++++---------- tests/parallel_test/cases.task | 6 +++--- tests/script/tsim/mnode/basic5.sim | 6 +++--- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 369847b672..a6e0cb7659 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -944,6 +944,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { SSyncSnapshotSender* pSender = snapshotSenderCreate(pSyncNode, i); // ASSERT(pSender != NULL); (pSyncNode->senders)[i] = pSender; + sSTrace(pSender, "snapshot sender create new while open, data:%p", pSender); } // snapshot receivers @@ -971,7 +972,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { atomic_store_64(&pSyncNode->snapshottingIndex, SYNC_INDEX_INVALID); pSyncNode->isStart = true; - sNTrace(pSyncNode, "sync open"); + sNTrace(pSyncNode, "sync open, node:%p", pSyncNode); return pSyncNode; @@ -1041,14 +1042,10 @@ void syncNodePreClose(SSyncNode* pSyncNode) { void syncHbTimerDataFree(SSyncHbTimerData* pData) { taosMemoryFree(pData); } void syncNodeClose(SSyncNode* pSyncNode) { - if (pSyncNode == NULL) { - return; - } - int32_t ret; + if (pSyncNode == NULL) return; + sNTrace(pSyncNode, "sync close, data:%p", pSyncNode); - sNTrace(pSyncNode, "sync close"); - - ret = raftStoreClose(pSyncNode->pRaftStore); + int32_t ret = raftStoreClose(pSyncNode->pRaftStore); ASSERT(ret == 0); pSyncNode->pRaftStore = NULL; @@ -1077,6 +1074,7 @@ void syncNodeClose(SSyncNode* pSyncNode) { for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { if ((pSyncNode->senders)[i] != NULL) { + sSTrace((pSyncNode->senders)[i], "snapshot sender destroy while close, data:%p", (pSyncNode->senders)[i]); snapshotSenderDestroy((pSyncNode->senders)[i]); (pSyncNode->senders)[i] = NULL; } @@ -1423,15 +1421,17 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { if ((pSyncNode->senders)[i] == NULL) { (pSyncNode->senders)[i] = snapshotSenderCreate(pSyncNode, i); - sSTrace((pSyncNode->senders)[i], "snapshot sender create new"); + sSTrace((pSyncNode->senders)[i], "snapshot sender create new while reconfig, data:%p", (pSyncNode->senders)[i]); + } else { + sSTrace((pSyncNode->senders)[i], "snapshot sender already exist, data:%p", (pSyncNode->senders)[i]); } } // free old for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { if (oldSenders[i] != NULL) { + sNTrace(pSyncNode, "snapshot sender destroy old, data:%p replica-index:%d", oldSenders[i], i); snapshotSenderDestroy(oldSenders[i]); - sNTrace(pSyncNode, "snapshot sender delete old %p replica-index:%d", oldSenders[i], i); oldSenders[i] = NULL; } } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 2aac64d045..89e7e9f796 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -213,7 +213,7 @@ ,,n,script,./test.sh -f tsim/stream/basic0.sim -g ,,y,script,./test.sh -f tsim/stream/basic1.sim ,,y,script,./test.sh -f tsim/stream/basic2.sim -,,,script,./test.sh -f tsim/stream/drop_stream.sim +,,y,script,./test.sh -f tsim/stream/drop_stream.sim ,,y,script,./test.sh -f tsim/stream/fillHistoryBasic1.sim ,,y,script,./test.sh -f tsim/stream/fillHistoryBasic2.sim ,,y,script,./test.sh -f tsim/stream/fillHistoryBasic3.sim @@ -228,7 +228,7 @@ ,,y,script,./test.sh -f tsim/stream/partitionby.sim ,,y,script,./test.sh -f tsim/stream/partitionby1.sim ,,y,script,./test.sh -f tsim/stream/schedSnode.sim -,,,script,./test.sh -f tsim/stream/windowClose.sim +,,y,script,./test.sh -f tsim/stream/windowClose.sim ,,y,script,./test.sh -f tsim/stream/ignoreExpiredData.sim ,,y,script,./test.sh -f tsim/stream/sliding.sim ,,y,script,./test.sh -f tsim/stream/partitionbyColumnInterval.sim @@ -279,7 +279,7 @@ ,,y,script,./test.sh -f tsim/stable/vnode3.sim ,,y,script,./test.sh -f tsim/stable/metrics_idx.sim ,,,script,./test.sh -f tsim/sma/drop_sma.sim -,,,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim +,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim ,,n,script,./test.sh -f tsim/valgrind/checkError1.sim diff --git a/tests/script/tsim/mnode/basic5.sim b/tests/script/tsim/mnode/basic5.sim index 5bb1f41230..f0f0f18576 100644 --- a/tests/script/tsim/mnode/basic5.sim +++ b/tests/script/tsim/mnode/basic5.sim @@ -96,7 +96,7 @@ sql_error drop mnode on dnode 4 sql_error drop mnode on dnode 5 sql_error drop mnode on dnode 6 -system sh/exec.sh -n dnode2 -s stop -x SIGKILL +system sh/exec.sh -n dnode2 -s stop $x = 0 step5: $x = $x + 1 @@ -151,7 +151,7 @@ if $data(4)[4] != ready then endi print =============== step6: stop mnode1 -system sh/exec.sh -n dnode1 -s stop -x SIGKILL +system sh/exec.sh -n dnode1 -s stop # sql_error drop mnode on dnode 1 $x = 0 @@ -205,7 +205,7 @@ if $data(4)[4] != ready then endi print =============== step8: stop mnode1 and drop it -system sh/exec.sh -n dnode1 -s stop -x SIGKILL +system sh/exec.sh -n dnode1 -s stop $x = 0 step81: From 83ee245715c49ee627e16eb64e0b2a4f17198eb1 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 18 Nov 2022 10:12:21 +0800 Subject: [PATCH 30/95] fix: only set scan info information when the interp input operator is table scan --- source/libs/executor/src/timewindowoperator.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 87b4ed4c4a..8348ce80c7 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2548,9 +2548,11 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode pInfo->interval.interval = pInterpPhyNode->interval; pInfo->current = pInfo->win.skey; - STableScanInfo* pScanInfo = (STableScanInfo*)downstream->info; - pScanInfo->cond.twindows = pInfo->win; - pScanInfo->cond.type = TIMEWINDOW_RANGE_EXTERNAL; + if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { + STableScanInfo* pScanInfo = (STableScanInfo*)downstream->info; + pScanInfo->cond.twindows = pInfo->win; + pScanInfo->cond.type = TIMEWINDOW_RANGE_EXTERNAL; + } setOperatorInfo(pOperator, "TimeSliceOperator", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, false, OP_NOT_OPENED, pInfo, pTaskInfo); From 484bf1c49a426b327733142eaff81460488de445 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 18 Nov 2022 10:40:49 +0800 Subject: [PATCH 31/95] test: remove unstable case --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 89e7e9f796..9013ac1c53 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -294,7 +294,7 @@ ,,,script,./test.sh -f tsim/vnode/replica3_basic.sim ,,y,script,./test.sh -f tsim/vnode/replica3_repeat.sim ,,y,script,./test.sh -f tsim/vnode/replica3_vgroup.sim -,,y,script,./test.sh -f tsim/vnode/replica3_many.sim +#,,y,script,./test.sh -f tsim/vnode/replica3_many.sim ,,y,script,./test.sh -f tsim/vnode/replica3_import.sim ,,y,script,./test.sh -f tsim/vnode/stable_balance_replica1.sim ,,y,script,./test.sh -f tsim/vnode/stable_dnode2_stop.sim From 41f4ae2e1402473ece4fd6852fcb681df9998991 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 18 Nov 2022 10:43:45 +0800 Subject: [PATCH 32/95] test: adjust unstable case --- tests/parallel_test/cases.task | 2 +- tests/script/tsim/vnode/replica3_many.sim | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 9013ac1c53..89e7e9f796 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -294,7 +294,7 @@ ,,,script,./test.sh -f tsim/vnode/replica3_basic.sim ,,y,script,./test.sh -f tsim/vnode/replica3_repeat.sim ,,y,script,./test.sh -f tsim/vnode/replica3_vgroup.sim -#,,y,script,./test.sh -f tsim/vnode/replica3_many.sim +,,y,script,./test.sh -f tsim/vnode/replica3_many.sim ,,y,script,./test.sh -f tsim/vnode/replica3_import.sim ,,y,script,./test.sh -f tsim/vnode/stable_balance_replica1.sim ,,y,script,./test.sh -f tsim/vnode/stable_dnode2_stop.sim diff --git a/tests/script/tsim/vnode/replica3_many.sim b/tests/script/tsim/vnode/replica3_many.sim index fbce960b09..5dba0b3914 100644 --- a/tests/script/tsim/vnode/replica3_many.sim +++ b/tests/script/tsim/vnode/replica3_many.sim @@ -201,31 +201,39 @@ system sh/exec.sh -n dnode2 -s start sleep 3000 print ======== step6 +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 50 then + return -1 + endi + sql select count(*) from db1.tb1 print select count(*) from db1.tb1 ==> $data00 $lastRows1 if $data00 <= $lastRows1 then - return -1 + goto step6 endi $lastRows1 = $data00 sql select count(*) from db2.tb2 print select count(*) from db2.tb2 ==> $data00 $lastRows2 if $data00 <= $lastRows2 then - return -1 + goto step6 endi $lastRows2 = $data00 sql select count(*) from db3.tb3 print select count(*) from db3.tb3 ==> $data00 $lastRows3 if $data00 <= $lastRows3 then - return -1 + goto step6 endi $lastRows3 = $data00 sql select count(*) from db4.tb4 print select count(*) from db4.tb4 ==> $data00 $lastRows4 if $data00 <= $lastRows4 then - return -1 + goto step6 endi $lastRows4 = $data00 From 3264750ca4390b69c17e8b96260261541eccc618 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 18 Nov 2022 10:47:33 +0800 Subject: [PATCH 33/95] test: remove unstable case --- tests/parallel_test/cases.task | 2 +- tests/script/tsim/vnode/replica3_many.sim | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 89e7e9f796..aa1e58b5d5 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -11,7 +11,7 @@ ,,y,script,./test.sh -f tsim/user/privilege_db.sim ,,y,script,./test.sh -f tsim/user/privilege_sysinfo.sim ,,y,script,./test.sh -f tsim/db/alter_option.sim -,,y,script,./test.sh -f tsim/db/alter_replica_13.sim +#,,y,script,./test.sh -f tsim/db/alter_replica_13.sim ,,y,script,./test.sh -f tsim/db/alter_replica_31.sim ,,y,script,./test.sh -f tsim/db/basic1.sim ,,y,script,./test.sh -f tsim/db/basic2.sim diff --git a/tests/script/tsim/vnode/replica3_many.sim b/tests/script/tsim/vnode/replica3_many.sim index 5dba0b3914..cac787d9f1 100644 --- a/tests/script/tsim/vnode/replica3_many.sim +++ b/tests/script/tsim/vnode/replica3_many.sim @@ -209,28 +209,28 @@ step6: return -1 endi -sql select count(*) from db1.tb1 +sql select count(*) from db1.tb1 -x step6 print select count(*) from db1.tb1 ==> $data00 $lastRows1 if $data00 <= $lastRows1 then goto step6 endi $lastRows1 = $data00 -sql select count(*) from db2.tb2 +sql select count(*) from db2.tb2 -x step6 print select count(*) from db2.tb2 ==> $data00 $lastRows2 if $data00 <= $lastRows2 then goto step6 endi $lastRows2 = $data00 -sql select count(*) from db3.tb3 +sql select count(*) from db3.tb3 -x step6 print select count(*) from db3.tb3 ==> $data00 $lastRows3 if $data00 <= $lastRows3 then goto step6 endi $lastRows3 = $data00 -sql select count(*) from db4.tb4 +sql select count(*) from db4.tb4 -x step6 print select count(*) from db4.tb4 ==> $data00 $lastRows4 if $data00 <= $lastRows4 then goto step6 From 04d814644be103a289f48c3232577e6679b97fce Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 18 Nov 2022 10:48:54 +0800 Subject: [PATCH 34/95] test: adjust case --- tests/script/tsim/vnode/replica3_many.sim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/script/tsim/vnode/replica3_many.sim b/tests/script/tsim/vnode/replica3_many.sim index cac787d9f1..e3c73b2018 100644 --- a/tests/script/tsim/vnode/replica3_many.sim +++ b/tests/script/tsim/vnode/replica3_many.sim @@ -201,11 +201,11 @@ system sh/exec.sh -n dnode2 -s start sleep 3000 print ======== step6 -$x = 0 +$y = 0 step6: - $x = $x + 1 + $y = $y + 1 sleep 1000 - if $x == 50 then + if $y == 50 then return -1 endi From 3306785e60145623791209c176ef715b3f80ff2b Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 18 Nov 2022 11:06:35 +0800 Subject: [PATCH 35/95] fix(sync): fix elect bug --- source/libs/sync/src/syncMain.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 369847b672..7444781f4c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1946,12 +1946,14 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { SSyncHbTimerData* pData = syncHbTimerDataAcquire(hbDataRid); if (pData == NULL) { + sError("hb timer get pData NULL, %" PRId64, hbDataRid); return; } SSyncNode* pSyncNode = syncNodeAcquire(pData->syncNodeRid); if (pSyncNode == NULL) { syncHbTimerDataRelease(pData); + sError("hb timer get pSyncNode NULL"); return; } @@ -1960,28 +1962,39 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { if (!pSyncNode->isStart) { syncNodeRelease(pSyncNode); syncHbTimerDataRelease(pData); + sError("vgId:%d, hb timer sync node already stop", pSyncNode->vgId); return; } if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { syncNodeRelease(pSyncNode); syncHbTimerDataRelease(pData); + sError("vgId:%d, hb timer sync node not leader", pSyncNode->vgId); return; } if (pSyncNode->pRaftStore == NULL) { syncNodeRelease(pSyncNode); syncHbTimerDataRelease(pData); + sError("vgId:%d, hb timer raft store already stop", pSyncNode->vgId); return; } - // sNTrace(pSyncNode, "eq peer hb timer"); - - int64_t timerLogicClock = atomic_load_64(&pSyncTimer->logicClock); - int64_t msgLogicClock = atomic_load_64(&pData->logicClock); + // sTrace("vgId:%d, eq peer hb timer", pSyncNode->vgId); if (pSyncNode->replicaNum > 1) { + int64_t timerLogicClock = atomic_load_64(&pSyncTimer->logicClock); + int64_t msgLogicClock = atomic_load_64(&pData->logicClock); + if (timerLogicClock == msgLogicClock) { + if (syncIsInit()) { + // sTrace("vgId:%d, reset peer hb timer", pSyncNode->vgId); + taosTmrReset(syncNodeEqPeerHeartbeatTimer, pSyncTimer->timerMS, (void*)hbDataRid, syncEnv()->pTimerManager, + &pSyncTimer->pTimer); + } else { + sError("sync env is stop, reset peer hb timer error"); + } + SRpcMsg rpcMsg = {0}; (void)syncBuildHeartbeat(&rpcMsg, pSyncNode->vgId); @@ -1996,16 +2009,9 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { // send msg syncNodeSendHeartbeat(pSyncNode, &pSyncMsg->destId, &rpcMsg); - if (syncIsInit()) { - taosTmrReset(syncNodeEqPeerHeartbeatTimer, pSyncTimer->timerMS, pData, syncEnv()->pTimerManager, - &pSyncTimer->pTimer); - } else { - sError("sync env is stop, syncNodeEqHeartbeatTimer"); - } - } else { - sTrace("==syncNodeEqPeerHeartbeatTimer== timerLogicClock:%" PRId64 ", msgLogicClock:%" PRId64 "", timerLogicClock, - msgLogicClock); + sTrace("vgId:%d, do not send hb, timerLogicClock:%" PRId64 ", msgLogicClock:%" PRId64 "", pSyncNode->vgId, + timerLogicClock, msgLogicClock); } } From 875989aa309196218519f16222e8c2653b93dede Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 18 Nov 2022 11:24:56 +0800 Subject: [PATCH 36/95] refactor: do some internal refactor. --- source/libs/executor/inc/executorimpl.h | 10 ++--- source/libs/executor/src/executorimpl.c | 57 +------------------------ source/libs/executor/src/scanoperator.c | 18 ++++---- 3 files changed, 13 insertions(+), 72 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 03dbca8668..be52a10dcb 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -163,7 +163,7 @@ typedef struct { SArray* pStopInfo; } STaskStopInfo; -typedef struct SExecTaskInfo { +struct SExecTaskInfo { STaskIdInfo id; uint32_t status; STimeWindow window; @@ -182,7 +182,7 @@ typedef struct SExecTaskInfo { struct SOperatorInfo* pRoot; SLocalFetch localFetch; STaskStopInfo stopInfo; -} SExecTaskInfo; +}; enum { OP_NOT_OPENED = 0x0, @@ -343,7 +343,6 @@ typedef struct STableScanInfo { } STableScanInfo; typedef struct STableMergeScanInfo { - STableListInfo* tableListInfo; int32_t tableStartIndex; int32_t tableEndIndex; bool hasGroupId; @@ -363,7 +362,6 @@ typedef struct STableMergeScanInfo { int32_t scanTimes; SSDataBlock* pResBlock; int32_t numOfOutput; - int32_t dataBlockLoadFlag; // if the upstream is an interval operator, the interval info is also kept here to get the time // window to check if current data block needs to be loaded. @@ -1041,8 +1039,8 @@ int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPos SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo, - SReadHandle* readHandle, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, + SExecTaskInfo* pTaskInfo); void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index c36175cd6a..2b4b274f97 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1000,12 +1000,6 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc return TSDB_CODE_SUCCESS; } -static void updateTableQueryInfoForReverseScan(STableQueryInfo* pTableQueryInfo) { - if (pTableQueryInfo == NULL) { - return; - } -} - void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { if (status == TASK_NOT_COMPLETED) { pTaskInfo->status = status; @@ -1665,55 +1659,6 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, const char* pKey); -static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char** buf, int32_t rowIndex) { - size_t size = taosArrayGetSize(groupInfo); - if (size == 0) { - return true; - } - - for (int32_t i = 0; i < size; ++i) { - int32_t* index = taosArrayGet(groupInfo, i); - - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, *index); - bool isNull = colDataIsNull(pColInfo, rowIndex, pBlock->info.rows, NULL); - - if ((isNull && buf[i] != NULL) || (!isNull && buf[i] == NULL)) { - return false; - } - - char* pCell = colDataGetData(pColInfo, rowIndex); - if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { - if (varDataLen(pCell) != varDataLen(buf[i])) { - return false; - } else { - if (memcmp(varDataVal(pCell), varDataVal(buf[i]), varDataLen(pCell)) != 0) { - return false; - } - } - } else { - if (memcmp(pCell, buf[i], pColInfo->info.bytes) != 0) { - return false; - } - } - } - - return 0; -} - -static bool saveCurrentTuple(char** rowColData, SArray* pColumnList, SSDataBlock* pBlock, int32_t rowIndex) { - int32_t size = (int32_t)taosArrayGetSize(pColumnList); - - for (int32_t i = 0; i < size; ++i) { - int32_t* index = taosArrayGet(pColumnList, i); - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, *index); - - char* data = colDataGetData(pColInfo, rowIndex); - memcpy(rowColData[i], data, colDataGetLength(pColInfo, rowIndex)); - } - - return true; -} - int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag) { // todo add more information about exchange operation int32_t type = pOperator->operatorType; @@ -2762,7 +2707,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } - pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo); + pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo); if (NULL == pOperator) { pTaskInfo->code = terrno; return NULL; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ee1114f330..8e8a86670b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -4392,7 +4392,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { int64_t st = taosGetTimestampUs(); - void* p = tableListGetInfo(pInfo->tableListInfo, readIdx + pInfo->tableStartIndex); + void* p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex); SReadHandle* pHandle = &pInfo->base.readHandle; int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, &pInfo->base.dataReader, GET_TASKID(pTaskInfo)); @@ -4486,10 +4486,10 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; { - size_t numOfTables = tableListGetSize(pInfo->tableListInfo); + size_t numOfTables = tableListGetSize(pTaskInfo->pTableInfoList); int32_t i = pInfo->tableStartIndex + 1; for (; i < numOfTables; ++i) { - STableKeyInfo* tableKeyInfo = tableListGetInfo(pInfo->tableListInfo, i); + STableKeyInfo* tableKeyInfo = tableListGetInfo(pTaskInfo->pTableInfoList, i); if (tableKeyInfo->groupId != pInfo->groupId) { break; } @@ -4613,7 +4613,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, code); } - size_t tableListSize = tableListGetSize(pInfo->tableListInfo); + size_t tableListSize = tableListGetSize(pTaskInfo->pTableInfoList); if (!pInfo->hasGroupId) { pInfo->hasGroupId = true; @@ -4622,7 +4622,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { return NULL; } pInfo->tableStartIndex = 0; - pInfo->groupId = ((STableKeyInfo*)tableListGetInfo(pInfo->tableListInfo, pInfo->tableStartIndex))->groupId; + pInfo->groupId = ((STableKeyInfo*)tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->tableStartIndex))->groupId; startGroupTableMergeScan(pOperator); } @@ -4641,7 +4641,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { break; } pInfo->tableStartIndex = pInfo->tableEndIndex + 1; - pInfo->groupId = tableListGetInfo(pInfo->tableListInfo, pInfo->tableStartIndex)->groupId; + pInfo->groupId = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->tableStartIndex)->groupId; startGroupTableMergeScan(pOperator); } } @@ -4698,8 +4698,8 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla return TSDB_CODE_SUCCESS; } -SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo, - SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, + SExecTaskInfo* pTaskInfo) { STableMergeScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableMergeScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -4733,14 +4733,12 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->interval = extractIntervalInfo(pTableScanNode); pInfo->sample.sampleRatio = pTableScanNode->ratio; pInfo->sample.seed = taosGetTimestampSec(); - pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired; code = filterInitFromNode((SNode*)pTableScanNode->scan.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); if (code != TSDB_CODE_SUCCESS) { goto _error; } - pInfo->tableListInfo = pTableListInfo; pInfo->base.scanFlag = MAIN_SCAN; initResultSizeInfo(&pOperator->resultInfo, 1024); From 1339e50a906292bc4f628e589988003c91298acd Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Fri, 18 Nov 2022 11:37:46 +0800 Subject: [PATCH 37/95] docs: correct error in taos.cfg --- packaging/cfg/taos.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index eeb089d1ab..e22aa85c97 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -73,10 +73,10 @@ # compressColData -1 # system time zone -# timezone Asia/Shanghai (CST, +0800) +# timezone UTC-8 # system time zone (for windows 10) -# timezone UTC-8 +# timezone Asia/Shanghai (CST, +0800) # system locale # locale en_US.UTF-8 @@ -179,4 +179,4 @@ # metaDebugFlag 131 # generate core file when service crash -# enableCoreFile 1 \ No newline at end of file +# enableCoreFile 1 From c2d4d0907d85389bfbe5add37c84446616dafdc9 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 18 Nov 2022 11:41:59 +0800 Subject: [PATCH 38/95] fix: fix asan issue --- source/libs/scalar/src/filter.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 4befe29e1d..cf2653a9e9 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3248,14 +3248,18 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, SColumnInfoData *pRes, SC for (uint32_t u = 0; u < group->unitNum; ++u) { uint32_t uidx = group->unitIdxs[u]; SFilterComUnit *cunit = &info->cunits[uidx]; - void *colData = colDataGetData((SColumnInfoData *)(cunit->colData), i); - + void *colData = NULL; + bool isNull = colDataIsNull((SColumnInfoData *)(cunit->colData), 0, i, NULL); // if (FILTER_UNIT_GET_F(info, uidx)) { // p[i] = FILTER_UNIT_GET_R(info, uidx); // } else { uint8_t optr = cunit->optr; - if (colData == NULL || colDataIsNull((SColumnInfoData *)(cunit->colData), 0, i, NULL)) { + if (!isNull) { + colData = colDataGetData((SColumnInfoData *)(cunit->colData), i); + } + + if (colData == NULL || isNull) { p[i] = optr == OP_TYPE_IS_NULL ? true : false; } else { if (optr == OP_TYPE_IS_NOT_NULL) { From 83815bc005bc315da23b15347bacca74d3e67e72 Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Fri, 18 Nov 2022 11:43:39 +0800 Subject: [PATCH 39/95] docs: update tmq demo Close [TD-20499](https://jira.taosdata.com:18080/browse/TD-20499) --- docs/examples/rust/nativeexample/Cargo.toml | 2 +- .../nativeexample/examples/stmt_example.rs | 9 ++++--- .../nativeexample/examples/subscribe_demo.rs | 26 ++++++------------- .../restexample/examples/insert_example.rs | 1 - 4 files changed, 15 insertions(+), 23 deletions(-) diff --git a/docs/examples/rust/nativeexample/Cargo.toml b/docs/examples/rust/nativeexample/Cargo.toml index cdf739d357..5ecc407854 100644 --- a/docs/examples/rust/nativeexample/Cargo.toml +++ b/docs/examples/rust/nativeexample/Cargo.toml @@ -10,4 +10,4 @@ chrono = "0.4" serde = { version = "1", features = ["derive"] } tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } -taos = { version = "0.*" } +taos = { version = "0.4.8" } diff --git a/docs/examples/rust/nativeexample/examples/stmt_example.rs b/docs/examples/rust/nativeexample/examples/stmt_example.rs index 9cf8e8e1fc..7d5a7c0f2b 100644 --- a/docs/examples/rust/nativeexample/examples/stmt_example.rs +++ b/docs/examples/rust/nativeexample/examples/stmt_example.rs @@ -12,7 +12,10 @@ async fn main() -> anyhow::Result<()> { // bind table name and tags stmt.set_tbname_tags( "d1001", - &[Value::VarChar("California.SanFransico".into()), Value::Int(2)], + &[ + Value::VarChar("California.SanFransico".into()), + Value::Int(2), + ], )?; // bind values. let values = vec![ @@ -30,9 +33,9 @@ async fn main() -> anyhow::Result<()> { ColumnView::from_floats(vec![0.33]), ]; stmt.bind(&values2)?; - + stmt.add_batch()?; - + // execute. let rows = stmt.execute()?; assert_eq!(rows, 2); diff --git a/docs/examples/rust/nativeexample/examples/subscribe_demo.rs b/docs/examples/rust/nativeexample/examples/subscribe_demo.rs index 11d6d4e004..7551ad46b1 100644 --- a/docs/examples/rust/nativeexample/examples/subscribe_demo.rs +++ b/docs/examples/rust/nativeexample/examples/subscribe_demo.rs @@ -50,7 +50,7 @@ async fn main() -> anyhow::Result<()> { // create super table format!("CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) TAGS (`groupid` INT, `location` BINARY(24))"), // create topic for subscription - format!("CREATE TOPIC tmq_meters with META AS DATABASE {db}") + format!("CREATE TOPIC tmq_meters AS SELECT * FROM `meters`") ]) .await?; @@ -64,13 +64,9 @@ async fn main() -> anyhow::Result<()> { let mut consumer = tmq.build()?; consumer.subscribe(["tmq_meters"]).await?; - { - let mut stream = consumer.stream(); - - while let Some((offset, message)) = stream.try_next().await? { - // get information from offset - - // the topic + consumer + .stream() + .try_for_each(|(offset, message)| async { let topic = offset.topic(); // the vgroup id, like partition id in kafka. let vgroup_id = offset.vgroup_id(); @@ -78,20 +74,14 @@ async fn main() -> anyhow::Result<()> { if let Some(data) = message.into_data() { while let Some(block) = data.fetch_raw_block().await? { - // one block for one table, get table name if needed - let name = block.table_name(); let records: Vec = block.deserialize().try_collect()?; - println!( - "** table: {}, got {} records: {:#?}\n", - name.unwrap(), - records.len(), - records - ); + println!("** read {} records: {:#?}\n", records.len(), records); } } consumer.commit(offset).await?; - } - } + Ok(()) + }) + .await?; consumer.unsubscribe().await; diff --git a/docs/examples/rust/restexample/examples/insert_example.rs b/docs/examples/rust/restexample/examples/insert_example.rs index 11a84f1661..4953a09b35 100644 --- a/docs/examples/rust/restexample/examples/insert_example.rs +++ b/docs/examples/rust/restexample/examples/insert_example.rs @@ -5,7 +5,6 @@ async fn main() -> anyhow::Result<()> { let dsn = "ws://"; let taos = TaosBuilder::from_dsn(dsn)?.build()?; - taos.exec_many([ "DROP DATABASE IF EXISTS power", "CREATE DATABASE power", From 1b5195e083a14af6c7996754106208580fea0c19 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 18 Nov 2022 12:13:58 +0800 Subject: [PATCH 40/95] test: add tsan case --- tests/parallel_test/cases.task | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index aa1e58b5d5..54877273cf 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -11,7 +11,7 @@ ,,y,script,./test.sh -f tsim/user/privilege_db.sim ,,y,script,./test.sh -f tsim/user/privilege_sysinfo.sim ,,y,script,./test.sh -f tsim/db/alter_option.sim -#,,y,script,./test.sh -f tsim/db/alter_replica_13.sim +,,y,script,./test.sh -f tsim/db/alter_replica_13.sim ,,y,script,./test.sh -f tsim/db/alter_replica_31.sim ,,y,script,./test.sh -f tsim/db/basic1.sim ,,y,script,./test.sh -f tsim/db/basic2.sim @@ -48,7 +48,7 @@ ,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim ,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim ,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim -,,,script,./test.sh -f tsim/dnode/drop_dnode_force.sim +,,y,script,./test.sh -f tsim/dnode/drop_dnode_force.sim ,,y,script,./test.sh -f tsim/dnode/offline_reason.sim ,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim ,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim From 8b1e13af5ee6c831aa7453c8ef946269f6fe956f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 18 Nov 2022 13:40:03 +0800 Subject: [PATCH 41/95] refactor: do some internal refactor. --- source/libs/executor/inc/executorimpl.h | 42 +++++++++---------------- source/libs/executor/src/scanoperator.c | 27 +++++++--------- 2 files changed, 26 insertions(+), 43 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index be52a10dcb..b4e786746b 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -325,16 +325,16 @@ typedef struct STableScanBase { SExprSupp pseudoSup; STableMetaCacheInfo metaCache; int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan + int32_t dataBlockLoadFlag; + SLimitInfo limitInfo; } STableScanBase; typedef struct STableScanInfo { STableScanBase base; - SLimitInfo limitInfo; SScanInfo scanInfo; int32_t scanTimes; SSDataBlock* pResBlock; int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan - int32_t dataBlockLoadFlag; SSampleExecInfo sample; // sample execution info int32_t currentGroupId; int32_t currentTable; @@ -361,13 +361,8 @@ typedef struct STableMergeScanInfo { SScanInfo scanInfo; int32_t scanTimes; SSDataBlock* pResBlock; - int32_t numOfOutput; - - // if the upstream is an interval operator, the interval info is also kept here to get the time - // window to check if current data block needs to be loaded. - SInterval interval; - SSampleExecInfo sample; // sample execution info - SSortExecInfo sortExecInfo; + SSampleExecInfo sample; // sample execution info + SSortExecInfo sortExecInfo; } STableMergeScanInfo; typedef struct STagScanInfo { @@ -380,17 +375,17 @@ typedef struct STagScanInfo { } STagScanInfo; typedef struct SLastrowScanInfo { - SSDataBlock* pRes; - SReadHandle readHandle; - void* pLastrowReader; - SColMatchInfo matchInfo; - int32_t* pSlotIds; - SExprSupp pseudoExprSup; - int32_t retrieveType; - int32_t currentGroupIndex; - SSDataBlock* pBufferredRes; - SArray* pUidList; - int32_t indexOfBufferedRes; + SSDataBlock* pRes; + SReadHandle readHandle; + void* pLastrowReader; + SColMatchInfo matchInfo; + int32_t* pSlotIds; + SExprSupp pseudoExprSup; + int32_t retrieveType; + int32_t currentGroupIndex; + SSDataBlock* pBufferredRes; + SArray* pUidList; + int32_t indexOfBufferedRes; } SLastrowScanInfo; typedef enum EStreamScanMode { @@ -407,13 +402,6 @@ enum { PROJECT_RETRIEVE_DONE = 0x2, }; -typedef struct SCatchSupporter { - SHashObj* pWindowHashTable; // quick locate the window object for each window - SDiskbasedBuf* pDataBuf; // buffer based on blocked-wised disk file - int32_t keySize; - int64_t* pKeyBuf; -} SCatchSupporter; - typedef struct SStreamAggSupporter { int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row SSDataBlock* pScanBlock; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8e8a86670b..26117b442e 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -374,23 +374,16 @@ void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo } } - - - - static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - STableScanInfo* pInfo = pOperator->info; - SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder; pCost->totalBlocks += 1; pCost->totalRows += pBlock->info.rows; bool loadSMA = false; - - *status = pInfo->dataBlockLoadFlag; + *status = pTableScanInfo->dataBlockLoadFlag; if (pOperator->exprSupp.pFilterInfo != NULL || overlapWithTimeWindow(&pTableScanInfo->pdInfo.interval, &pBlock->info, pTableScanInfo->cond.order)) { (*status) = FUNC_DATA_REQUIRED_DATA_LOAD; @@ -489,10 +482,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca } } - applyLimitOffset(&pInfo->limitInfo, pBlock, pTaskInfo, pOperator); + applyLimitOffset(&pTableScanInfo->limitInfo, pBlock, pTaskInfo, pOperator); pCost->totalRows += pBlock->info.rows; - pInfo->limitInfo.numOfOutputRows = pCost->totalRows; + pTableScanInfo->limitInfo.numOfOutputRows = pCost->totalRows; return TSDB_CODE_SUCCESS; } @@ -871,8 +864,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { // reset value for the next group data output pOperator->status = OP_OPENED; - pInfo->limitInfo.numOfOutputRows = 0; - pInfo->limitInfo.remainOffset = pInfo->limitInfo.limit.offset; + pInfo->base.limitInfo.numOfOutputRows = 0; + pInfo->base.limitInfo.remainOffset = pInfo->base.limitInfo.limit.offset; int32_t num = 0; STableKeyInfo* pList = NULL; @@ -936,7 +929,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, goto _error; } - initLimitInfo(pScanNode->node.pLimit, pScanNode->node.pSlimit, &pInfo->limitInfo); + initLimitInfo(pScanNode->node.pLimit, pScanNode->node.pSlimit, &pInfo->base.limitInfo); code = initQueryTableDataCond(&pInfo->base.cond, pTableScanNode); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -954,7 +947,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->sample.sampleRatio = pTableScanNode->ratio; pInfo->sample.seed = taosGetTimestampSec(); - pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired; + pInfo->base.dataBlockLoadFlag = pTableScanNode->dataRequired; initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pResBlock = createResDataBlock(pDescNode); @@ -4729,8 +4722,11 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; + pInfo->base.dataBlockLoadFlag = FUNC_DATA_REQUIRED_DATA_LOAD; + pInfo->base.scanFlag = MAIN_SCAN; pInfo->base.readHandle = *readHandle; - pInfo->interval = extractIntervalInfo(pTableScanNode); + initLimitInfo(pTableScanNode->scan.node.pLimit, pTableScanNode->scan.node.pSlimit, &pInfo->base.limitInfo); + pInfo->sample.sampleRatio = pTableScanNode->ratio; pInfo->sample.seed = taosGetTimestampSec(); @@ -4739,7 +4735,6 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN goto _error; } - pInfo->base.scanFlag = MAIN_SCAN; initResultSizeInfo(&pOperator->resultInfo, 1024); pInfo->pResBlock = createResDataBlock(pDescNode); From 371b35fcbc000fb83ce4ddd8055433d3375f633a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Fri, 18 Nov 2022 14:23:20 +0800 Subject: [PATCH 42/95] test: refine query cases --- tests/system-test/2-query/case_when.py | 338 +++++++++++++++++++++++++ 1 file changed, 338 insertions(+) create mode 100755 tests/system-test/2-query/case_when.py diff --git a/tests/system-test/2-query/case_when.py b/tests/system-test/2-query/case_when.py new file mode 100755 index 0000000000..cfe0399553 --- /dev/null +++ b/tests/system-test/2-query/case_when.py @@ -0,0 +1,338 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import os +import time +import taos +import subprocess +from faker import Faker +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes +from util.dnodes import * + +class TDTestCase: + updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"querySmaOptimize":1} + + def init(self, conn, logSql, replicaVar): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.testcasePath = os.path.split(__file__)[0] + self.testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) + + self.db = "case_when" + + def dropandcreateDB_random(self,database,n): + ts = 1630000000000 + num_random = 10 + fake = Faker('zh_CN') + tdSql.execute('''drop database if exists %s ;''' %database) + tdSql.execute('''create database %s keep 36500 ;'''%(database)) + tdSql.execute('''use %s;'''%database) + + tdSql.execute('''create stable %s.stable_1 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , \ + q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) \ + tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);'''%database) + tdSql.execute('''create stable %s.stable_2 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , \ + q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) \ + tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);'''%database) + + for i in range(num_random): + tdSql.execute('''create table %s.table_%d \ + (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp ) ;'''%(database,i)) + tdSql.execute('''create table %s.stable_1_%d using %s.stable_1 tags('stable_1_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(database,i,database,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + + tdSql.execute('''create table %s.stable_%d_a using %s.stable_2 tags('stable_2_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(database,i,database,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + tdSql.execute('''create table %s.stable_%d_b using %s.stable_2 tags('stable_2_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(database,i,database,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + + # insert data + for i in range(num_random): + for j in range(n): + tdSql.execute('''insert into %s.stable_1_%d (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts)\ + values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d) ;''' + % (database,i,ts + i*1000 + j, fake.random_int(min=-2147483647, max=2147483647, step=1), + fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i)) + + tdSql.execute('''insert into %s.table_%d (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) \ + values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d) ;''' + % (database,i,ts + i*1000 + j, fake.random_int(min=-2147483647, max=2147483647, step=1), + fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i )) + + tdSql.execute('''insert into %s.stable_%d_a (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts)\ + values(%d, %d, %d, %d, %d, %f, %f, 1, 'binary.%s', 'nchar.%s', %d) ;''' + % (database,i,ts + i*1000 + j, fake.random_int(min=0, max=2147483647, step=1), + fake.random_int(min=0, max=9223372036854775807, step=1), + fake.random_int(min=0, max=32767, step=1) , fake.random_int(min=0, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i)) + + tdSql.query("select count(*) from %s.stable_1;" %database) + tdSql.checkData(0,0,num_random*n) + tdSql.query("select count(*) from %s.table_0;"%database) + tdSql.checkData(0,0,n) + + + def users_bug(self,database): + sql1 = "select (case when `q_smallint` >0 then 'many--' when `q_smallint`<0 then 'little' end),q_int,loc from %s.stable_1 where tbname = 'stable_1_1' limit 100;" %database + sql2 = "select (case when `q_smallint` >0 then 'many--' when `q_smallint`<0 then 'little' end),q_int,loc from %s.stable_1_1 limit 100;" %database + self.constant_check(database,sql1,sql2,0) + + sql1 = "select (case when `q_smallint` >0 then 'many![;;][][]]' when `q_smallint`<0 then 'little' end),q_int,loc from %s.stable_1 where tbname = 'stable_1_1' limit 100;" %database + sql2 = "select (case when `q_smallint` >0 then 'many![;;][][]]' when `q_smallint`<0 then 'little' end),q_int,loc from %s.stable_1_1 limit 100;" %database + self.constant_check(database,sql1,sql2,0) + + sql1 = "select (case when sum(q_smallint)=0 then null else sum(q_smallint) end) from %s.stable_1 where tbname = 'stable_1_1' limit 100;" %database + sql2 = "select (case when sum(q_smallint)=0 then null else sum(q_smallint) end) from %s.stable_1_1 limit 100;" %database + self.constant_check(database,sql1,sql2,0) + + sql1 = "select _wstart,avg(q_int),min(q_smallint) from %s.stable_1 where tbname = 'stable_1_1' and ts < now state_window(case when q_smallint <0 then 1 else 0 end);" %database + sql2 = "select _wstart,avg(q_int),min(q_smallint) from %s.stable_1_1 where ts < now state_window(case when q_smallint <0 then 1 else 0 end);" %database + self.constant_check(database,sql1,sql2,0) + self.constant_check(database,sql1,sql2,1) + self.constant_check(database,sql1,sql2,2) + + def casewhen_list(self): + a1,a2,a3 = random.randint(-2147483647,2147483647),random.randint(-2147483647,2147483647),random.randint(-2147483647,2147483647) + casewhen_lists = ['first case when %d then %d end last' %(a1,a2) , #'first case when 3 then 4 end last' , + 'first case when 0 then %d end last' %(a1), #'first case when 0 then 4 end last' , + 'first case when null then %d end last' %(a1) , #'first case when null then 4 end last' , + 'first case when 1 then %d+(%d) end last' %(a1,a2) , #'first case when 1 then 4+1 end last' , + 'first case when %d-(%d) then 0 end last' %(a1,a1) , #'first case when 1-1 then 0 end last' , + 'first case when %d+(%d) then 0 end last' %(a1,a1), #'first case when 1+1 then 0 end last' , + 'first case when 1 then %d-(%d)+(%d) end last' %(a1,a1,a2), #'first case when 1 then 1-1+2 end last' , + 'first case when %d > 0 then %d < %d end last' %(a1,a1,a2), #'first case when 1 > 0 then 1 < 2 end last' , + 'first case when %d > %d then %d < %d end last' %(a1,a2,a1,a2), #'first case when 1 > 2 then 1 < 2 end last' , + 'first case when abs(%d) then abs(-(%d)) end last' %(a1,a2) ,#'first case when abs(3) then abs(-1) end last' , + 'first case when abs(%d+(%d)) then abs(-(%d))+abs(%d) end last' %(a1,a2,a1,a2) , #'first case when abs(1+1) then abs(-1)+abs(3) end last' , + 'first case when 0 then %d else %d end last' %(a1,a2), #'first case when 0 then 1 else 3 end last' , + 'first case when 0 then %d when 1 then %d else %d end last' %(a1,a1,a3), #'first case when 0 then 1 when 1 then 0 else 3 end last' , + 'first case when 0 then %d when 1 then %d when 2 then %d end last' %(a1,a1,a3), #'first case when 0 then 1 when 1 then 0 when 2 then 3 end last' , + 'first case when \'a\' then \'b\' when null then 0 end last' , #'first case when \'a\' then \'b\' when null then 0 end last' , + 'first case when \'%d\' then \'b\' when null then %d end last' %(a1,a2), #'first case when \'2\' then \'b\' when null then 0 end last' , + 'first case when \'%d\' then \'b\' else null end last' %(a1), #'first case when \'0\' then \'b\' else null end last', + 'first case when \'%d\' then \'b\' else %d end last' %(a1,a2), #'first case when \'0\' then \'b\' else 2 end last', + 'first case when sum(%d) then sum(%d)-sum(%d) end last' %(a1,a1,a3), #'first case when sum(2) then sum(2)-sum(1) end last' , + 'first case when sum(%d) then abs(-(%d)) end last' %(a1,a2), #'first case when sum(2) then abs(-2) end last' , + 'first case when q_int then ts end last' , + 'first case when q_int then q_int when q_int + (%d) then q_int + (%d) else q_int is null end last' %(a1,a2) , #'first case when q_int then q_int when q_int + 1 then q_int + 1 else q_int is null end last' , + 'first case when q_int then %d when ts then ts end last' %(a1), #'first case when q_int then 3 when ts then ts end last' , + 'first case when %d then q_int end last' %(a1), #'first case when 3 then q_int end last' , + 'first case when q_int then %d when %d then %d end last' %(a1,a1,a3), #'first case when q_int then 3 when 1 then 2 end last' , + 'first case when sum(q_int) then sum(q_int)-abs(-(%d)) end last' %(a1), #'first case when sum(q_int) then sum(q_int)-abs(-1) end last' , + 'first case when q_int < %d then %d when q_int >= %d then %d else %d end last' %(a1,a2,a1,a2,a3), #'first case when q_int < 3 then 1 when q_int >= 3 then 2 else 3 end last' , + 'first cast(case q_int when q_int then q_int + (%d) else q_int is null end as double) last' %(a1), #'first cast(case q_int when q_int then q_int + 1 else q_int is null end as double) last' , + 'first sum(case q_int when q_int then q_int + (%d) else q_int is null end + (%d)) last' %(a1,a2), #'first sum(case q_int when q_int then q_int + 1 else q_int is null end + 1) last' , + 'first case when q_int is not null then case when q_int <= %d then q_int else q_int * (%d) end else -(%d) end last' %(a1,a1,a3), #'first case when q_int is not null then case when q_int <= 0 then q_int else q_int * 10 end else -1 end last' , + 'first case %d when %d then %d end last' %(a1,a2,a3), # 'first case 3 when 3 then 4 end last' , + 'first case %d when %d then %d end last' %(a1,a2,a3), # 'first case 3 when 1 then 4 end last' , + 'first case %d when %d then %d else %d end last' %(a1,a1,a2,a3), # 'first case 3 when 1 then 4 else 2 end last' , + 'first case %d when null then %d when \'%d\' then %d end last' %(a1,a1,a2,a3) , # 'first case 3 when null then 4 when \'3\' then 1 end last' , + 'first case \'%d\' when null then %d when %d then %d end last' %(a1,a1,a2,a3), # 'first case \'3\' when null then 4 when 3 then 1 end last' , + 'first case null when null then %d when %d then %d end last' %(a1,a2,a3), # 'first case null when null then 4 when 3 then 1 end last' , + 'first case %d.0 when null then %d when \'%d\' then %d end last' %(a1,a1,a2,a3) , # 'first case 3.0 when null then 4 when \'3\' then 1 end last' , + 'first case q_double when \'a\' then %d when \'%d\' then %d end last' %(a1,a2,a3) , # 'first case q_double when \'a\' then 4 when \'0\' then 1 end last' , + 'first case q_double when q_int then q_int when q_int - (%d) then q_int else %d end last' %(a1,a2), # 'first case q_double when q_int then q_int when q_int - 1 then q_int else 99 end last' , + 'first case cast(q_double as int) when %d then q_double when q_int then %d else ts end last' %(a1,a2), #'first case cast(q_double as int) when 0 then q_double when q_int then 11 else ts end last' , + 'first case q_int + (%d) when %d then %d when %d then %d else %d end last' %(a1,a2,a3,a1,a2,a3), #'first case q_int + 1 when 1 then 1 when 2 then 2 else 3 end last' , + 'first case when \'a\' then \'b\' when null then %d end last' %(a1), # 'first case when \'a\' then \'b\' when null then 0 end last' , + 'first case when \'%d\' then \'b\' when null then %d end last' %(a1,a2), # 'first case when \'2\' then \'b\' when null then 0 end last' , + 'first case when %d then \'b\' else null end last' %(a1), # 'first case when 0 then \'b\' else null end last' , + 'first case when %d then \'b\' else %d+abs(%d) end last' %(a1,a2,a3), # 'first case when 0 then \'b\' else 2+abs(-2) end last' , + 'first case when %d then %d end last' %(a1,a2), # 'first case when 3 then 4 end last' , + 'first case when %d then %d end last' %(a1,a2), # 'first case when 0 then 4 end last' , + 'first case when null then %d end last' %(a1), # 'first case when null then 4 end last' , + 'first case when %d then %d+(%d) end last' %(a1,a2,a3), # 'first case when 1 then 4+1 end last' , + 'first case when %d-(%d) then %d end last' %(a1,a2,a3), # 'first case when 1-1 then 0 end last' , + 'first case when %d+(%d) then %d end last' %(a1,a2,a3), # 'first case when 1+1 then 0 end last' , + 'first case when abs(%d) then abs(%d) end last' %(a1,a2), # 'first case when abs(3) then abs(-1) end last' , + 'first case when abs(%d+(%d)) then abs(%d)+abs(%d) end last' %(a1,a2,a3,a1), # 'first case when abs(1+1) then abs(-1)+abs(3) end last' , + 'first case when %d then %d else %d end last' %(a1,a2,a3), # 'first case when 0 then 1 else 3 end last' , + 'first case when %d then %d when %d then %d else %d end last' %(a1,a2,a3,a1,a2), # 'first case when 0 then 1 when 1 then 0 else 3 end last' , + 'first case when %d then %d when %d then %d when %d then %d end last' %(a1,a2,a3,a1,a2,a3), # 'first case when 0 then 1 when 1 then 0 when 2 then 3 end last' , + 'first case %d when %d then %d end last' %(a1,a1,a3), # 'first case 3 when 3 then 4 end last' , + 'first case %d when %d then %d end last' %(a1,a2,a3), # 'first case 3 when 1 then 4 end last' , + 'first case %d when %d then %d else %d end last' %(a1,a2,a3,a1), # 'first case 3 when 1 then 4 else 2 end last' , + 'first case %d when null then %d when \'%d\' then %d end last' %(a1,a2,a1,a3), # 'first case 3 when null then 4 when \'3\' then 1 end last' , + 'first case null when null then %d when %d then %d end last' %(a1,a2,a3), # 'first case null when null then 4 when 3 then 1 end last' , + 'first case %d.0 when null then %d when \'%d\' then %d end last' %(a1,a2,a1,a3), # 'first case 3.0 when null then 4 when \'3\' then 1 end last' , + 'first q_double,case q_double when \'a\' then %d when \'%d\' then %d end last' %(a1,a2,a3), #'first q_double,case q_double when \'a\' then 4 when \'0\' then 1 end last' , + 'first case null when null then %d when %d then %d end last' %(a1,a2,a3), #'first case null when null then 4 when 3 then 1 end last' , + 'first q_double,q_int,case q_double when q_int then q_int when q_int - (%d ) then q_int else %d end last' %(a1,a2), # 'first q_double,q_int,case q_double when q_int then q_int when q_int - 1 then q_int else 99 end last' , + 'first case cast(q_double as int) when %d then q_double when q_int then %d else ts end last' %(a1,a2), # 'first case cast(q_double as int) when 0 then q_double when q_int then 11 else ts end last' , + 'first q_int, case q_int + (%d) when %d then %d when %d then %d else %d end last' %(a1,a1,a1,a2,a2,a3), #'first q_int, case q_int + 1 when 1 then 1 when 2 then 2 else 3 end last' , + 'first distinct loc, case t_int when t_bigint then t_ts else t_smallint + (%d) end last' %(a1), #'first distinct loc, case t_int when t_bigint then t_ts else t_smallint + 100 end last' , + ] + #num = len(casewhen_lists) + + casewhen_list = str(random.sample(casewhen_lists,50)).replace("[","").replace("]","").replace("'first","").replace("last'","").replace("\"first","").replace("last\"","") + + return casewhen_list + + def base_case(self,database): + + for i in range(30): + cs = self.casewhen_list().split(',')[i] + sql1 = "select %s from %s.stable_1 where tbname = 'stable_1_1';" % (cs ,database) + sql2 = "select %s from %s.stable_1_1 ;" % (cs ,database) + self.constant_check(database,sql1,sql2,0) + + + def state_window_list(self): + a1,a2,a3 = random.randint(-2147483647,2147483647),random.randint(-2147483647,2147483647),random.randint(-2147483647,2147483647) + state_window_lists = ['first case when %d then %d end last' %(a1,a2) , #'first case when 3 then 4 end last' , + 'first case when 0 then %d end last' %(a1), #'first case when 0 then 4 end last' , + 'first case when null then %d end last' %(a1) , #'first case when null then 4 end last' , + 'first case when %d-(%d) then 0 end last' %(a1,a1) , #'first case when 1-1 then 0 end last' , + 'first case when %d+(%d) then 0 end last' %(a1,a1), #'first case when 1+1 then 0 end last' , + 'first case when %d > 0 then %d < %d end last' %(a1,a1,a2), #'first case when 1 > 0 then 1 < 2 end last' , + 'first case when %d > %d then %d < %d end last' %(a1,a2,a1,a2), #'first case when 1 > 2 then 1 < 2 end last' , + 'first case when abs(%d) then abs(-(%d)) end last' %(a1,a2) ,#'first case when abs(3) then abs(-1) end last' , + 'first case when 0 then %d else %d end last' %(a1,a2), #'first case when 0 then 1 else 3 end last' , + 'first case when 0 then %d when 1 then %d else %d end last' %(a1,a1,a3), #'first case when 0 then 1 when 1 then 0 else 3 end last' , + 'first case when 0 then %d when 1 then %d when 2 then %d end last' %(a1,a1,a3), #'first case when 0 then 1 when 1 then 0 when 2 then 3 end last' , + 'first case when \'a\' then \'b\' when null then 0 end last' , #'first case when \'a\' then \'b\' when null then 0 end last' , + 'first case when \'%d\' then \'b\' when null then %d end last' %(a1,a2) , #'first case when \'2\' then \'b\' when null then 0 end last' , + 'first case when \'%d\' then \'b\' else null end last' %(a1), #'first case when \'0\' then \'b\' else null end last', + 'first case when \'%d\' then \'b\' else %d end last' %(a1,a2), #'first case when \'0\' then \'b\' else 2 end last', + 'first case when q_int then q_int when q_int + (%d) then q_int + (%d) else q_int is null end last' %(a1,a2) , #'first case when q_int then q_int when q_int + 1 then q_int + 1 else q_int is null end last' , + 'first case when q_int then %d when ts then ts end last' %(a1), #'first case when q_int then 3 when ts then ts end last' , + 'first case when %d then q_int end last' %(a1), #'first case when 3 then q_int end last' , + 'first case when q_int then %d when %d then %d end last' %(a1,a1,a3), #'first case when q_int then 3 when 1 then 2 end last' , + 'first case when q_int < %d then %d when q_int >= %d then %d else %d end last' %(a1,a2,a1,a2,a3), #'first case when q_int < 3 then 1 when q_int >= 3 then 2 else 3 end last' , + 'first case when q_int is not null then case when q_int <= %d then q_int else q_int * (%d) end else -(%d) end last' %(a1,a1,a3), #'first case when q_int is not null then case when q_int <= 0 then q_int else q_int * 10 end else -1 end last' , + 'first case %d when %d then %d end last' %(a1,a2,a3), # 'first case 3 when 3 then 4 end last' , + 'first case %d when %d then %d end last' %(a1,a2,a3), # 'first case 3 when 1 then 4 end last' , + 'first case %d when %d then %d else %d end last' %(a1,a1,a2,a3), # 'first case 3 when 1 then 4 else 2 end last' , + 'first case %d when null then %d when \'%d\' then %d end last' %(a1,a1,a2,a3) , # 'first case 3 when null then 4 when \'3\' then 1 end last' , + 'first case \'%d\' when null then %d when %d then %d end last' %(a1,a1,a2,a3), # 'first case \'3\' when null then 4 when 3 then 1 end last' , + 'first case null when null then %d when %d then %d end last' %(a1,a2,a3), # 'first case null when null then 4 when 3 then 1 end last' , + 'first case %d.0 when null then %d when \'%d\' then %d end last' %(a1,a1,a2,a3) , # 'first case 3.0 when null then 4 when \'3\' then 1 end last' , + 'first case q_double when \'a\' then %d when \'%d\' then %d end last' %(a1,a2,a3) , # 'first case q_double when \'a\' then 4 when \'0\' then 1 end last' , + 'first case q_double when q_int then q_int when q_int - (%d) then q_int else %d end last' %(a1,a2), # 'first case q_double when q_int then q_int when q_int - 1 then q_int else 99 end last' , + 'first case q_int + (%d) when %d then %d when %d then %d else %d end last' %(a1,a2,a3,a1,a2,a3), #'first case q_int + 1 when 1 then 1 when 2 then 2 else 3 end last' , + 'first case when \'a\' then \'b\' when null then %d end last' %(a1), # 'first case when \'a\' then \'b\' when null then 0 end last' , + 'first case when \'%d\' then \'b\' when null then %d end last' %(a1,a2), # 'first case when \'2\' then \'b\' when null then 0 end last' , + 'first case when %d then \'b\' else null end last' %(a1), # 'first case when 0 then \'b\' else null end last' , + 'first case when %d then \'b\' else %d+abs(%d) end last' %(a1,a2,a3), # 'first case when 0 then \'b\' else 2+abs(-2) end last' , + 'first case when %d then %d end last' %(a1,a2), # 'first case when 3 then 4 end last' , + 'first case when %d then %d end last' %(a1,a2), # 'first case when 0 then 4 end last' , + 'first case when null then %d end last' %(a1), # 'first case when null then 4 end last' , + #'first case when %d then %d+(%d) end last' %(a1,a2,a3), # 'first case when 1 then 4+1 end last' , + 'first case when %d-(%d) then %d end last' %(a1,a2,a3), # 'first case when 1-1 then 0 end last' , + 'first case when %d+(%d) then %d end last' %(a1,a2,a3), # 'first case when 1+1 then 0 end last' , + 'first case when abs(%d) then abs(%d) end last' %(a1,a2), # 'first case when abs(3) then abs(-1) end last' , + #'first case when abs(%d+(%d)) then abs(%d)+abs(%d) end last' %(a1,a2,a3,a1), # 'first case when abs(1+1) then abs(-1)+abs(3) end last' , + 'first case when %d then %d else %d end last' %(a1,a2,a3), # 'first case when 0 then 1 else 3 end last' , + 'first case when %d then %d when %d then %d else %d end last' %(a1,a2,a3,a1,a2), # 'first case when 0 then 1 when 1 then 0 else 3 end last' , + 'first case when %d then %d when %d then %d when %d then %d end last' %(a1,a2,a3,a1,a2,a3), # 'first case when 0 then 1 when 1 then 0 when 2 then 3 end last' , + 'first case %d when %d then %d end last' %(a1,a1,a3), # 'first case 3 when 3 then 4 end last' , + 'first case %d when %d then %d end last' %(a1,a2,a3), # 'first case 3 when 1 then 4 end last' , + 'first case %d when %d then %d else %d end last' %(a1,a2,a3,a1), # 'first case 3 when 1 then 4 else 2 end last' , + 'first case %d when null then %d when \'%d\' then %d end last' %(a1,a2,a1,a3), # 'first case 3 when null then 4 when \'3\' then 1 end last' , + 'first case null when null then %d when %d then %d end last' %(a1,a2,a3), # 'first case null when null then 4 when 3 then 1 end last' , + 'first case %d.0 when null then %d when \'%d\' then %d end last' %(a1,a2,a1,a3), # 'first case 3.0 when null then 4 when \'3\' then 1 end last' , + 'first case null when null then %d when %d then %d end last' %(a1,a2,a3), #'first case null when null then 4 when 3 then 1 end last' , + 'first q_int, case q_int + (%d) when %d then %d when %d then %d else %d end last' %(a1,a1,a1,a2,a2,a3), #'first q_int, case q_int + 1 when 1 then 1 when 2 then 2 else 3 end last' , + ] + + state_window_list = str(random.sample(state_window_lists,50)).replace("[","").replace("]","").replace("'first","").replace("last'","").replace("\"first","").replace("last\"","") + + return state_window_list + + def state_window_case(self,database): + + for i in range(30): + cs = self.state_window_list().split(',')[i] + sql1 = "select _wstart,avg(q_int),min(q_smallint) from %s.stable_1 where tbname = 'stable_1_1' state_window(%s);" % (database,cs) + sql2 = "select _wstart,avg(q_int),min(q_smallint) from %s.stable_1_1 state_window(%s) ;" % (database,cs) + self.constant_check(database,sql1,sql2,0) + self.constant_check(database,sql1,sql2,1) + self.constant_check(database,sql1,sql2,2) + + + + def constant_check(self,database,sql1,sql2,column): + #column =0 代表0列， column = n代表n-1列 + tdLog.info("\n=============sql1:(%s)___sql2:(%s) ====================\n" %(sql1,sql2)) + tdSql.query(sql1) + queryRows = len(tdSql.queryResult) + + for i in range(queryRows): + tdSql.query(sql1) + sql1_value = tdSql.getData(i,column) + tdSql.execute(" flush database %s;" %database) + tdSql.query(sql2) + sql2_value = tdSql.getData(i,column) + self.value_check(sql1_value,sql2_value) + + def value_check(self,base_value,check_value): + if base_value==check_value: + tdLog.info(f"checkEqual success, base_value={base_value},check_value={check_value}") + else : + tdLog.exit(f"checkEqual error, base_value=={base_value},check_value={check_value}") + + def run(self): + fake = Faker('zh_CN') + fake_data = fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1) + fake_float = fake.pyfloat() + fake_str = fake.pystr() + + startTime = time.time() + + os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) + + self.dropandcreateDB_random("%s" %self.db, 10) + + self.users_bug("%s" %self.db) + + self.base_case("%s" %self.db) + + self.state_window_case("%s" %self.db) + + + + #taos -f sql + print("taos -f sql start!") + taos_cmd1 = "taos -f %s/%s.sql" % (self.testcasePath,self.testcaseFilename) + _ = subprocess.check_output(taos_cmd1, shell=True) + print("taos -f sql over!") + + + endTime = time.time() + print("total time %ds" % (endTime - startTime)) + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From a295aa206b37e4c0c1b39709aa27aa664c91fd9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Fri, 18 Nov 2022 14:23:42 +0800 Subject: [PATCH 43/95] test: refine query cases --- tests/script/tsim/parser/limit2_query.sim | 34 +++++++---------------- 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/tests/script/tsim/parser/limit2_query.sim b/tests/script/tsim/parser/limit2_query.sim index 3c5002d591..64eb26429c 100644 --- a/tests/script/tsim/parser/limit2_query.sim +++ b/tests/script/tsim/parser/limit2_query.sim @@ -321,55 +321,41 @@ endi ### [TBASE-350] ## stb + interval + fill + group by + limit offset -sql select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') limit 2 offset 10 +sql select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') limit 2 offset 10 if $rows != 2 then return -1 endi -#add one more test case -sql select max(c1), last(c8) from lm2_db0.lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(linear) limit 10 offset 4089;" - $limit = 5 $offset = $rowNum * 2 $offset = $offset - 2 -sql select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') order by t1 limit $limit offset $offset -if $rows != $tbNum then +sql select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') order by t1 limit $limit offset $offset +if $rows != 1 then return -1 endi -if $data00 != @18-11-25 19:30:00.000@ then +if $data00 != 9 then return -1 endi if $data01 != 9 then return -1 endi -if $data12 != 9 then +if $data02 != 9.000000000 then return -1 endi -if $data23 != 9.000000000 then +if $data03 != 9.000000000 then return -1 endi -if $data34 != 9.000000000 then +if $data04 != 1 then return -1 endi -if $data45 != 1 then +if $data05 != binary9 then return -1 endi -if $data56 != binary9 then - return -1 -endi -if $data68 != 6 then - return -1 -endi -if $data72 != -2 then - return -1 -endi -if $data84 != -2.000000000 then - return -1 -endi -if $data98 != 9 then +if $data06 != nchar9 then return -1 endi + #add one more test case sql select max(c1), last(c8) from lm2_db0.lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(linear) limit 10 offset 4089;" From a60cea30865efa1089630d1605ad77c15963b79b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Fri, 18 Nov 2022 14:26:54 +0800 Subject: [PATCH 44/95] test: refine query cases --- tests/parallel_test/cases.task | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 54877273cf..7925f7e291 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -113,7 +113,7 @@ ,,y,script,./test.sh -f tsim/parser/first_last.sim ,,y,script,./test.sh -f tsim/parser/fill_stb.sim ,,y,script,./test.sh -f tsim/parser/interp.sim -#,,y,script,./test.sh -f tsim/parser/limit2.sim +,,y,script,./test.sh -f tsim/parser/limit2.sim ,,y,script,./test.sh -f tsim/parser/fourArithmetic-basic.sim ,,y,script,./test.sh -f tsim/parser/function.sim ,,y,script,./test.sh -f tsim/parser/groupby-basic.sim @@ -607,6 +607,8 @@ ,,,system-test,python3 ./test.py -f 2-query/upper.py -R ,,,system-test,python3 ./test.py -f 2-query/varchar.py ,,,system-test,python3 ./test.py -f 2-query/varchar.py -R +,,,system-test,python3 ./test.py -f 2-query/case_when.py +,,,system-test,python3 ./test.py -f 2-query/case_when.py -R ,,,system-test,python3 ./test.py -f 1-insert/update_data.py ,,,system-test,python3 ./test.py -f 1-insert/tb_100w_data_order.py ,,,system-test,python3 ./test.py -f 1-insert/delete_stable.py @@ -813,6 +815,7 @@ ,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/sml.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/case_when.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/between.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/distinct.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/varchar.py -Q 3 @@ -906,6 +909,7 @@ ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/sml.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/case_when.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/between.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/distinct.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/varchar.py -Q 4 @@ -998,6 +1002,7 @@ ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/sml.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/case_when.py -Q 4 #develop test ,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py From 7291f2bd5adb979f7a8bfa449c4765000a037340 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 18 Nov 2022 09:43:40 +0800 Subject: [PATCH 45/95] change interp linear behavior --- source/libs/executor/src/timewindowoperator.c | 13 +- tests/system-test/2-query/interp.py | 482 +++++++++++++----- 2 files changed, 370 insertions(+), 125 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 7582107c93..5e1a5d1ec4 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2020,7 +2020,7 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo } -static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock) { +static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, bool beforeTs) { int32_t rows = pResBlock->info.rows; blockDataEnsureCapacity(pResBlock, rows + 1); // todo set the correct primary timestamp column @@ -2071,6 +2071,11 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp SPoint end = pLinearInfo->end; SPoint current = {.key = pSliceInfo->current}; + // do not interpolate before ts range, only increate pSliceInfo->current + if (beforeTs && !pLinearInfo->isEndSet) { + return true; + } + if (!pLinearInfo->isStartSet || !pLinearInfo->isEndSet) { hasInterp = false; break; @@ -2319,7 +2324,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { + if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, false) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { break; } else { pSliceInfo->current = @@ -2343,7 +2348,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doKeepLinearInfo(pSliceInfo, pBlock, i); while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { - if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { + if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, true) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { break; } else { pSliceInfo->current = @@ -2372,7 +2377,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { // except for fill(next), fill(linear) while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT && pSliceInfo->fillType != TSDB_FILL_LINEAR) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, false); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); } diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index fa5c35f782..b5bf6f9dbe 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -908,224 +908,464 @@ class TDTestCase: tdSql.checkData(0, 0, 0) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:03') every(1s) fill(linear)") - tdSql.checkRows(1) + tdSql.checkRows(4) tdSql.checkData(0, 0, 0) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:05') every(1s) fill(linear)") - tdSql.checkRows(1) + tdSql.checkRows(6) tdSql.checkData(0, 0, 0) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:08') every(1s) fill(linear)") tdSql.checkRows(9) - tdSql.checkData(0, 0, 0) - tdSql.checkData(1, 0, 1) - tdSql.checkData(2, 0, 2) - tdSql.checkData(3, 0, 3) - tdSql.checkData(4, 0, 4) - tdSql.checkData(5, 0, 5) - tdSql.checkData(6, 0, 6) - tdSql.checkData(7, 0, 7) - tdSql.checkData(8, 0, 8) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:01', '2020-02-02 00:00:03') every(1s) fill(linear)") - tdSql.checkRows(0) + tdSql.checkRows(3) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:03', '2020-02-02 00:00:08') every(1s) fill(linear)") tdSql.checkRows(6) - tdSql.checkData(0, 0, 3) - tdSql.checkData(1, 0, 4) - tdSql.checkData(2, 0, 5) - tdSql.checkData(3, 0, 6) - tdSql.checkData(4, 0, 7) - tdSql.checkData(5, 0, 8) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:05', '2020-02-02 00:00:10') every(1s) fill(linear)") - tdSql.checkRows(1) - tdSql.checkData(0, 0, 10) + tdSql.checkRows(6) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, 10) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:05', '2020-02-02 00:00:15') every(1s) fill(linear)") - tdSql.checkRows(1) - tdSql.checkData(0, 0, 10) + tdSql.checkRows(11) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, 10) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:05', '2020-02-02 00:00:18') every(1s) fill(linear)") - tdSql.checkRows(9) - tdSql.checkData(0, 0, 10) - tdSql.checkData(1, 0, 11) - tdSql.checkData(2, 0, 12) - tdSql.checkData(3, 0, 13) - tdSql.checkData(4, 0, 14) - tdSql.checkData(5, 0, 15) - tdSql.checkData(6, 0, 16) - tdSql.checkData(7, 0, 17) - tdSql.checkData(8, 0, 18) + tdSql.checkRows(14) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, 10) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, None) + tdSql.checkData(11, 0, None) + tdSql.checkData(12, 0, None) + tdSql.checkData(13, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:05', '2020-02-02 00:00:20') every(1s) fill(linear)") - tdSql.checkRows(11) - tdSql.checkData(0, 0, 10) - tdSql.checkData(1, 0, 11) - tdSql.checkData(2, 0, 12) - tdSql.checkData(3, 0, 13) - tdSql.checkData(4, 0, 14) - tdSql.checkData(5, 0, 15) - tdSql.checkData(6, 0, 16) - tdSql.checkData(7, 0, 17) - tdSql.checkData(8, 0, 18) - tdSql.checkData(9, 0, 19) - tdSql.checkData(10, 0, 20) + tdSql.checkRows(16) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, 10) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, None) + tdSql.checkData(11, 0, None) + tdSql.checkData(12, 0, None) + tdSql.checkData(13, 0, None) + tdSql.checkData(14, 0, None) + tdSql.checkData(15, 0, 20) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:09', '2020-02-02 00:00:11') every(1s) fill(linear)") - tdSql.checkRows(1) - tdSql.checkData(0, 0, 10) + tdSql.checkRows(3) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 10) + tdSql.checkData(2, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:10', '2020-02-02 00:00:15') every(1s) fill(linear)") - tdSql.checkRows(1) + tdSql.checkRows(6) tdSql.checkData(0, 0, 10) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:12', '2020-02-02 00:00:13') every(1s) fill(linear)") - tdSql.checkRows(0) + tdSql.checkRows(2) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:12', '2020-02-02 00:00:15') every(1s) fill(linear)") - tdSql.checkRows(0) + tdSql.checkRows(4) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:12', '2020-02-02 00:00:18') every(1s) fill(linear)") tdSql.checkRows(7) - tdSql.checkData(0, 0, 12) - tdSql.checkData(1, 0, 13) - tdSql.checkData(2, 0, 14) - tdSql.checkData(3, 0, 15) - tdSql.checkData(4, 0, 16) - tdSql.checkData(5, 0, 17) - tdSql.checkData(6, 0, 18) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:30', '2020-02-02 00:00:40') every(1s) fill(linear)") tdSql.checkRows(11) tdSql.checkData(0, 0, 30) + tdSql.checkData(1, 0, 31) + tdSql.checkData(2, 0, 32) + tdSql.checkData(3, 0, 33) + tdSql.checkData(4, 0, 34) + tdSql.checkData(5, 0, 35) + tdSql.checkData(6, 0, 36) + tdSql.checkData(7, 0, 37) + tdSql.checkData(8, 0, 38) + tdSql.checkData(9, 0, 39) tdSql.checkData(10, 0, 40) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:25', '2020-02-02 00:00:45') every(1s) fill(linear)") - tdSql.checkRows(11) - tdSql.checkData(0, 0, 30) - tdSql.checkData(10, 0, 40) - - tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:25', '2020-02-02 00:00:45') every(1s) fill(linear)") - tdSql.checkRows(11) - tdSql.checkData(0, 0, 30) - tdSql.checkData(10, 0, 40) + tdSql.checkRows(21) + tdSql.checkData(5, 0, 30) + tdSql.checkData(6, 0, 31) + tdSql.checkData(7, 0, 32) + tdSql.checkData(8, 0, 33) + tdSql.checkData(9, 0, 34) + tdSql.checkData(10, 0, 35) + tdSql.checkData(11, 0, 36) + tdSql.checkData(12, 0, 37) + tdSql.checkData(13, 0, 38) + tdSql.checkData(14, 0, 39) + tdSql.checkData(15, 0, 40) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:20', '2020-02-02 00:00:40') every(1s) fill(linear)") tdSql.checkRows(21) tdSql.checkData(0, 0, 20) + tdSql.checkData(10, 0, 30) + tdSql.checkData(11, 0, 31) + tdSql.checkData(12, 0, 32) + tdSql.checkData(13, 0, 33) + tdSql.checkData(14, 0, 34) + tdSql.checkData(15, 0, 35) + tdSql.checkData(16, 0, 36) + tdSql.checkData(17, 0, 37) + tdSql.checkData(18, 0, 38) + tdSql.checkData(19, 0, 39) tdSql.checkData(20, 0, 40) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:30', '2020-02-02 00:00:50') every(1s) fill(linear)") tdSql.checkRows(21) tdSql.checkData(0, 0, 30) + tdSql.checkData(1, 0, 31) + tdSql.checkData(2, 0, 32) + tdSql.checkData(3, 0, 33) + tdSql.checkData(4, 0, 34) + tdSql.checkData(5, 0, 35) + tdSql.checkData(6, 0, 36) + tdSql.checkData(7, 0, 37) + tdSql.checkData(8, 0, 38) + tdSql.checkData(9, 0, 39) + tdSql.checkData(10, 0, 40) tdSql.checkData(20, 0, 50) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:20', '2020-02-02 00:00:50') every(1s) fill(linear)") tdSql.checkRows(31) tdSql.checkData(0, 0, 20) + tdSql.checkData(10, 0, 30) + tdSql.checkData(11, 0, 31) + tdSql.checkData(12, 0, 32) + tdSql.checkData(13, 0, 33) + tdSql.checkData(14, 0, 34) + tdSql.checkData(15, 0, 35) + tdSql.checkData(16, 0, 36) + tdSql.checkData(17, 0, 37) + tdSql.checkData(18, 0, 38) + tdSql.checkData(19, 0, 39) + tdSql.checkData(20, 0, 40) tdSql.checkData(30, 0, 50) # check c1 tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:05') every(1s) fill(linear)") - tdSql.checkRows(0) + tdSql.checkRows(6) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:05') every(1s) fill(linear)") - tdSql.checkRows(0) + tdSql.checkRows(6) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:08') every(1s) fill(linear)") - tdSql.checkRows(0) + tdSql.checkRows(9) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:10') every(1s) fill(linear)") - tdSql.checkRows(1) - tdSql.checkData(0, 0, 10) + tdSql.checkRows(11) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, 10) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:15') every(1s) fill(linear)") - tdSql.checkRows(1) - tdSql.checkData(0, 0, 10) + tdSql.checkRows(16) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, 10) + tdSql.checkData(11, 0, None) + tdSql.checkData(12, 0, None) + tdSql.checkData(13, 0, None) + tdSql.checkData(14, 0, None) + tdSql.checkData(15, 0, None) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:20') every(1s) fill(linear)") - tdSql.checkRows(1) - tdSql.checkData(0, 0, 10) + tdSql.checkRows(21) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, 10) + tdSql.checkData(11, 0, None) + tdSql.checkData(12, 0, None) + tdSql.checkData(13, 0, None) + tdSql.checkData(14, 0, None) + tdSql.checkData(15, 0, None) + tdSql.checkData(16, 0, None) + tdSql.checkData(17, 0, None) + tdSql.checkData(18, 0, None) + tdSql.checkData(19, 0, None) + tdSql.checkData(20, 0, None) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:25') every(1s) fill(linear)") - tdSql.checkRows(1) - tdSql.checkData(0, 0, 10) + tdSql.checkRows(26) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, 10) + tdSql.checkData(11, 0, None) + tdSql.checkData(12, 0, None) + tdSql.checkData(13, 0, None) + tdSql.checkData(14, 0, None) + tdSql.checkData(15, 0, None) + tdSql.checkData(16, 0, None) + tdSql.checkData(17, 0, None) + tdSql.checkData(18, 0, None) + tdSql.checkData(19, 0, None) + tdSql.checkData(20, 0, None) + tdSql.checkData(21, 0, None) + tdSql.checkData(22, 0, None) + tdSql.checkData(23, 0, None) + tdSql.checkData(24, 0, None) + tdSql.checkData(25, 0, None) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:30') every(1s) fill(linear)") - tdSql.checkRows(21) - tdSql.checkData(0, 0, 10) - tdSql.checkData(1, 0, 11) - tdSql.checkData(2, 0, 12) - tdSql.checkData(3, 0, 13) - tdSql.checkData(4, 0, 14) - tdSql.checkData(5, 0, 15) - tdSql.checkData(6, 0, 16) - tdSql.checkData(7, 0, 17) - tdSql.checkData(8, 0, 18) - tdSql.checkData(9, 0, 19) - tdSql.checkData(10, 0, 20) - tdSql.checkData(11, 0, 21) - tdSql.checkData(12, 0, 22) - tdSql.checkData(13, 0, 23) - tdSql.checkData(14, 0, 24) - tdSql.checkData(15, 0, 25) - tdSql.checkData(16, 0, 26) - tdSql.checkData(17, 0, 27) - tdSql.checkData(18, 0, 28) - tdSql.checkData(19, 0, 29) - tdSql.checkData(20, 0, 30) + tdSql.checkRows(31) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, 10) + tdSql.checkData(11, 0, None) + tdSql.checkData(12, 0, None) + tdSql.checkData(13, 0, None) + tdSql.checkData(14, 0, None) + tdSql.checkData(15, 0, None) + tdSql.checkData(16, 0, None) + tdSql.checkData(17, 0, None) + tdSql.checkData(18, 0, None) + tdSql.checkData(19, 0, None) + tdSql.checkData(20, 0, None) + tdSql.checkData(21, 0, None) + tdSql.checkData(22, 0, None) + tdSql.checkData(23, 0, None) + tdSql.checkData(24, 0, None) + tdSql.checkData(25, 0, None) + tdSql.checkData(26, 0, None) + tdSql.checkData(27, 0, None) + tdSql.checkData(28, 0, None) + tdSql.checkData(29, 0, None) + tdSql.checkData(30, 0, 30) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:35') every(1s) fill(linear)") - tdSql.checkRows(21) - tdSql.checkData(0, 0, 10) - tdSql.checkData(20, 0, 30) + tdSql.checkRows(36) + tdSql.checkData(10, 0, 10) + tdSql.checkData(30, 0, 30) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:40') every(1s) fill(linear)") - tdSql.checkRows(31) + tdSql.checkRows(41) tdSql.checkData(0, 0, 10) - tdSql.checkData(30, 0, 40) + tdSql.checkData(30, 0, 30) + tdSql.checkData(40, 0, 40) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:45') every(1s) fill(linear)") - tdSql.checkRows(36) + tdSql.checkRows(46) tdSql.checkData(0, 0, 10) - tdSql.checkData(35, 0, 45) + tdSql.checkData(30, 0, 30) + tdSql.checkData(40, 0, 40) + tdSql.checkData(41, 0, 41) + tdSql.checkData(42, 0, 42) + tdSql.checkData(43, 0, 43) + tdSql.checkData(44, 0, 44) + tdSql.checkData(45, 0, 45) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:50') every(1s) fill(linear)") - tdSql.checkRows(36) - tdSql.checkData(0, 0, 10) - tdSql.checkData(35, 0, 45) - - tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:55') every(1s) fill(linear)") - tdSql.checkRows(36) - tdSql.checkData(0, 0, 10) - tdSql.checkData(35, 0, 45) - - tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(linear)") tdSql.checkRows(51) tdSql.checkData(0, 0, 10) - tdSql.checkData(50, 0, 60) + tdSql.checkData(30, 0, 30) + tdSql.checkData(40, 0, 40) + tdSql.checkData(41, 0, 41) + tdSql.checkData(42, 0, 42) + tdSql.checkData(43, 0, 43) + tdSql.checkData(44, 0, 44) + tdSql.checkData(45, 0, 45) + + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:55') every(1s) fill(linear)") + tdSql.checkRows(56) + tdSql.checkData(0, 0, 10) + tdSql.checkData(30, 0, 30) + tdSql.checkData(40, 0, 40) + tdSql.checkData(41, 0, 41) + tdSql.checkData(42, 0, 42) + tdSql.checkData(43, 0, 43) + tdSql.checkData(44, 0, 44) + tdSql.checkData(45, 0, 45) + + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(linear)") + tdSql.checkRows(60) + tdSql.checkData(0, 0, 10) + tdSql.checkData(30, 0, 30) + tdSql.checkData(40, 0, 40) + tdSql.checkData(41, 0, 41) + tdSql.checkData(42, 0, 42) + tdSql.checkData(43, 0, 43) + tdSql.checkData(44, 0, 44) + tdSql.checkData(45, 0, 45) + tdSql.checkData(60, 0, 60) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:40', '2020-02-02 00:00:45') every(1s) fill(linear)") tdSql.checkRows(6) tdSql.checkData(0, 0, 40) + tdSql.checkData(1, 0, 41) + tdSql.checkData(2, 0, 42) + tdSql.checkData(3, 0, 43) + tdSql.checkData(4, 0, 44) tdSql.checkData(5, 0, 45) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:35', '2020-02-02 00:00:50') every(1s) fill(linear)") - tdSql.checkRows(6) - tdSql.checkData(0, 0, 40) - tdSql.checkData(5, 0, 45) + tdSql.checkRows(16) + tdSql.checkData(5, 0, 40) + tdSql.checkData(6, 0, 41) + tdSql.checkData(7, 0, 42) + tdSql.checkData(8, 0, 43) + tdSql.checkData(9, 0, 44) + tdSql.checkData(10, 0, 45) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:35', '2020-02-02 00:00:55') every(1s) fill(linear)") - tdSql.checkRows(6) - tdSql.checkData(0, 0, 40) - tdSql.checkData(5, 0, 45) + tdSql.checkRows(21) + tdSql.checkData(5, 0, 40) + tdSql.checkData(6, 0, 41) + tdSql.checkData(7, 0, 42) + tdSql.checkData(8, 0, 43) + tdSql.checkData(9, 0, 44) + tdSql.checkData(10, 0, 45) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:30', '2020-02-02 00:01:00') every(1s) fill(linear)") tdSql.checkRows(31) tdSql.checkData(0, 0, 30) + tdSql.checkData(10, 0, 40) + tdSql.checkData(11, 0, 41) + tdSql.checkData(12, 0, 42) + tdSql.checkData(13, 0, 43) + tdSql.checkData(14, 0, 44) + tdSql.checkData(15, 0, 45) tdSql.checkData(30, 0, 60) tdLog.printNoPrefix("==========step11:test multi-interp cases") From 9f5a406441d8dba1bc12a762fe62bfa6e4e4203b Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 18 Nov 2022 14:59:42 +0800 Subject: [PATCH 46/95] update --- tests/pytest/crash_gen/crash_gen_main.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 90d27c3b8d..2aa499285f 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -2039,7 +2039,7 @@ class TdSuperTable: conf.set("enable.auto.commit", "true") def tmq_commit_cb_print(tmq, resp, offset, param=None): print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}") - conf.set_auto_commit_cb(tmq_commit_cb_print, None) + # conf.set_auto_commit_cb(tmq_commit_cb_print, None) consumer = conf.new_consumer() topic_list = TaosTmqList() for topic in current_topic_list: @@ -2049,13 +2049,12 @@ class TdSuperTable: except TmqError as e : pass - # consumer work only 30 sec + # consumer with random work life time_start = time.time() while 1: res = consumer.poll(1000) - if time.time() - time_start >5 : + if time.time() - time_start >random.randint(5,50) : break - # time.sleep(10) try: consumer.unsubscribe() except TmqError as e : From 0e6481a4a0b108a34715a2a3aef6b479a9b6af5e Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 18 Nov 2022 15:10:23 +0800 Subject: [PATCH 47/95] update --- tests/pytest/crash_gen/crash_gen_main.py | 34 +++++++++--------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 2aa499285f..49b68965d0 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1978,15 +1978,8 @@ class TdSuperTable: def drop(self, dbc, skipCheck = False): dbName = self._dbName if self.exists(dbc) : # if myself exists - fullTableName = dbName + '.' + self._stName - - try: - dbc.execute("DROP TABLE {}".format(fullTableName)) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [1011,0x3F3,0x03f3,0x2662]: # table doesn't exist # Stream must be dropped first, SQL: DROP TABLE db_0.fs_table - pass - + fullTableName = dbName + '.' + self._stName + dbc.execute("DROP TABLE {}".format(fullTableName)) else: if not skipCheck: raise CrashGenError("Cannot drop non-existant super table: {}".format(self._stName)) @@ -2004,16 +1997,9 @@ class TdSuperTable: fullTableName = dbName + '.' + self._stName if dbc.existsSuperTable(self._stName): - if dropIfExists: - try: - dbc.execute("DROP TABLE {}".format(fullTableName)) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [1011,0x3F3,0x03f3,0x2662]: # table doesn't exist # Stream must be dropped first, SQL: DROP TABLE db_0.fs_table - pass - - pass - # dbc.execute("DROP TABLE {}".format(fullTableName)) + if dropIfExists: + dbc.execute("DROP TABLE {}".format(fullTableName)) + else: # error raise CrashGenError("Cannot create super table, already exists: {}".format(self._stName)) @@ -2434,11 +2420,15 @@ class TaskDropSuperTable(StateTransitionTask): for i in tblSeq: regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) try: - self.execWtSql(wt, "drop table {}.{}". format(self._db.getName(), regTableName)) # nRows always 0, like MySQL except taos.error.ProgrammingError as err: - pass + # correcting for strange error number scheme + errno2 = Helper.convertErrno(err.errno) + if (errno2 in [0x362]): # mnode invalid table name + isSuccess = False + Logging.debug("[DB] Acceptable error when dropping a table") + continue # try to delete next regular table if (not tickOutput): @@ -2956,6 +2946,8 @@ class ThreadStacks: # stack info for all threads print(" {}".format(frame.line)) stackFrame += 1 print("-----> End of Thread Info ----->\n") + if self.current_time-last_sql_commit_time >100: # dead lock occured + print("maybe dead locked of thread {} ".format(shortTid)) class ClientManager: def __init__(self): From 4703ccc4bcc4b8c967948b0f4d698079915c4706 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 18 Nov 2022 15:53:58 +0800 Subject: [PATCH 48/95] fix fill prev/next error --- source/libs/executor/src/timewindowoperator.c | 28 +++++---- tests/system-test/2-query/interp.py | 60 ++++++++++++++----- 2 files changed, 64 insertions(+), 24 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 5e1a5d1ec4..0dd3251b0c 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1945,10 +1945,8 @@ static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - // null data should not be kept since it can not be used to perform interpolation - if (!colDataIsNull_s(pColInfoData, i)) { - SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, i); - + SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, i); + if (!colDataIsNull_s(pColInfoData, rowIndex)) { pkey->isNull = false; char* val = colDataGetData(pColInfoData, rowIndex); if (!IS_VAR_DATA_TYPE(pkey->type)) { @@ -1956,6 +1954,8 @@ static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock } else { memcpy(pkey->pData, val, varDataLen(val)); } + } else { + pkey->isNull = true; } } @@ -1967,10 +1967,8 @@ static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - // null data should not be kept since it can not be used to perform interpolation - if (!colDataIsNull_s(pColInfoData, i)) { - SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, i); - + SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, i); + if (!colDataIsNull_s(pColInfoData, rowIndex)) { pkey->isNull = false; char* val = colDataGetData(pColInfoData, rowIndex); if (!IS_VAR_DATA_TYPE(pkey->type)) { @@ -1978,6 +1976,8 @@ static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock } else { memcpy(pkey->pData, val, varDataLen(val)); } + } else { + pkey->isNull = true; } } @@ -2100,7 +2100,11 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot); - colDataAppend(pDst, rows, pkey->pData, false); + if (pkey->isNull == false) { + colDataAppend(pDst, rows, pkey->pData, false); + } else { + colDataAppendNULL(pDst, rows); + } break; } @@ -2111,7 +2115,11 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, srcSlot); - colDataAppend(pDst, rows, pkey->pData, false); + if (pkey->isNull == false) { + colDataAppend(pDst, rows, pkey->pData, false); + } else { + colDataAppendNULL(pDst, rows); + } break; } diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index b5bf6f9dbe..a0ac4ab03b 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -39,8 +39,6 @@ class TDTestCase: tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:10', 10, 10, 10, 10, 10.0, 10.0, true, 'varchar', 'nchar')") tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:15', 15, 15, 15, 15, 15.0, 15.0, true, 'varchar', 'nchar')") - tdSql.execute(f"insert into {dbname}.{tbname} (ts) values (now)") - tdLog.printNoPrefix("==========step3:fill null") ## {. . .} @@ -248,7 +246,7 @@ class TDTestCase: ## {. . .} tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(next)") - tdSql.checkRows(13) + tdSql.checkRows(12) tdSql.checkData(0, 0, 5) tdSql.checkData(1, 0, 5) tdSql.checkData(2, 0, 10) @@ -298,14 +296,14 @@ class TDTestCase: ## ..{.} tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:13', '2020-02-01 00:00:17') every(1s) fill(next)") - tdSql.checkRows(5) + tdSql.checkRows(3) tdSql.checkData(0, 0, 15) tdSql.checkData(1, 0, 15) tdSql.checkData(2, 0, 15) ## ... {} tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(next)") - tdSql.checkRows(4) + tdSql.checkRows(0) tdLog.printNoPrefix("==========step7:fill linear") @@ -513,7 +511,7 @@ class TDTestCase: tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(next)") - tdSql.checkRows(13) + tdSql.checkRows(12) tdSql.checkCols(2) tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') @@ -925,7 +923,7 @@ class TDTestCase: tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:08') every(1s) fill(linear)") tdSql.checkRows(9) - tdSql.checkData(0, 0, None) + tdSql.checkData(0, 0, 0) tdSql.checkData(1, 0, None) tdSql.checkData(2, 0, None) tdSql.checkData(3, 0, None) @@ -1274,18 +1272,18 @@ class TDTestCase: tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:35') every(1s) fill(linear)") tdSql.checkRows(36) - tdSql.checkData(10, 0, 10) + tdSql.checkData(10, 0, 10) tdSql.checkData(30, 0, 30) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:40') every(1s) fill(linear)") tdSql.checkRows(41) - tdSql.checkData(0, 0, 10) + tdSql.checkData(10, 0, 10) tdSql.checkData(30, 0, 30) tdSql.checkData(40, 0, 40) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:45') every(1s) fill(linear)") tdSql.checkRows(46) - tdSql.checkData(0, 0, 10) + tdSql.checkData(10, 0, 10) tdSql.checkData(30, 0, 30) tdSql.checkData(40, 0, 40) tdSql.checkData(41, 0, 41) @@ -1296,7 +1294,7 @@ class TDTestCase: tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:50') every(1s) fill(linear)") tdSql.checkRows(51) - tdSql.checkData(0, 0, 10) + tdSql.checkData(10, 0, 10) tdSql.checkData(30, 0, 30) tdSql.checkData(40, 0, 40) tdSql.checkData(41, 0, 41) @@ -1308,7 +1306,7 @@ class TDTestCase: tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:55') every(1s) fill(linear)") tdSql.checkRows(56) - tdSql.checkData(0, 0, 10) + tdSql.checkData(10, 0, 10) tdSql.checkData(30, 0, 30) tdSql.checkData(40, 0, 40) tdSql.checkData(41, 0, 41) @@ -1319,8 +1317,8 @@ class TDTestCase: tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(linear)") - tdSql.checkRows(60) - tdSql.checkData(0, 0, 10) + tdSql.checkRows(61) + tdSql.checkData(10, 0, 10) tdSql.checkData(30, 0, 30) tdSql.checkData(40, 0, 40) tdSql.checkData(41, 0, 41) @@ -1368,6 +1366,40 @@ class TDTestCase: tdSql.checkData(15, 0, 45) tdSql.checkData(30, 0, 60) + # two interps + tdSql.query(f"select interp(c0),interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(linear)") + tdSql.checkRows(61) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 0) + tdSql.checkData(10, 0, 10) + tdSql.checkData(20, 0, 20) + tdSql.checkData(30, 0, 30) + tdSql.checkData(31, 0, 31) + tdSql.checkData(32, 0, 32) + tdSql.checkData(33, 0, 33) + tdSql.checkData(34, 0, 34) + tdSql.checkData(35, 0, 35) + tdSql.checkData(36, 0, 36) + tdSql.checkData(37, 0, 37) + tdSql.checkData(38, 0, 38) + tdSql.checkData(39, 0, 39) + tdSql.checkData(40, 0, 40) + tdSql.checkData(50, 0, 50) + tdSql.checkData(60, 0, 55) + + tdSql.checkData(0, 1, None) + tdSql.checkData(10, 1, 10) + tdSql.checkData(20, 1, None) + tdSql.checkData(30, 1, 30) + tdSql.checkData(40, 1, 40) + tdSql.checkData(41, 1, 41) + tdSql.checkData(42, 1, 42) + tdSql.checkData(43, 1, 43) + tdSql.checkData(44, 1, 44) + tdSql.checkData(45, 1, 45) + tdSql.checkData(50, 1, None) + tdSql.checkData(60, 1, 60) + tdLog.printNoPrefix("==========step11:test multi-interp cases") tdSql.query(f"select interp(c0),interp(c1),interp(c2),interp(c3) from {dbname}.{tbname} range('2020-02-09 00:00:05', '2020-02-13 00:00:05') every(1d) fill(null)") tdSql.checkRows(5) From 7ff95b426e3ea3448b6c124116a7e0e6f8b4068c Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Fri, 18 Nov 2022 15:56:08 +0800 Subject: [PATCH 49/95] feat: send alert in script for cloud version --- packaging/docker/DockerfileCloud | 3 + packaging/docker/run.sh | 182 +++++++++++++++++++++++++++++-- 2 files changed, 178 insertions(+), 7 deletions(-) mode change 100644 => 100755 packaging/docker/run.sh diff --git a/packaging/docker/DockerfileCloud b/packaging/docker/DockerfileCloud index 2b060c1b91..21e387bab3 100644 --- a/packaging/docker/DockerfileCloud +++ b/packaging/docker/DockerfileCloud @@ -7,6 +7,9 @@ ARG dirName ARG cpuType RUN echo ${pkgFile} && echo ${dirName} +RUN apt update +RUN apt install -y curl + COPY ${pkgFile} /root/ ENV TINI_VERSION v0.19.0 ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${cpuType} /tini diff --git a/packaging/docker/run.sh b/packaging/docker/run.sh old mode 100644 new mode 100755 index 2700b0b060..b5d6b011ea --- a/packaging/docker/run.sh +++ b/packaging/docker/run.sh @@ -1,16 +1,160 @@ #!/bin/bash + TAOS_RUN_TAOSBENCHMARK_TEST_ONCE=0 +#ADMIN_URL=${ADMIN_URL:-http://172.26.10.84:10001} +TAOSD_STARTUP_TIMEOUT_SECOND=${TAOSD_STARTUP_TIMEOUT_SECOND:-160} +TAOS_TIMEOUT_SECOND=${TAOS_TIMEOUT_SECOND:-5} +BACKUP_CORE_FOLDER=/data/corefile +ALERT_URL=app/system/alert/add + +echo "ADMIN_URL: ${ADMIN_URL}" +echo "TAOS_TIMEOUT_SECOND: ${TAOS_TIMEOUT_SECOND}" + +function set_service_state() { + #echo "set service state: $1, $2" + service_state="$1" + service_msg="$2" +} +set_service_state "init" "ok" +app_name=`hostname |cut -d\- -f1` + +function check_taosd() { + timeout $TAOS_TIMEOUT_SECOND taos -s "show databases;" >/dev/null + local ret=$? + if [ $ret -ne 0 ]; then + echo "`date` check taosd error $ret" + if [ "x$1" != "xignore" ]; then + set_service_state "error" "taos check failed $ret" + fi + else + set_service_state "ready" "ok" + fi +} +function post_error_msg() { + if [ ! -z "${ADMIN_URL}" ]; then + taos_version=`taos --version` + echo "app_name: ${app_name}" + echo "service_state: ${service_state}" + echo "`date` service_msg: ${service_msg}" + echo "${taos_version}" + curl -X POST -H "Content-Type: application/json" \ + -d"{\"appName\":\"${app_name}\",\ + \"alertLevel\":\"${service_state}\",\ + \"taosVersion\":\"${taos_version}\",\ + \"alertMsg\":\"${service_msg}\"}" \ + ${ADMIN_URL}/${ALERT_URL} + fi +} +function check_taosd_exit_type() { + local core_pattern=`cat /proc/sys/kernel/core_pattern` + echo "$core_pattern" | grep -q "^/" + if [ $? -eq 0 ]; then + core_folder=`dirname $core_pattern` + core_prefix=`basename $core_pattern | sed "s/%.*//"` + else + core_folder=`pwd` + core_prefix="$core_pattern" + fi + local core_files=`ls $core_folder | grep "^${core_prefix}"` + if [ ! -z "$core_files" ]; then + # move core files to another folder + mkdir -p ${BACKUP_CORE_FOLDER} + mv ${core_folder}/${core_prefix}* ${BACKUP_CORE_FOLDER}/ + set_service_state "error" "taosd exit with core file" + else + set_service_state "error" "taosd exit without core file" + fi +} +disk_usage_level=(60 80 99) +current_disk_level=0 +disk_state="ok" +disk_msg="ok" +get_usage_ok="yes" +function post_disk_error_msg() { + if [ ! -z "${ADMIN_URL}" ]; then + taos_version=`taos --version` + echo "app_name: ${app_name}" + echo "disk_state: ${disk_state}" + echo "`date` disk_msg: ${disk_msg}" + echo "${taos_version}" + curl -X POST -H "Content-Type: application/json" \ + -d"{\"appName\":\"${app_name}\",\ + \"alertLevel\":\"${disk_state}\",\ + \"taosVersion\":\"${taos_version}\",\ + \"alertMsg\":\"${disk_msg}\"}" \ + ${ADMIN_URL}/${ALERT_URL} + fi +} +function check_disk() { + local folder=`cat /etc/taos/taos.cfg|grep -v "^#"|grep dataDir|awk '{print $NF}'` + if [ -z "$folder" ]; then + folder="/var/lib/taos" + fi + local mount_point="$folder" + local usage="" + while [ -z "$usage" ]; do + usage=`df -h|grep -w "${mount_point}"|awk '{print $5}'|grep -v Use|sed "s/%$//"` + if [ "x${mount_point}" = "x/" ]; then + break + fi + mount_point=`dirname ${mount_point}` + done + if [ -z "$usage" ]; then + disk_state="error" + disk_msg="cannot get disk usage" + if [ "$get_usage_ok" = "yes" ]; then + post_disk_error_msg + get_usage_ok="no" + fi + else + get_usage_ok="yes" + local current_level=0 + for level in ${disk_usage_level[*]}; do + if [ ${usage} -ge ${level} ]; then + disk_state="error" + disk_msg="disk usage over ${level}%" + current_level=${level} + fi + done + if [ ${current_level} -gt ${current_disk_level} ]; then + post_disk_error_msg + elif [ ${current_level} -lt ${current_disk_level} ]; then + echo "disk usage reduced from ${current_disk_level} to ${current_level}" + fi + current_disk_level=${current_level} + fi +} +function run_taosd() { + taosd + set_service_state "error" "taosd exit" + # post error msg + # check crash or OOM + check_taosd_exit_type + post_error_msg +} +function print_service_state_change() { + if [ "x$1" != "x${service_state}" ]; then + echo "`date` service state: ${service_state}, ${service_msg}" + fi +} +taosd_start_time=`date +%s` while ((1)) do + check_disk # echo "outer loop: $a" - sleep 10 - output=`taos -k` - status=${output:0:1} + output=`timeout $TAOS_TIMEOUT_SECOND taos -k` + if [ -z "${output}" ]; then + echo "`date` taos -k error" + status="" + else + status=${output:0:1} + fi # echo $output # echo $status if [ "$status"x = "0"x ] then - taosd & + # taosd_start_time=`date +%s` + run_taosd & fi # echo "$status"x "$TAOS_RUN_TAOSBENCHMARK_TEST"x "$TAOS_RUN_TAOSBENCHMARK_TEST_ONCE"x if [ "$status"x = "2"x ] && [ "$TAOS_RUN_TAOSBENCHMARK_TEST"x = "1"x ] && [ "$TAOS_RUN_TAOSBENCHMARK_TEST_ONCE"x = "0"x ] @@ -24,13 +168,37 @@ do taos -s "select stable_name from information_schema.ins_stables where db_name = 'test';"|grep -q -w meters if [ $? -ne 0 ]; then taosBenchmark -y -t 1000 -n 1000 -S 900000 - taos -s "create user admin_user pass 'NDS65R6t' sysinfo 0;" - taos -s "GRANT ALL on test.* to admin_user;" + taos -s "create user admin_user pass 'NDS65R6t' sysinfo 0;" + taos -s "GRANT ALL on test.* to admin_user;" fi fi + # check taosd status + if [ "$service_state" = "ready" ]; then + # check taosd status + check_taosd + print_service_state_change "ready" + if [ "$service_state" = "error" ]; then + post_error_msg + fi + elif [ "$service_state" = "init" ]; then + check_taosd "ignore" + # check timeout + current_time=`date +%s` + time_elapsed=$(( current_time - taosd_start_time )) + if [ ${time_elapsed} -gt ${TAOSD_STARTUP_TIMEOUT_SECOND} ]; then + set_service_state "error" "taosd startup timeout" + post_error_msg + fi + print_service_state_change "init" + elif [ "$service_state" = "error" ]; then + # check taosd status + check_taosd + print_service_state_change "error" + fi # check taosadapter nc -z localhost 6041 if [ $? -ne 0 ]; then - taosadapter & + taosadapter & fi + sleep 30 done From 54bc2fc2ae1ee6a35ab30256de3882832fe1e36e Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 18 Nov 2022 16:17:29 +0800 Subject: [PATCH 50/95] update case for restful mode run --- tests/system-test/2-query/abs.py | 14 ++++---------- tests/system-test/2-query/arccos.py | 17 ++++------------- tests/system-test/2-query/arcsin.py | 18 +++++------------- tests/system-test/2-query/arctan.py | 16 +++------------- tests/system-test/2-query/ceil.py | 11 +++-------- tests/system-test/2-query/cos.py | 20 ++------------------ tests/system-test/2-query/floor.py | 11 +++-------- tests/system-test/2-query/log.py | 1 - tests/system-test/2-query/round.py | 12 ++++-------- 9 files changed, 28 insertions(+), 92 deletions(-) diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index d7478a55a8..485abf6605 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -204,18 +204,12 @@ class TDTestCase: row_check.append(elem) auto_result.append(row_check) - check_status = True + + tdSql.query(abs_query) for row_index, row in enumerate(abs_result): for col_index, elem in enumerate(row): - if auto_result[row_index][col_index] != elem: - check_status = False - if not check_status: - tdLog.notice( - "abs function value has not as expected , sql is \"%s\" " % abs_query) - sys.exit(1) - else: - tdLog.info( - "abs value check pass , it work as expected ,sql is \"%s\" " % abs_query) + tdSql.checkData(row_index,col_index,auto_result[row_index][col_index]) + def test_errors(self): dbname = "testdb" diff --git a/tests/system-test/2-query/arccos.py b/tests/system-test/2-query/arccos.py index ed717741c5..df3061a0e0 100644 --- a/tests/system-test/2-query/arccos.py +++ b/tests/system-test/2-query/arccos.py @@ -86,21 +86,12 @@ class TDTestCase: row_check.append(elem) auto_result.append(row_check) - check_status = True - + tdSql.query(pow_query) for row_index , row in enumerate(pow_result): for col_index , elem in enumerate(row): - if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): - check_status = False - elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): - check_status = False - else: - pass - if not check_status: - tdLog.notice("acos function value has not as expected , sql is \"%s\" "%pow_query ) - sys.exit(1) - else: - tdLog.info("acos value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + tdSql.checkData(row_index,col_index,auto_result[row_index][col_index]) + def test_errors(self, dbname="db"): error_sql_lists = [ diff --git a/tests/system-test/2-query/arcsin.py b/tests/system-test/2-query/arcsin.py index 71de088979..73985005ba 100644 --- a/tests/system-test/2-query/arcsin.py +++ b/tests/system-test/2-query/arcsin.py @@ -86,21 +86,13 @@ class TDTestCase: row_check.append(elem) auto_result.append(row_check) - check_status = True - + + tdSql.query(pow_query) for row_index , row in enumerate(pow_result): for col_index , elem in enumerate(row): - if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): - check_status = False - elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): - check_status = False - else: - pass - if not check_status: - tdLog.notice("asin function value has not as expected , sql is \"%s\" "%pow_query ) - sys.exit(1) - else: - tdLog.info("asin value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + tdSql.checkData(row_index,col_index,auto_result[row_index][col_index]) + + def test_errors(self, dbname="db"): error_sql_lists = [ diff --git a/tests/system-test/2-query/arctan.py b/tests/system-test/2-query/arctan.py index 9780f9855b..5964a2955a 100644 --- a/tests/system-test/2-query/arctan.py +++ b/tests/system-test/2-query/arctan.py @@ -84,22 +84,12 @@ class TDTestCase: row_check.append(elem) auto_result.append(row_check) - check_status = True - + tdSql.query(pow_query) for row_index , row in enumerate(pow_result): for col_index , elem in enumerate(row): - if auto_result[row_index][col_index] == None and elem: - check_status = False - elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): - check_status = False - else: - pass - if not check_status: - tdLog.notice("atan function value has not as expected , sql is \"%s\" "%pow_query ) - sys.exit(1) - else: - tdLog.info("atan value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + tdSql.checkData(row_index,col_index,auto_result[row_index][col_index]) + def test_errors(self, dbname="db"): error_sql_lists = [ f"select atan from {dbname}.t1", diff --git a/tests/system-test/2-query/ceil.py b/tests/system-test/2-query/ceil.py index fffd484720..afc70d23a5 100644 --- a/tests/system-test/2-query/ceil.py +++ b/tests/system-test/2-query/ceil.py @@ -85,16 +85,11 @@ class TDTestCase: row_check.append(elem) auto_result.append(row_check) - check_status = True + tdSql.query(ceil_query) for row_index , row in enumerate(ceil_result): for col_index , elem in enumerate(row): - if auto_result[row_index][col_index] != elem: - check_status = False - if not check_status: - tdLog.notice("ceil function value has not as expected , sql is \"%s\" "%ceil_query ) - sys.exit(1) - else: - tdLog.info("ceil value check pass , it work as expected ,sql is \"%s\" "%ceil_query ) + tdSql.checkData(row_index,col_index,auto_result[row_index][col_index]) + def test_errors(self, dbname="db"): error_sql_lists = [ diff --git a/tests/system-test/2-query/cos.py b/tests/system-test/2-query/cos.py index d6bddc4e84..40b6257a5f 100644 --- a/tests/system-test/2-query/cos.py +++ b/tests/system-test/2-query/cos.py @@ -84,26 +84,10 @@ class TDTestCase: row_check.append(elem) auto_result.append(row_check) - check_status = True - print("========",pow_query, origin_query ) - + tdSql.query(pow_query) for row_index , row in enumerate(pow_result): for col_index , elem in enumerate(row): - if auto_result[row_index][col_index] == None and elem: - check_status = False - elif auto_result[row_index][col_index] != None and ((auto_result[row_index][col_index] != elem) and (str(auto_result[row_index][col_index])[:6] != str(elem)[:6] )): - # elif auto_result[row_index][col_index] != None and (abs(auto_result[row_index][col_index] - elem) > 0.000001): - print("=====") - print(row_index, col_index) - print(auto_result[row_index][col_index], elem, origin_result[row_index][col_index]) - check_status = False - else: - pass - if not check_status: - tdLog.notice("cos function value has not as expected , sql is \"%s\" "%pow_query ) - sys.exit(1) - else: - tdLog.info("cos value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + tdSql.checkData(row_index,col_index,auto_result[row_index][col_index]) def test_errors(self, dbname="db"): error_sql_lists = [ diff --git a/tests/system-test/2-query/floor.py b/tests/system-test/2-query/floor.py index 6a75872bcf..2b53868b1a 100644 --- a/tests/system-test/2-query/floor.py +++ b/tests/system-test/2-query/floor.py @@ -85,16 +85,11 @@ class TDTestCase: row_check.append(elem) auto_result.append(row_check) - check_status = True + tdSql.query(floor_query) for row_index , row in enumerate(floor_result): for col_index , elem in enumerate(row): - if auto_result[row_index][col_index] != elem: - check_status = False - if not check_status: - tdLog.notice("floor function value has not as expected , sql is \"%s\" "%floor_query ) - sys.exit(1) - else: - tdLog.info("floor value check pass , it work as expected ,sql is \"%s\" "%floor_query ) + tdSql.checkData(row_index,col_index,auto_result[row_index][col_index]) + def test_errors(self, dbname=DBNAME): error_sql_lists = [ diff --git a/tests/system-test/2-query/log.py b/tests/system-test/2-query/log.py index 7305a44f56..ba4fb8cc96 100644 --- a/tests/system-test/2-query/log.py +++ b/tests/system-test/2-query/log.py @@ -91,7 +91,6 @@ class TDTestCase: elem = math.log(elem , base) elif elem <=0: elem = None - row_check.append(elem) auto_result.append(row_check) diff --git a/tests/system-test/2-query/round.py b/tests/system-test/2-query/round.py index e3d98d6986..1ad96b93d7 100644 --- a/tests/system-test/2-query/round.py +++ b/tests/system-test/2-query/round.py @@ -81,16 +81,12 @@ class TDTestCase: row_check.append(elem) auto_result.append(row_check) - check_status = True + tdSql.query(round_query) for row_index , row in enumerate(round_result): for col_index , elem in enumerate(row): - if auto_result[row_index][col_index] != elem: - check_status = False - if not check_status: - tdLog.notice("round function value has not as expected , sql is \"%s\" "%round_query ) - sys.exit(1) - else: - tdLog.info("round value check pass , it work as expected ,sql is \"%s\" "%round_query ) + tdSql.checkData(row_index , col_index ,auto_result[row_index][col_index]) + + def test_errors(self, dbname="db"): error_sql_lists = [ From c5cb72291fed1106e97958d71bb4aaa397e6ac4e Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 18 Nov 2022 16:24:28 +0800 Subject: [PATCH 51/95] fix: fix windows compilation error --- source/libs/scalar/src/filter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index cd8606bc8e..239db9d25d 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1704,7 +1704,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) } qDebug("GROUP Num:%u", info->groupNum); - uint32_t maxDbgGrpNum = MIN(info->groupNum, 1000); + uint32_t maxDbgGrpNum = TMIN(info->groupNum, 1000); for (uint32_t i = 0; i < maxDbgGrpNum; ++i) { SFilterGroup *group = &info->groups[i]; qDebug("Group%d : unit num[%u]", i, group->unitNum); From 53fbc7b2567889558c088bfef7a39a396f0eebc7 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 18 Nov 2022 16:31:22 +0800 Subject: [PATCH 52/95] add more test cases --- tests/system-test/2-query/interp.py | 405 ++++++++++++++++++++++++++-- 1 file changed, 389 insertions(+), 16 deletions(-) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index a0ac4ab03b..ce57357abd 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -900,6 +900,8 @@ class TDTestCase: tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:55', NULL, NULL)") tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:01:00', 55, 60)") + # test fill linear + # check c0 tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:00') every(1s) fill(linear)") tdSql.checkRows(1) @@ -1370,35 +1372,406 @@ class TDTestCase: tdSql.query(f"select interp(c0),interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(linear)") tdSql.checkRows(61) tdSql.checkCols(2) - tdSql.checkData(0, 0, 0) - tdSql.checkData(10, 0, 10) - tdSql.checkData(20, 0, 20) - tdSql.checkData(30, 0, 30) + tdSql.checkData(0, 0, 0) # + tdSql.checkData(1, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) # + tdSql.checkData(6, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, 10) # + tdSql.checkData(11, 0, None) + tdSql.checkData(14, 0, None) + tdSql.checkData(15, 0, None) # + tdSql.checkData(16, 0, None) + tdSql.checkData(19, 0, None) + tdSql.checkData(20, 0, 20) # + tdSql.checkData(21, 0, None) + tdSql.checkData(24, 0, None) + tdSql.checkData(25, 0, None) # + tdSql.checkData(26, 0, None) + tdSql.checkData(29, 0, None) + tdSql.checkData(30, 0, 30) # tdSql.checkData(31, 0, 31) tdSql.checkData(32, 0, 32) tdSql.checkData(33, 0, 33) tdSql.checkData(34, 0, 34) - tdSql.checkData(35, 0, 35) + tdSql.checkData(35, 0, 35) # tdSql.checkData(36, 0, 36) tdSql.checkData(37, 0, 37) tdSql.checkData(38, 0, 38) tdSql.checkData(39, 0, 39) - tdSql.checkData(40, 0, 40) - tdSql.checkData(50, 0, 50) - tdSql.checkData(60, 0, 55) + tdSql.checkData(40, 0, 40) # + tdSql.checkData(41, 0, None) + tdSql.checkData(44, 0, None) + tdSql.checkData(45, 0, None) # + tdSql.checkData(46, 0, None) + tdSql.checkData(49, 0, None) + tdSql.checkData(50, 0, 50) # + tdSql.checkData(51, 0, None) + tdSql.checkData(54, 0, None) + tdSql.checkData(55, 0, None) # + tdSql.checkData(56, 0, None) + tdSql.checkData(59, 0, None) + tdSql.checkData(60, 0, 55) # - tdSql.checkData(0, 1, None) - tdSql.checkData(10, 1, 10) - tdSql.checkData(20, 1, None) - tdSql.checkData(30, 1, 30) - tdSql.checkData(40, 1, 40) + tdSql.checkData(0, 1, None) # + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, None) + tdSql.checkData(5, 1, None) # + tdSql.checkData(6, 1, None) + tdSql.checkData(9, 1, None) + tdSql.checkData(10, 1, 10) # + tdSql.checkData(11, 1, None) + tdSql.checkData(14, 1, None) + tdSql.checkData(15, 1, None) # + tdSql.checkData(16, 1, None) + tdSql.checkData(19, 1, None) + tdSql.checkData(20, 1, None) # + tdSql.checkData(21, 1, None) + tdSql.checkData(24, 1, None) + tdSql.checkData(25, 1, None) # + tdSql.checkData(26, 1, None) + tdSql.checkData(29, 1, None) + tdSql.checkData(30, 1, 30) # + tdSql.checkData(31, 1, None) + tdSql.checkData(34, 1, None) + tdSql.checkData(35, 1, None) # + tdSql.checkData(36, 1, None) + tdSql.checkData(39, 1, None) + tdSql.checkData(40, 1, 40) # tdSql.checkData(41, 1, 41) tdSql.checkData(42, 1, 42) tdSql.checkData(43, 1, 43) tdSql.checkData(44, 1, 44) - tdSql.checkData(45, 1, 45) - tdSql.checkData(50, 1, None) - tdSql.checkData(60, 1, 60) + tdSql.checkData(45, 1, 45) # + tdSql.checkData(46, 1, None) + tdSql.checkData(49, 1, None) + tdSql.checkData(50, 1, None) # + tdSql.checkData(51, 1, None) + tdSql.checkData(54, 1, None) + tdSql.checkData(55, 1, None) # + tdSql.checkData(56, 1, None) + tdSql.checkData(59, 1, None) + tdSql.checkData(60, 1, 60) # + + # test fill null + tdSql.query(f"select interp(c0),interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(null)") + tdSql.checkRows(61) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 0) # + tdSql.checkData(1, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) # + tdSql.checkData(6, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, 10) # + tdSql.checkData(11, 0, None) + tdSql.checkData(14, 0, None) + tdSql.checkData(15, 0, None) # + tdSql.checkData(16, 0, None) + tdSql.checkData(19, 0, None) + tdSql.checkData(20, 0, 20) # + tdSql.checkData(21, 0, None) + tdSql.checkData(24, 0, None) + tdSql.checkData(25, 0, None) # + tdSql.checkData(26, 0, None) + tdSql.checkData(29, 0, None) + tdSql.checkData(30, 0, 30) # + tdSql.checkData(31, 0, None) + tdSql.checkData(34, 0, None) + tdSql.checkData(35, 0, 35) # + tdSql.checkData(36, 0, None) + tdSql.checkData(39, 0, None) + tdSql.checkData(40, 0, 40) # + tdSql.checkData(41, 0, None) + tdSql.checkData(44, 0, None) + tdSql.checkData(45, 0, None) # + tdSql.checkData(46, 0, None) + tdSql.checkData(49, 0, None) + tdSql.checkData(50, 0, 50) # + tdSql.checkData(51, 0, None) + tdSql.checkData(54, 0, None) + tdSql.checkData(55, 0, None) # + tdSql.checkData(56, 0, None) + tdSql.checkData(59, 0, None) + tdSql.checkData(60, 0, 55) # + + tdSql.checkData(0, 1, None) # + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, None) + tdSql.checkData(5, 1, None) # + tdSql.checkData(6, 1, None) + tdSql.checkData(9, 1, None) + tdSql.checkData(10, 1, 10) # + tdSql.checkData(11, 1, None) + tdSql.checkData(14, 1, None) + tdSql.checkData(15, 1, None) # + tdSql.checkData(16, 1, None) + tdSql.checkData(19, 1, None) + tdSql.checkData(20, 1, None) # + tdSql.checkData(21, 1, None) + tdSql.checkData(24, 1, None) + tdSql.checkData(25, 1, None) # + tdSql.checkData(26, 1, None) + tdSql.checkData(29, 1, None) + tdSql.checkData(30, 1, 30) # + tdSql.checkData(31, 1, None) + tdSql.checkData(34, 1, None) + tdSql.checkData(35, 1, None) # + tdSql.checkData(36, 1, None) + tdSql.checkData(39, 1, None) + tdSql.checkData(40, 1, 40) # + tdSql.checkData(41, 1, None) + tdSql.checkData(44, 1, None) + tdSql.checkData(45, 1, 45) # + tdSql.checkData(46, 1, None) + tdSql.checkData(49, 1, None) + tdSql.checkData(50, 1, None) # + tdSql.checkData(51, 1, None) + tdSql.checkData(54, 1, None) + tdSql.checkData(55, 1, None) # + tdSql.checkData(56, 1, None) + tdSql.checkData(59, 1, None) + tdSql.checkData(60, 1, 60) # + + # test fill value + tdSql.query(f"select interp(c0),interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(value, 123)") + tdSql.checkRows(61) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 0) # + tdSql.checkData(1, 0, 123) + tdSql.checkData(4, 0, 123) + tdSql.checkData(5, 0, None) # + tdSql.checkData(6, 0, 123) + tdSql.checkData(9, 0, 123) + tdSql.checkData(10, 0, 10) # + tdSql.checkData(11, 0, 123) + tdSql.checkData(14, 0, 123) + tdSql.checkData(15, 0, None) # + tdSql.checkData(16, 0, 123) + tdSql.checkData(19, 0, 123) + tdSql.checkData(20, 0, 20) # + tdSql.checkData(21, 0, 123) + tdSql.checkData(24, 0, 123) + tdSql.checkData(25, 0, None) # + tdSql.checkData(26, 0, 123) + tdSql.checkData(29, 0, 123) + tdSql.checkData(30, 0, 30) # + tdSql.checkData(31, 0, 123) + tdSql.checkData(34, 0, 123) + tdSql.checkData(35, 0, 35) # + tdSql.checkData(36, 0, 123) + tdSql.checkData(39, 0, 123) + tdSql.checkData(40, 0, 40) # + tdSql.checkData(41, 0, 123) + tdSql.checkData(44, 0, 123) + tdSql.checkData(45, 0, None) # + tdSql.checkData(46, 0, 123) + tdSql.checkData(49, 0, 123) + tdSql.checkData(50, 0, 50) # + tdSql.checkData(51, 0, 123) + tdSql.checkData(54, 0, 123) + tdSql.checkData(55, 0, None) # + tdSql.checkData(59, 0, 123) + tdSql.checkData(60, 0, 55) # + + tdSql.checkData(0, 1, None) # + tdSql.checkData(1, 1, 123) + tdSql.checkData(4, 1, 123) + tdSql.checkData(5, 1, None) # + tdSql.checkData(6, 1, 123) + tdSql.checkData(9, 1, 123) + tdSql.checkData(10, 1, 10) # + tdSql.checkData(11, 1, 123) + tdSql.checkData(14, 1, 123) + tdSql.checkData(15, 1, None) # + tdSql.checkData(16, 1, 123) + tdSql.checkData(19, 1, 123) + tdSql.checkData(20, 1, None) # + tdSql.checkData(21, 1, 123) + tdSql.checkData(24, 1, 123) + tdSql.checkData(25, 1, None) # + tdSql.checkData(26, 1, 123) + tdSql.checkData(29, 1, 123) + tdSql.checkData(30, 1, 30) # + tdSql.checkData(31, 1, 123) + tdSql.checkData(34, 1, 123) + tdSql.checkData(35, 1, None) # + tdSql.checkData(36, 1, 123) + tdSql.checkData(39, 1, 123) + tdSql.checkData(40, 1, 40) # + tdSql.checkData(41, 1, 123) + tdSql.checkData(44, 1, 123) + tdSql.checkData(45, 1, 45) # + tdSql.checkData(46, 1, 123) + tdSql.checkData(49, 1, 123) + tdSql.checkData(50, 1, None) # + tdSql.checkData(51, 1, 123) + tdSql.checkData(54, 1, 123) + tdSql.checkData(55, 1, None) # + tdSql.checkData(56, 1, 123) + tdSql.checkData(59, 1, 123) + tdSql.checkData(60, 1, 60) # + + # test fill prev + tdSql.query(f"select interp(c0),interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(prev)") + tdSql.checkRows(61) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 0) # + tdSql.checkData(1, 0, 0) + tdSql.checkData(4, 0, 0) + tdSql.checkData(5, 0, None) # + tdSql.checkData(6, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, 10) # + tdSql.checkData(11, 0, 10) + tdSql.checkData(14, 0, 10) + tdSql.checkData(15, 0, None) # + tdSql.checkData(16, 0, None) + tdSql.checkData(19, 0, None) + tdSql.checkData(20, 0, 20) # + tdSql.checkData(21, 0, 20) + tdSql.checkData(24, 0, 20) + tdSql.checkData(25, 0, None) # + tdSql.checkData(26, 0, None) + tdSql.checkData(29, 0, None) + tdSql.checkData(30, 0, 30) # + tdSql.checkData(31, 0, 30) + tdSql.checkData(34, 0, 30) + tdSql.checkData(35, 0, 35) # + tdSql.checkData(36, 0, 35) + tdSql.checkData(39, 0, 35) + tdSql.checkData(40, 0, 40) # + tdSql.checkData(41, 0, 40) + tdSql.checkData(44, 0, 40) + tdSql.checkData(45, 0, None) # + tdSql.checkData(46, 0, None) + tdSql.checkData(49, 0, None) + tdSql.checkData(50, 0, 50) # + tdSql.checkData(51, 0, 50) + tdSql.checkData(54, 0, 50) + tdSql.checkData(55, 0, None) # + tdSql.checkData(56, 0, None) + tdSql.checkData(59, 0, None) + tdSql.checkData(60, 0, 55) # + + tdSql.checkData(0, 1, None) # + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, None) + tdSql.checkData(5, 1, None) # + tdSql.checkData(6, 1, None) + tdSql.checkData(9, 1, None) + tdSql.checkData(10, 1, 10) # + tdSql.checkData(11, 1, 10) + tdSql.checkData(14, 1, 10) + tdSql.checkData(15, 1, None) # + tdSql.checkData(16, 1, None) + tdSql.checkData(19, 1, None) + tdSql.checkData(20, 1, None) # + tdSql.checkData(21, 1, None) + tdSql.checkData(24, 1, None) + tdSql.checkData(25, 1, None) # + tdSql.checkData(26, 1, None) + tdSql.checkData(29, 1, None) + tdSql.checkData(30, 1, 30) # + tdSql.checkData(31, 1, 30) + tdSql.checkData(34, 1, 30) + tdSql.checkData(35, 1, None) # + tdSql.checkData(36, 1, None) + tdSql.checkData(39, 1, None) + tdSql.checkData(40, 1, 40) # + tdSql.checkData(41, 1, 40) + tdSql.checkData(44, 1, 40) + tdSql.checkData(45, 1, 45) # + tdSql.checkData(46, 1, 45) + tdSql.checkData(49, 1, 45) + tdSql.checkData(50, 1, None) # + tdSql.checkData(51, 1, None) + tdSql.checkData(54, 1, None) + tdSql.checkData(55, 1, None) # + tdSql.checkData(56, 1, None) + tdSql.checkData(59, 1, None) + tdSql.checkData(60, 1, 60) # + + # test fill next + tdSql.query(f"select interp(c0),interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(next)") + tdSql.checkRows(61) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 0) # + tdSql.checkData(1, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) # + tdSql.checkData(6, 0, 10) + tdSql.checkData(9, 0, 10) + tdSql.checkData(10, 0, 10) # + tdSql.checkData(11, 0, None) + tdSql.checkData(14, 0, None) + tdSql.checkData(15, 0, None) # + tdSql.checkData(16, 0, 20) + tdSql.checkData(19, 0, 20) + tdSql.checkData(20, 0, 20) # + tdSql.checkData(21, 0, None) + tdSql.checkData(24, 0, None) + tdSql.checkData(25, 0, None) # + tdSql.checkData(26, 0, 30) + tdSql.checkData(29, 0, 30) + tdSql.checkData(30, 0, 30) # + tdSql.checkData(31, 0, 35) + tdSql.checkData(34, 0, 35) + tdSql.checkData(35, 0, 35) # + tdSql.checkData(36, 0, 40) + tdSql.checkData(39, 0, 40) + tdSql.checkData(40, 0, 40) # + tdSql.checkData(41, 0, None) + tdSql.checkData(44, 0, None) + tdSql.checkData(45, 0, None) # + tdSql.checkData(46, 0, 50) + tdSql.checkData(49, 0, 50) + tdSql.checkData(50, 0, 50) # + tdSql.checkData(51, 0, None) + tdSql.checkData(54, 0, None) + tdSql.checkData(55, 0, None) # + tdSql.checkData(56, 0, 55) + tdSql.checkData(59, 0, 55) + tdSql.checkData(60, 0, 55) # + + tdSql.checkData(0, 1, None) # + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, None) + tdSql.checkData(5, 1, None) # + tdSql.checkData(6, 1, 10) + tdSql.checkData(9, 1, 10) + tdSql.checkData(10, 1, 10) # + tdSql.checkData(11, 1, None) + tdSql.checkData(14, 1, None) + tdSql.checkData(15, 1, None) # + tdSql.checkData(16, 1, None) + tdSql.checkData(19, 1, None) + tdSql.checkData(20, 1, None) # + tdSql.checkData(21, 1, None) + tdSql.checkData(24, 1, None) + tdSql.checkData(25, 1, None) # + tdSql.checkData(26, 1, 30) + tdSql.checkData(29, 1, 30) + tdSql.checkData(30, 1, 30) # + tdSql.checkData(31, 1, None) + tdSql.checkData(34, 1, None) + tdSql.checkData(35, 1, None) # + tdSql.checkData(36, 1, 40) + tdSql.checkData(39, 1, 40) + tdSql.checkData(40, 1, 40) # + tdSql.checkData(41, 1, 45) + tdSql.checkData(44, 1, 45) + tdSql.checkData(45, 1, 45) # + tdSql.checkData(46, 1, None) + tdSql.checkData(49, 1, None) + tdSql.checkData(50, 1, None) # + tdSql.checkData(51, 1, None) + tdSql.checkData(54, 1, None) + tdSql.checkData(55, 1, None) # + tdSql.checkData(56, 1, 60) + tdSql.checkData(59, 1, 60) + tdSql.checkData(60, 1, 60) # tdLog.printNoPrefix("==========step11:test multi-interp cases") tdSql.query(f"select interp(c0),interp(c1),interp(c2),interp(c3) from {dbname}.{tbname} range('2020-02-09 00:00:05', '2020-02-13 00:00:05') every(1d) fill(null)") From 3f9fd91ad7f504891f263fc95bf1e8fad2ca1bbb Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 18 Nov 2022 17:03:14 +0800 Subject: [PATCH 53/95] update --- tests/system-test/2-query/tail.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/system-test/2-query/tail.py b/tests/system-test/2-query/tail.py index f925380c09..fe2f24ad25 100644 --- a/tests/system-test/2-query/tail.py +++ b/tests/system-test/2-query/tail.py @@ -436,16 +436,16 @@ class TDTestCase: ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+1s, -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) - tdSql.query(f"select tail(c2,2) from {dbname}.sub1_bound order by 1 desc") - tdSql.checkRows(2) - tdSql.checkData(0,0,9223372036854775803) + tdSql.query(f"select tail(c2,1) from {dbname}.sub1_bound order by 1 desc") + tdSql.checkRows(1) + tdSql.checkData(0,0,-9223372036854775803) def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() From a39dc7b2bb906898d454ce64198f1163831b26e1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 18 Nov 2022 17:07:09 +0800 Subject: [PATCH 54/95] fix(query): init cache. --- source/libs/executor/src/scanoperator.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 26117b442e..a110b73763 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -4722,6 +4722,12 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; + pInfo->base.metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5); + if (pInfo->base.metaCache.pTableMetaEntryCache == NULL) { + code = terrno; + goto _error; + } + pInfo->base.dataBlockLoadFlag = FUNC_DATA_REQUIRED_DATA_LOAD; pInfo->base.scanFlag = MAIN_SCAN; pInfo->base.readHandle = *readHandle; From 0cb539b181b28b434dbed7cbd53fdf6b273a555c Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 18 Nov 2022 17:09:43 +0800 Subject: [PATCH 55/95] update --- tests/system-test/2-query/avg.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/system-test/2-query/avg.py b/tests/system-test/2-query/avg.py index ec7ec34ed3..f68d0f5197 100644 --- a/tests/system-test/2-query/avg.py +++ b/tests/system-test/2-query/avg.py @@ -114,16 +114,10 @@ class TDTestCase: avg_result = tdSql.getResult(origin_query) origin_result = tdSql.getResult(check_query) - check_status = True + tdSql.query(origin_query) for row_index , row in enumerate(avg_result): for col_index , elem in enumerate(row): - if avg_result[row_index][col_index] != origin_result[row_index][col_index]: - check_status = False - if not check_status: - tdLog.notice("avg function value has not as expected , sql is \"%s\" "%origin_query ) - sys.exit(1) - else: - tdLog.info("avg value check pass , it work as expected ,sql is \"%s\" "%check_query ) + tdSql.checkData(row_index,col_index,origin_result[row_index][col_index]) def test_errors(self, dbname="db"): error_sql_lists = [ From 963ca6b14be17800ac8bb9b6cff33cbe6e0ce54a Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 18 Nov 2022 17:18:01 +0800 Subject: [PATCH 56/95] update --- tests/system-test/2-query/avg.py | 14 +++++++------- .../system-test/2-query/function_stateduration.py | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/system-test/2-query/avg.py b/tests/system-test/2-query/avg.py index f68d0f5197..910dd524cb 100644 --- a/tests/system-test/2-query/avg.py +++ b/tests/system-test/2-query/avg.py @@ -372,33 +372,33 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483645, 9223372036854775805, 32765, 125, 3.40E+37, 1.7e+307, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, 2147483645, 9223372036854775805, 32765, 125, 3.40E+37, 1.7e+307, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483644, 9223372036854775804, 32764, 124, 3.40E+37, 1.7e+307, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483644, 9223372036854775804, 32764, 124, 3.40E+37, 1.7e+307, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+15s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+20s, 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) #self.check_avg(f"select avg(c1), avg(c2), avg(c3) , avg(c4), avg(c5) ,avg(c6) from {dbname}.sub1_bound " , f" select sum(c1)/count(c1), sum(c2)/count(c2) ,sum(c3)/count(c3), sum(c4)/count(c4), sum(c5)/count(c5) ,sum(c6)/count(c6) from {dbname}.sub1_bound ") diff --git a/tests/system-test/2-query/function_stateduration.py b/tests/system-test/2-query/function_stateduration.py index ad9b8b0d79..728df0b91f 100644 --- a/tests/system-test/2-query/function_stateduration.py +++ b/tests/system-test/2-query/function_stateduration.py @@ -364,10 +364,10 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( @@ -375,15 +375,15 @@ class TDTestCase: ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+15s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.query(f"select stateduration(c1,'GT',1,1s) from {dbname}.sub1_bound") From 54e73354775b4e83abcd09222b87ac8a02a4f20c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 18 Nov 2022 17:27:05 +0800 Subject: [PATCH 57/95] fix(query): set the scan flag. --- source/libs/executor/inc/executorimpl.h | 1 - source/libs/executor/src/scanoperator.c | 7 ++++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index b4e786746b..c330ee0596 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -334,7 +334,6 @@ typedef struct STableScanInfo { SScanInfo scanInfo; int32_t scanTimes; SSDataBlock* pResBlock; - int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan SSampleExecInfo sample; // sample execution info int32_t currentGroupId; int32_t currentTable; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a110b73763..8e86cae4bd 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -767,7 +767,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { if (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); - pTableScanInfo->scanFlag = REPEAT_SCAN; + pTableScanInfo->base.scanFlag = REPEAT_SCAN; qDebug("start to repeat ascending order scan data blocks due to query func required, %s", GET_TASKID(pTaskInfo)); // do prepare for the next round table scan operation @@ -793,7 +793,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { if (pTableScanInfo->scanTimes < total) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); - pTableScanInfo->scanFlag = REPEAT_SCAN; + pTableScanInfo->base.scanFlag = REPEAT_SCAN; qDebug("%s start to repeat descending order scan data blocks", GET_TASKID(pTaskInfo)); tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); @@ -942,6 +942,8 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, } pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; + + pInfo->base.scanFlag = MAIN_SCAN; pInfo->base.pdInfo.interval = extractIntervalInfo(pTableScanNode); pInfo->base.readHandle = *readHandle; pInfo->sample.sampleRatio = pTableScanNode->ratio; @@ -958,7 +960,6 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, goto _error; } - pInfo->scanFlag = MAIN_SCAN; pInfo->currentGroupId = -1; pInfo->assignBlockUid = pTableScanNode->assignBlockUid; From d07dc94446c3356b4fb1062f7f27601f51d84b96 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 18 Nov 2022 18:14:52 +0800 Subject: [PATCH 58/95] update --- tests/system-test/2-query/abs.py | 8 ++-- tests/system-test/2-query/and_or_for_byte.py | 8 ++-- tests/system-test/2-query/arccos.py | 8 ++-- tests/system-test/2-query/arcsin.py | 8 ++-- tests/system-test/2-query/arctan.py | 8 ++-- tests/system-test/2-query/ceil.py | 10 ++--- tests/system-test/2-query/cos.py | 6 +-- tests/system-test/2-query/floor.py | 10 ++--- tests/system-test/2-query/last_row.py | 8 ++-- tests/system-test/2-query/log.py | 8 ++-- tests/system-test/2-query/pow.py | 8 ++-- tests/system-test/2-query/round.py | 10 ++--- tests/system-test/2-query/sin.py | 8 ++-- tests/system-test/2-query/sqrt.py | 8 ++-- tests/system-test/2-query/statecount.py | 10 ++--- tests/system-test/2-query/tail.py | 10 ++--- tests/system-test/2-query/tan.py | 8 ++-- tests/system-test/2-query/txt.txt | 40 ++++++++++++++++++++ tests/system-test/2-query/unique.py | 12 +++--- 19 files changed, 118 insertions(+), 78 deletions(-) create mode 100644 tests/system-test/2-query/txt.txt diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index 485abf6605..d64d550bc4 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -460,19 +460,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto(f"select c1, c2, c3 , c4, c5 ,c6 from {dbname}.sub1_bound ", f"select abs(c1), abs(c2) ,abs(c3), abs(c4), abs(c5) ,abs(c6) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/and_or_for_byte.py b/tests/system-test/2-query/and_or_for_byte.py index 479918f2f9..15e9110b3b 100644 --- a/tests/system-test/2-query/and_or_for_byte.py +++ b/tests/system-test/2-query/and_or_for_byte.py @@ -426,19 +426,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_function("&", False , f"{dbname}.sub1_bound" ,"c1","c2","c3","c4","c5","c6" ) self.check_function("&", False , f"{dbname}.sub1_bound","abs(c1)","abs(c2)","abs(c3)","abs(c4)","abs(c5)","abs(c6)" ) diff --git a/tests/system-test/2-query/arccos.py b/tests/system-test/2-query/arccos.py index df3061a0e0..f22d393ecd 100644 --- a/tests/system-test/2-query/arccos.py +++ b/tests/system-test/2-query/arccos.py @@ -405,19 +405,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto_acos( f"select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from {dbname}.sub1_bound ", f"select acos(abs(c1)), acos(abs(c2)) ,acos(abs(c3)), acos(abs(c4)), acos(abs(c5)) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/arcsin.py b/tests/system-test/2-query/arcsin.py index 73985005ba..1872518c5d 100644 --- a/tests/system-test/2-query/arcsin.py +++ b/tests/system-test/2-query/arcsin.py @@ -406,19 +406,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto_asin( f"select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from {dbname}.sub1_bound ", f"select asin(abs(c1)), asin(abs(c2)) ,asin(abs(c3)), asin(abs(c4)), asin(abs(c5)) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/arctan.py b/tests/system-test/2-query/arctan.py index 5964a2955a..9561637b76 100644 --- a/tests/system-test/2-query/arctan.py +++ b/tests/system-test/2-query/arctan.py @@ -402,19 +402,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto_atan( f"select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from {dbname}.sub1_bound ", f"select atan(abs(c1)), atan(abs(c2)) ,atan(abs(c3)), atan(abs(c4)), atan(abs(c5)) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/ceil.py b/tests/system-test/2-query/ceil.py index afc70d23a5..aabc716a74 100644 --- a/tests/system-test/2-query/ceil.py +++ b/tests/system-test/2-query/ceil.py @@ -372,10 +372,10 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( @@ -383,15 +383,15 @@ class TDTestCase: ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+15s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto( f"select c1, c2, c3 , c4, c5 ,c6 from {dbname}.sub1_bound ", f"select ceil(c1), ceil(c2) ,ceil(c3), ceil(c4), ceil(c5) ,ceil(c6) from {dbname}.sub1_bound") self.check_result_auto( f"select c1, c2, c3 , c3, c2 ,c1 from {dbname}.sub1_bound ", f"select ceil(c1), ceil(c2) ,ceil(c3), ceil(c3), ceil(c2) ,ceil(c1) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/cos.py b/tests/system-test/2-query/cos.py index 40b6257a5f..d2056805eb 100644 --- a/tests/system-test/2-query/cos.py +++ b/tests/system-test/2-query/cos.py @@ -397,16 +397,16 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) # self.check_result_auto_cos( f"select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from {dbname}.sub1_bound ", f"select cos(abs(c1)), cos(abs(c2)) ,cos(abs(c3)), cos(abs(c4)), cos(abs(c5)) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/floor.py b/tests/system-test/2-query/floor.py index 2b53868b1a..bf78aa8bfa 100644 --- a/tests/system-test/2-query/floor.py +++ b/tests/system-test/2-query/floor.py @@ -383,10 +383,10 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( @@ -394,15 +394,15 @@ class TDTestCase: ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+15s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto( f"select c1, c2, c3 , c4, c5 ,c6 from {dbname}.sub1_bound ", f"select floor(c1), floor(c2) ,floor(c3), floor(c4), floor(c5) ,floor(c6) from {dbname}.sub1_bound") self.check_result_auto( f"select c1, c2, c3 , c3, c2 ,c1 from {dbname}.sub1_bound ", f"select floor(c1), floor(c2) ,floor(c3), floor(c3), floor(c2) ,floor(c1) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index f8d6ce4c6c..01da658989 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -572,19 +572,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) # check basic elem for table per row diff --git a/tests/system-test/2-query/log.py b/tests/system-test/2-query/log.py index ba4fb8cc96..a05de53c42 100644 --- a/tests/system-test/2-query/log.py +++ b/tests/system-test/2-query/log.py @@ -518,19 +518,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto_log(None , f"select c1, c2, c3 , c4, c5 ,c6 from {dbname}.sub1_bound ", f"select log(c1), log(c2) ,log(c3), log(c4), log(c5) ,log(c6) from {dbname}.sub1_bound") self.check_result_auto_log( 2 , f"select c1, c2, c3 , c4, c5 ,c6 from {dbname}.sub1_bound ", f"select log(c1,2), log(c2,2) ,log(c3,2), log(c4,2), log(c5,2) ,log(c6,2) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/pow.py b/tests/system-test/2-query/pow.py index a067d66547..5647d81e27 100644 --- a/tests/system-test/2-query/pow.py +++ b/tests/system-test/2-query/pow.py @@ -507,19 +507,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto_pow(2, f"select c1, c3 , c4, c5 from {dbname}.sub1_bound ", f"select pow(c1,2), pow(c3,2), pow(c4,2), pow(c5,2) from {dbname}.sub1_bound") self.check_result_auto_pow(3, f"select c1, c3 , c4, c5 from {dbname}.sub1_bound ", f"select pow(c1,3), pow(c3,3), pow(c4,3), pow(c5,3) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/round.py b/tests/system-test/2-query/round.py index 1ad96b93d7..d647f516ae 100644 --- a/tests/system-test/2-query/round.py +++ b/tests/system-test/2-query/round.py @@ -384,10 +384,10 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( @@ -395,15 +395,15 @@ class TDTestCase: ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+15s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto( f"select c1, c2, c3 , c4, c5 ,c6 from {dbname}.sub1_bound ", f"select round(c1), round(c2) ,round(c3), round(c4), round(c5) ,round(c6) from {dbname}.sub1_bound") self.check_result_auto( f"select c1, c2, c3 , c3, c2 ,c1 from {dbname}.sub1_bound ", f"select round(c1), round(c2) ,round(c3), round(c3), round(c2) ,round(c1) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/sin.py b/tests/system-test/2-query/sin.py index 4fdec8fd73..c65e58852d 100644 --- a/tests/system-test/2-query/sin.py +++ b/tests/system-test/2-query/sin.py @@ -394,19 +394,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto_sin( f"select abs(c1), abs(c2), abs(c3) , abs(c4) from {dbname}.sub1_bound ", f"select sin(abs(c1)), sin(abs(c2)) ,sin(abs(c3)), sin(abs(c4)) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/sqrt.py b/tests/system-test/2-query/sqrt.py index 9229444f74..4db166808e 100644 --- a/tests/system-test/2-query/sqrt.py +++ b/tests/system-test/2-query/sqrt.py @@ -443,19 +443,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto_sqrt( f"select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from {dbname}.sub1_bound ", f"select sqrt(abs(c1)), sqrt(abs(c2)) ,sqrt(abs(c3)), sqrt(abs(c4)), sqrt(abs(c5)) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/statecount.py b/tests/system-test/2-query/statecount.py index 2aa9194d37..f76e153014 100644 --- a/tests/system-test/2-query/statecount.py +++ b/tests/system-test/2-query/statecount.py @@ -451,10 +451,10 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( @@ -462,15 +462,15 @@ class TDTestCase: ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+15s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.query(f"select statecount(c1,'GT',1) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/tail.py b/tests/system-test/2-query/tail.py index fe2f24ad25..43321810d0 100644 --- a/tests/system-test/2-query/tail.py +++ b/tests/system-test/2-query/tail.py @@ -421,10 +421,10 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( @@ -432,15 +432,15 @@ class TDTestCase: ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()+1s, -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+15s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.query(f"select tail(c2,1) from {dbname}.sub1_bound order by 1 desc") diff --git a/tests/system-test/2-query/tan.py b/tests/system-test/2-query/tan.py index 27e6efb475..e689eaba20 100644 --- a/tests/system-test/2-query/tan.py +++ b/tests/system-test/2-query/tan.py @@ -391,19 +391,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto_tan( f"select abs(c1), abs(c2), abs(c3) , abs(c4) from {dbname}.sub1_bound ", f"select tan(abs(c1)), tan(abs(c2)) ,tan(abs(c3)), tan(abs(c4)) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/txt.txt b/tests/system-test/2-query/txt.txt new file mode 100644 index 0000000000..fd3b0b5e36 --- /dev/null +++ b/tests/system-test/2-query/txt.txt @@ -0,0 +1,40 @@ +abs.py: def check_boundary_values(self): +abs.py: self.check_boundary_values() +and_or_for_byte.py: def check_boundary_values(self, dbname="bound_test"): +and_or_for_byte.py: self.check_boundary_values() +arccos.py: def check_boundary_values(self, dbname="bound_test"): +arccos.py: self.check_boundary_values() +arcsin.py: def check_boundary_values(self, dbname="bound_test"): +arcsin.py: self.check_boundary_values() +arctan.py: def check_boundary_values(self, dbname="bound_test"): +arctan.py: self.check_boundary_values() +avg.py: def check_boundary_values(self, dbname="bound_test"): +avg.py: self.check_boundary_values() +ceil.py: def check_boundary_values(self, dbname="bound_test"): +ceil.py: self.check_boundary_values() +cos.py: def check_boundary_values(self, dbname="bound_test"): +cos.py: self.check_boundary_values() +floor.py: def check_boundary_values(self, dbname="bound_test"): +floor.py: self.check_boundary_values() +function_stateduration.py: def check_boundary_values(self, dbname="bound_test"): +function_stateduration.py: self.check_boundary_values() +last_row.py: def check_boundary_values(self, dbname="bound_test"): +last_row.py: self.check_boundary_values() +log.py: def check_boundary_values(self, dbname="bound_test"): +log.py: self.check_boundary_values() +pow.py: def check_boundary_values(self, dbname="bound_test"): +pow.py: self.check_boundary_values() +round.py: def check_boundary_values(self, dbname="bound_test"): +round.py: self.check_boundary_values() +sin.py: def check_boundary_values(self, dbname="testdb"): +sin.py: self.check_boundary_values() +sqrt.py: def check_boundary_values(self, dbname="bound_test"): +sqrt.py: self.check_boundary_values() +statecount.py: def check_boundary_values(self, dbname="bound_test"): +statecount.py: self.check_boundary_values() +tail.py: def check_boundary_values(self, dbname="bound_test"): +tail.py: self.check_boundary_values() +tan.py: def check_boundary_values(self, dbname="bound_test"): +tan.py: self.check_boundary_values() +unique.py: def check_boundary_values(self, dbname="bound_test"): +unique.py: self.check_boundary_values() diff --git a/tests/system-test/2-query/unique.py b/tests/system-test/2-query/unique.py index 6c51854b43..2b0336d2d7 100644 --- a/tests/system-test/2-query/unique.py +++ b/tests/system-test/2-query/unique.py @@ -451,26 +451,26 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()+1s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()+2s, 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()+3s, -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+20s, -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+30s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.query(f"select unique(c2) from {dbname}.sub1_bound order by 1 desc") From 67758885453a156f99386564ef22c669f36f3e70 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 18 Nov 2022 18:21:22 +0800 Subject: [PATCH 59/95] fix memleak --- source/libs/executor/src/scanoperator.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index d9b985516e..e1d34034ef 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -524,7 +524,7 @@ static STableCachedVal* createTableCacheVal(const SMetaReader* pMetaReader) { // only child table has tag value if (pMetaReader->me.type == TSDB_CHILD_TABLE) { - STag* pTag = (STag*) pMetaReader->me.ctbEntry.pTags; + STag* pTag = (STag*)pMetaReader->me.ctbEntry.pTags; pVal->pTags = taosMemoryMalloc(pTag->len); memcpy(pVal->pTags, pTag, pTag->len); } @@ -560,7 +560,8 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int code = metaGetTableEntryByUid(&mr, pBlock->info.uid); if (code != TSDB_CODE_SUCCESS) { if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { - qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), idStr); + qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid, + tstrerror(terrno), idStr); } else { qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), idStr); } @@ -583,9 +584,11 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int code = metaGetTableEntryByUid(&mr, pBlock->info.uid); if (code != TSDB_CODE_SUCCESS) { if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { - qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), idStr); + qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", + pBlock->info.uid, tstrerror(terrno), idStr); } else { - qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), idStr); + qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), + idStr); } metaReaderClear(&mr); return terrno; @@ -4675,6 +4678,7 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { taosArrayClear(pInfo->sortSourceParams); tsortDestroySortHandle(pInfo->pSortHandle); + pInfo->pSortHandle = NULL; for (int32_t i = 0; i < taosArrayGetSize(pInfo->queryConds); i++) { SQueryTableDataCond* cond = taosArrayGet(pInfo->queryConds, i); @@ -4773,6 +4777,8 @@ void destroyTableMergeScanOperatorInfo(void* param) { } taosArrayDestroy(pTableScanInfo->sortSourceParams); + tsortDestroySortHandle(pTableScanInfo->pSortHandle); + pTableScanInfo->pSortHandle = NULL; tsdbReaderClose(pTableScanInfo->pReader); pTableScanInfo->pReader = NULL; @@ -4866,7 +4872,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false); initLimitInfo(pTableScanNode->scan.node.pLimit, pTableScanNode->scan.node.pSlimit, &pInfo->limitInfo); - int32_t rowSize = pInfo->pResBlock->info.rowSize; + int32_t rowSize = pInfo->pResBlock->info.rowSize; uint32_t nCols = taosArrayGetSize(pInfo->pResBlock->pDataBlock); pInfo->bufPageSize = getProperSortPageSize(rowSize, nCols); From d8429c0ad9b6fd802247db78f0635197685a4ee5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 18 Nov 2022 18:24:17 +0800 Subject: [PATCH 60/95] fix memleak --- tests/parallel_test/cases.task | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 54877273cf..8a5ae3fc9f 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -136,8 +136,8 @@ ,,y,script,./test.sh -f tsim/parser/lastrow.sim ,,y,script,./test.sh -f tsim/parser/lastrow2.sim ,,y,script,./test.sh -f tsim/parser/like.sim -,,,script,./test.sh -f tsim/parser/limit.sim -,,,script,./test.sh -f tsim/parser/limit1.sim +,,y,script,./test.sh -f tsim/parser/limit.sim +,,y,script,./test.sh -f tsim/parser/limit1.sim ,,y,script,./test.sh -f tsim/parser/mixed_blocks.sim ,,y,script,./test.sh -f tsim/parser/nchar.sim ,,y,script,./test.sh -f tsim/parser/nestquery.sim @@ -163,7 +163,7 @@ ,,y,script,./test.sh -f tsim/parser/timestamp.sim ,,y,script,./test.sh -f tsim/parser/top_groupby.sim ,,y,script,./test.sh -f tsim/parser/topbot.sim -,,,script,./test.sh -f tsim/parser/union.sim +,,y,script,./test.sh -f tsim/parser/union.sim ,,y,script,./test.sh -f tsim/parser/union_sysinfo.sim ,,y,script,./test.sh -f tsim/parser/where.sim ,,y,script,./test.sh -f tsim/query/charScalarFunction.sim From 788a1031328895e87b0a819427109587cf523748 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 18 Nov 2022 18:42:08 +0800 Subject: [PATCH 61/95] refactor(sync): optimize leader transfer --- source/libs/sync/src/syncMain.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 39c37a3860..88d4475d0f 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -436,8 +436,15 @@ int32_t syncNodeLeaderTransfer(SSyncNode* pSyncNode) { } int32_t ret = 0; - if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { + if (pSyncNode->state == TAOS_SYNC_STATE_LEADER && pSyncNode->replicaNum > 1) { SNodeInfo newLeader = (pSyncNode->peersNodeInfo)[0]; + if (pSyncNode->peersNum == 2) { + SyncIndex matchIndex0 = syncIndexMgrGetIndex(pSyncNode->pMatchIndex, &(pSyncNode->peersId[0])); + SyncIndex matchIndex1 = syncIndexMgrGetIndex(pSyncNode->pMatchIndex, &(pSyncNode->peersId[1])); + if (matchIndex1 > matchIndex0) { + newLeader = (pSyncNode->peersNodeInfo)[1]; + } + } ret = syncNodeLeaderTransferTo(pSyncNode, newLeader); } From a12cb5e9cf8b4128123f47cf40d29735b06c30f5 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 18 Nov 2022 18:47:59 +0800 Subject: [PATCH 62/95] fix: remove --win_service from taosadapter of windows installer (#18269) * fix: sc create service in tdengine.iss * fix: upx on windows * fix: add sc start/stop to installer * fix: taosadapteer no --win_service --- packaging/tools/tdengine.iss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/tools/tdengine.iss b/packaging/tools/tdengine.iss index 1c0c105179..2e9c46d06b 100644 --- a/packaging/tools/tdengine.iss +++ b/packaging/tools/tdengine.iss @@ -63,7 +63,7 @@ Source: {#MyAppSourceDir}{#MyAppTaosdemoExeName}; DestDir: "{app}"; Flags: igNor [run] Filename: {sys}\sc.exe; Parameters: "create taosd start= DEMAND binPath= ""C:\\TDengine\\taosd.exe --win_service""" ; Flags: runhidden -Filename: {sys}\sc.exe; Parameters: "create taosadapter start= DEMAND binPath= ""C:\\TDengine\\taosadapter.exe --win_service""" ; Flags: runhidden +Filename: {sys}\sc.exe; Parameters: "create taosadapter start= DEMAND binPath= ""C:\\TDengine\\taosadapter.exe""" ; Flags: runhidden [UninstallRun] RunOnceId: "stoptaosd"; Filename: {sys}\sc.exe; Parameters: "stop taosd" ; Flags: runhidden From ff2a0db9f7284ffd229a7627107f2b0f69e86b4e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 18 Nov 2022 18:58:50 +0800 Subject: [PATCH 63/95] refactor(sync): fix asan error --- source/libs/sync/src/syncMain.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 88d4475d0f..a427d7aa0c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1403,7 +1403,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // reset sender bool reset = false; for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) { - if (syncUtilSameId(&(pSyncNode->replicasId)[i], &oldReplicasId[j])) { + if (syncUtilSameId(&(pSyncNode->replicasId)[i], &oldReplicasId[j]) && oldSenders[j] != NULL) { char host[128]; uint16_t port; syncUtilU642Addr((pSyncNode->replicasId)[i].addr, host, sizeof(host), &port); @@ -1420,6 +1420,8 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde sNTrace(pSyncNode, "snapshot sender udpate replicaIndex from %d to %d, %s:%d, %p, reset:%d", oldreplicaIndex, i, host, port, (pSyncNode->senders)[i], reset); + + break; } } } From 9c426540e7e47f7aedb0b0a062ffea27ecbd3446 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 18 Nov 2022 20:15:13 +0800 Subject: [PATCH 64/95] fix: memory leaks while check mnode state --- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 3 +++ source/dnode/mnode/impl/src/mndMain.c | 2 +- source/libs/transport/src/tmsgcb.c | 8 +++++++- source/libs/transport/src/transSvr.c | 14 +++++++++----- tests/parallel_test/cases.task | 14 +++++++------- tests/script/tsim/db/alter_replica_13.sim | 4 ++-- 6 files changed, 29 insertions(+), 16 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index e50b527bac..d3d92e1bbf 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -56,6 +56,9 @@ static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { if (IsReq(pMsg) && pMsg->info.handle != NULL && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && terrno != 0) code = terrno; mmSendRsp(pMsg, code); + } else { + rpcFreeCont(pMsg->info.rsp); + pMsg->info.rsp = NULL; } if (code == TSDB_CODE_RPC_REDIRECT) { diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index dcfc046a1e..b5ddd6f279 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -577,9 +577,9 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet); pMsg->info.rsp = rpcMallocCont(contLen); - pMsg->info.hasEpSet = 1; if (pMsg->info.rsp != NULL) { tSerializeSEpSet(pMsg->info.rsp, contLen, &epSet); + pMsg->info.hasEpSet = 1; pMsg->info.rspLen = contLen; terrno = TSDB_CODE_RPC_REDIRECT; } else { diff --git a/source/libs/transport/src/tmsgcb.c b/source/libs/transport/src/tmsgcb.c index 1cd1903851..0d1644be2d 100644 --- a/source/libs/transport/src/tmsgcb.c +++ b/source/libs/transport/src/tmsgcb.c @@ -44,7 +44,13 @@ int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) { return code; } -void tmsgSendRsp(SRpcMsg* pMsg) { return (*defaultMsgCb.sendRspFp)(pMsg); } +void tmsgSendRsp(SRpcMsg* pMsg) { +#if 1 + rpcSendResponse(pMsg); +#else + return (*defaultMsgCb.sendRspFp)(pMsg); +#endif +} void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet) { (*defaultMsgCb.sendRedirectRspFp)(pMsg, pNewEpSet); } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 5f36d91023..8dd3628c5f 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -462,8 +462,6 @@ static void uvStartSendResp(SSvrMsg* smsg) { if (pConn->broken == true) { // persist by destroySmsg(smsg); - // transFreeMsg(smsg->msg.pCont); - // taosMemoryFree(smsg); transUnrefSrvHandle(pConn); return; } @@ -1234,7 +1232,9 @@ int transReleaseSrvHandle(void* handle) { m->type = Release; tTrace("%s conn %p start to release", transLabel(pThrd->pTransInst), exh->handle); - transAsyncSend(pThrd->asyncPool, &m->q); + if (0 != transAsyncSend(pThrd->asyncPool, &m->q)) { + destroySmsg(m); + } transReleaseExHandle(transGetRefMgt(), refId); return 0; @@ -1269,7 +1269,9 @@ int transSendResponse(const STransMsg* msg) { STraceId* trace = (STraceId*)&msg->info.traceId; tGTrace("conn %p start to send resp (1/2)", exh->handle); - transAsyncSend(pThrd->asyncPool, &m->q); + if (0 != transAsyncSend(pThrd->asyncPool, &m->q)) { + destroySmsg(m); + } transReleaseExHandle(transGetRefMgt(), refId); return 0; @@ -1303,7 +1305,9 @@ int transRegisterMsg(const STransMsg* msg) { STrans* pTransInst = pThrd->pTransInst; tTrace("%s conn %p start to register brokenlink callback", transLabel(pTransInst), exh->handle); - transAsyncSend(pThrd->asyncPool, &m->q); + if (0 != transAsyncSend(pThrd->asyncPool, &m->q)) { + destroySmsg(m); + } transReleaseExHandle(transGetRefMgt(), refId); return 0; diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 713d3f4271..985f7cafbf 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -56,7 +56,7 @@ ,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim ,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim ,,y,script,./test.sh -f tsim/dnode/vnode_clean.sim -,,,script,./test.sh -f tsim/dnode/use_dropped_dnode.sim +,,y,script,./test.sh -f tsim/dnode/use_dropped_dnode.sim ,,y,script,./test.sh -f tsim/dnode/split_vgroup_replica1.sim ,,y,script,./test.sh -f tsim/dnode/split_vgroup_replica3.sim ,,y,script,./test.sh -f tsim/import/basic.sim @@ -176,11 +176,11 @@ ,,y,script,./test.sh -f tsim/query/udf.sim ,,y,script,./test.sh -f tsim/qnode/basic1.sim ,,y,script,./test.sh -f tsim/snode/basic1.sim -,,,script,./test.sh -f tsim/mnode/basic1.sim -,,,script,./test.sh -f tsim/mnode/basic2.sim -,,,script,./test.sh -f tsim/mnode/basic3.sim -,,,script,./test.sh -f tsim/mnode/basic4.sim -,,,script,./test.sh -f tsim/mnode/basic5.sim +,,y,script,./test.sh -f tsim/mnode/basic1.sim +,,y,script,./test.sh -f tsim/mnode/basic2.sim +,,y,script,./test.sh -f tsim/mnode/basic3.sim +,,y,script,./test.sh -f tsim/mnode/basic4.sim +,,y,script,./test.sh -f tsim/mnode/basic5.sim ,,y,script,./test.sh -f tsim/show/basic.sim ,,y,script,./test.sh -f tsim/table/autocreate.sim ,,y,script,./test.sh -f tsim/table/basic1.sim @@ -291,7 +291,7 @@ ,,n,script,./test.sh -f tsim/valgrind/checkError7.sim ,,n,script,./test.sh -f tsim/valgrind/checkError8.sim ,,n,script,./test.sh -f tsim/valgrind/checkUdf.sim -,,,script,./test.sh -f tsim/vnode/replica3_basic.sim +,,y,script,./test.sh -f tsim/vnode/replica3_basic.sim ,,y,script,./test.sh -f tsim/vnode/replica3_repeat.sim ,,y,script,./test.sh -f tsim/vnode/replica3_vgroup.sim ,,y,script,./test.sh -f tsim/vnode/replica3_many.sim diff --git a/tests/script/tsim/db/alter_replica_13.sim b/tests/script/tsim/db/alter_replica_13.sim index 007bb00f07..b3231cc24b 100644 --- a/tests/script/tsim/db/alter_replica_13.sim +++ b/tests/script/tsim/db/alter_replica_13.sim @@ -138,10 +138,10 @@ while $i < 10 if $data[0][4] != leader then return -1 endi - if $data[0][6] != follower then + if $data[0][6] == leader then return -1 endi - if $data[0][8] != follower then + if $data[0][8] == leader then return -1 endi endw From 596f7c1c11e3f14a6c97c9fda0d0319786148678 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 18 Nov 2022 20:19:12 +0800 Subject: [PATCH 65/95] test: remove asan case --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 985f7cafbf..63b9f592f8 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -278,7 +278,7 @@ ,,y,script,./test.sh -f tsim/stable/values.sim ,,y,script,./test.sh -f tsim/stable/vnode3.sim ,,y,script,./test.sh -f tsim/stable/metrics_idx.sim -,,,script,./test.sh -f tsim/sma/drop_sma.sim +,,n,script,./test.sh -f tsim/sma/drop_sma.sim ,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim From f24eb634efbdc0ca87122f578a0c74c352ffcf0e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 18 Nov 2022 22:38:50 +0800 Subject: [PATCH 66/95] fix(query): fix memory leak. --- source/libs/executor/src/scanoperator.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 5809a45ec5..3c34622e22 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -4685,6 +4685,10 @@ void destroyTableMergeScanOperatorInfo(void* param) { taosArrayDestroy(pTableScanInfo->pSortInfo); cleanupExprSupp(&pTableScanInfo->base.pseudoSup); + tsdbReaderClose(pTableScanInfo->base.dataReader); + pTableScanInfo->base.dataReader = NULL; + taosLRUCacheCleanup(pTableScanInfo->base.metaCache.pTableMetaEntryCache); + taosMemoryFreeClear(param); } From 5c60bf9a47ecc8cbb17f81b667316675d6dbca27 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 18 Nov 2022 23:16:33 +0800 Subject: [PATCH 67/95] fix(query): disable limit in table merge scan. --- source/libs/executor/src/scanoperator.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 3c34622e22..916b6df969 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -4746,7 +4746,9 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->base.dataBlockLoadFlag = FUNC_DATA_REQUIRED_DATA_LOAD; pInfo->base.scanFlag = MAIN_SCAN; pInfo->base.readHandle = *readHandle; - initLimitInfo(pTableScanNode->scan.node.pLimit, pTableScanNode->scan.node.pSlimit, &pInfo->base.limitInfo); + + pInfo->base.limitInfo.limit.limit = -1; + pInfo->base.limitInfo.slimit.limit = -1; pInfo->sample.sampleRatio = pTableScanNode->ratio; pInfo->sample.seed = taosGetTimestampSec(); From cdaafb7b8994dd12f5dec8e38670fa38a74cc0d3 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 18 Nov 2022 19:26:04 +0800 Subject: [PATCH 68/95] fix(meta): new error for creating sub tb already exist in other stables --- include/util/taoserror.h | 1 + source/dnode/vnode/src/meta/metaTable.c | 5 +++++ source/util/src/terror.c | 1 + 3 files changed, 7 insertions(+) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 636decc60b..e5d0bcb249 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -352,6 +352,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x061A) #define TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER TAOS_DEF_ERROR_CODE(0, 0x061B) #define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061C) +#define TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE TAOS_DEF_ERROR_CODE(0, 0x061D) // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 5921adfbfa..8250d68ae1 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -403,6 +403,11 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMe // validate req metaReaderInit(&mr, pMeta, 0); if (metaGetTableEntryByName(&mr, pReq->name) == 0) { + if (pReq->type == TSDB_CHILD_TABLE && pReq->ctb.suid != mr.me.ctbEntry.suid) { + terrno = TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE; + metaReaderClear(&mr); + return -1; + } pReq->uid = mr.me.uid; if (pReq->type == TSDB_CHILD_TABLE) { pReq->ctb.suid = mr.me.ctbEntry.suid; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 43602a607a..4d889843e8 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -344,6 +344,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_ALREADY_EXIST, "Stable already exists TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_NOT_EXIST, "Stable not exists") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER, "Table schema is old") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR, "TDB env open error") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE, "Table already exists in other stables") // query TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, "Invalid handle") From 77e3cc42b93407dd223a4537d0c214a6040be2ba Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sat, 19 Nov 2022 10:05:24 +0800 Subject: [PATCH 69/95] fix: change example name in document --- docs/en/07-develop/09-udf.md | 6 +++--- docs/en/12-taos-sql/26-udf.md | 4 ++-- docs/zh/07-develop/09-udf.md | 6 +++--- docs/zh/12-taos-sql/26-udf.md | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md index 253f1270f5..699b3ebe5f 100644 --- a/docs/en/07-develop/09-udf.md +++ b/docs/en/07-develop/09-udf.md @@ -205,13 +205,13 @@ Additional functions are defined in `taosudf.h` to make it easier to work with t To use your user-defined function in TDengine, first compile it to a dynamically linked library (DLL). -For example, the sample UDF `add_one.c` can be compiled into a DLL as follows: +For example, the sample UDF `bit_and.c` can be compiled into a DLL as follows: ```bash -gcc -g -O0 -fPIC -shared add_one.c -o add_one.so +gcc -g -O0 -fPIC -shared bit_and.c -o libbitand.so ``` -The generated DLL file `add_one.so` can now be used to implement your function. Note: GCC 7.5 or later is required. +The generated DLL file `libbitand.so` can now be used to implement your function. Note: GCC 7.5 or later is required. ## Manage and Use User-Defined Functions After compiling your function into a DLL, you add it to TDengine. For more information, see [User-Defined Functions](../12-taos-sql/26-udf.md). diff --git a/docs/en/12-taos-sql/26-udf.md b/docs/en/12-taos-sql/26-udf.md index e6199e8b31..977f3bcc08 100644 --- a/docs/en/12-taos-sql/26-udf.md +++ b/docs/en/12-taos-sql/26-udf.md @@ -62,7 +62,7 @@ SHOW FUNCTIONS; The function name specified when creating UDF can be used directly in SQL statements, just like builtin functions. For example: ```sql -SELECT X(c1,c2) FROM table/stable; +SELECT bit_and(c1,c2) FROM table; ``` -The above SQL statement invokes function X for column c1 and c2. You can use query keywords like WHERE with user-defined functions. +The above SQL statement invokes function X for column c1 and c2 on table. You can use query keywords like WHERE with user-defined functions. diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index 3239eae49b..7a3a3822d5 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -205,13 +205,13 @@ typedef struct SUdfInterBuf { 用户定义函数的 C 语言源代码无法直接被 TDengine 系统使用，而是需要先编译为 动态链接库，之后才能载入 TDengine 系统。 -例如，按照上一章节描述的规则准备好了用户定义函数的源代码 add_one.c，以 Linux 为例可以执行如下指令编译得到动态链接库文件： +例如，按照上一章节描述的规则准备好了用户定义函数的源代码 bit_and.c，以 Linux 为例可以执行如下指令编译得到动态链接库文件： ```bash -gcc -g -O0 -fPIC -shared add_one.c -o add_one.so +gcc -g -O0 -fPIC -shared bit_and.c -o libbitand.so ``` -这样就准备好了动态链接库 add_one.so 文件，可以供后文创建 UDF 时使用了。为了保证可靠的系统运行，编译器 GCC 推荐使用 7.5 及以上版本。 +这样就准备好了动态链接库 libbitand.so 文件，可以供后文创建 UDF 时使用了。为了保证可靠的系统运行，编译器 GCC 推荐使用 7.5 及以上版本。 ## 管理和使用UDF 编译好的UDF，还需要将其加入到系统才能被正常的SQL调用。关于如何管理和使用UDF，参见[UDF使用说明](../12-taos-sql/26-udf.md) diff --git a/docs/zh/12-taos-sql/26-udf.md b/docs/zh/12-taos-sql/26-udf.md index 6dc1b6eb5f..cb349109a7 100644 --- a/docs/zh/12-taos-sql/26-udf.md +++ b/docs/zh/12-taos-sql/26-udf.md @@ -63,7 +63,7 @@ SHOW FUNCTIONS; 在 SQL 指令中，可以直接以在系统中创建 UDF 时赋予的函数名来调用用户定义函数。例如： ```sql -SELECT X(c1,c2) FROM table/stable; +SELECT bit_and(c1,c2) FROM table; ``` -表示对名为 c1, c2 的数据列调用名为 X 的用户定义函数。SQL 指令中用户定义函数可以配合 WHERE 等查询特性来使用。 +表示对表 table 上名为 c1, c2 的数据列调用名为 bit_and 的用户定义函数。SQL 指令中用户定义函数可以配合 WHERE 等查询特性来使用。 From aa4b2aaa745f0eb87934f0475b32b8e6fdf4f9fa Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Sat, 19 Nov 2022 19:31:04 +0800 Subject: [PATCH 70/95] fix: compile taosws on loongarch Close [TD-20176](https://jira.taosdata.com:18080/browse/TD-20176) --- cmake/taosws_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index 7d48eb9d8a..e79c6b799f 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git - GIT_TAG 9843872 + GIT_TAG f406d51 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 0464a21d0e5402ee3fddd4e580bd5fabf827e437 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 19 Nov 2022 10:54:38 +0800 Subject: [PATCH 71/95] test: adjust pytest frame --- tests/pytest/util/dnodes.py | 68 ++++++++++++++++++++-- tests/pytest/util/log.py | 6 +- tests/script/sh/sigint_stop_dnodes.sh | 16 ++++++ tests/script/sh/stop_dnodes.sh | 17 ++---- tests/script/test.sh | 3 - tests/system-test/test.py | 11 +++- tests/system-test/test.sh | 83 +++++++++++++++++++++++++++ 7 files changed, 180 insertions(+), 24 deletions(-) create mode 100755 tests/script/sh/sigint_stop_dnodes.sh create mode 100755 tests/system-test/test.sh diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index b762f8c77f..0e8ec148b4 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -116,6 +116,7 @@ class TDDnode: self.deployed = 0 self.testCluster = False self.valgrind = 0 + self.asan = False self.remoteIP = "" self.cfgDict = { "monitor": "0", @@ -158,6 +159,11 @@ class TDDnode: def setValgrind(self, value): self.valgrind = value + def setAsan(self, value): + self.asan = value + if value: + self.execPath = os.path.abspath(self.path + "/tests/script/sh/exec.sh") + def getDataSize(self): totalSize = 0 @@ -383,8 +389,14 @@ class TDDnode: cmd = "mintty -h never %s -c %s" % ( binPath, self.cfgDir) else: - cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( - binPath, self.cfgDir) + if self.asan: + asanDir = "%s/sim/tsim/asan/dnode%d.asan" % ( + self.path, self.index) + cmd = "nohup %s -c %s > /dev/null 2> %s & " % ( + binPath, self.cfgDir, asanDir) + else: + cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( + binPath, self.cfgDir) else: valgrindCmdline = "valgrind --log-file=\"%s/../log/valgrind.log\" --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"%self.cfgDir @@ -444,8 +456,14 @@ class TDDnode: tdLog.exit("dnode:%d is not deployed" % (self.index)) if self.valgrind == 0: - cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( - binPath, self.cfgDir) + if self.asan: + asanDir = "%s/sim/tsim/asan/dnode%d.asan" % ( + self.path, self.index) + cmd = "nohup %s -c %s > /dev/null 2> %s & " % ( + binPath, self.cfgDir, asanDir) + else: + cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( + binPath, self.cfgDir) else: valgrindCmdline = "valgrind --log-file=\"%s/../log/valgrind.log\" --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"%self.cfgDir @@ -464,6 +482,12 @@ class TDDnode: tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) def stop(self): + if self.asan: + stopCmd = "%s -s stop -n dnode%d" % (self.execPath, self.index) + tdLog.info("execute script: " + stopCmd) + os.system(stopCmd) + return + if (not self.remoteIP == ""): self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].running=1\ntdDnodes.dnodes[%d].stop()"%(self.index-1,self.index-1)) tdLog.info("stop dnode%d"%self.index) @@ -501,6 +525,12 @@ class TDDnode: def stoptaosd(self): + if self.asan: + stopCmd = "%s -s stop -n dnode%d" % (self.execPath, self.index) + tdLog.info("execute script: " + stopCmd) + os.system(stopCmd) + return + if (not self.remoteIP == ""): self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].running=1\ntdDnodes.dnodes[%d].stop()"%(self.index-1,self.index-1)) tdLog.info("stop dnode%d"%self.index) @@ -534,6 +564,13 @@ class TDDnode: tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index)) def forcestop(self): + if self.asan: + stopCmd = "%s -s stop -n dnode%d -x SIGKILL" + \ + (self.execPath, self.index) + tdLog.info("execute script: " + stopCmd) + os.system(stopCmd) + return + if (not self.remoteIP == ""): self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].running=1\ntdDnodes.dnodes[%d].forcestop()"%(self.index-1,self.index-1)) return @@ -606,6 +643,7 @@ class TDDnodes: self.simDeployed = False self.testCluster = False self.valgrind = 0 + self.asan = False self.killValgrind = 1 def init(self, path, remoteIP = ""): @@ -629,6 +667,13 @@ class TDDnodes: def setValgrind(self, value): self.valgrind = value + def setAsan(self, value): + self.asan = value + if value: + self.stopDnodesPath = os.path.abspath(self.path + "/tests/script/sh/stop_dnodes.sh") + self.stopDnodesSigintPath = os.path.abspath(self.path + "/tests/script/sh/sigint_stop_dnodes.sh") + tdLog.info("run in address sanitizer mode") + def setKillValgrind(self, value): self.killValgrind = value @@ -642,6 +687,7 @@ class TDDnodes: self.check(index) self.dnodes[index - 1].setTestCluster(self.testCluster) self.dnodes[index - 1].setValgrind(self.valgrind) + self.dnodes[index - 1].setAsan(self.asan) self.dnodes[index - 1].deploy(updatecfgDict) def cfg(self, index, option, value): @@ -692,8 +738,22 @@ class TDDnodes: if index < 1 or index > 10: tdLog.exit("index:%d should on a scale of [1, 10]" % (index)) + def StopAllSigint(self): + tdLog.info("stop all dnodes sigint") + if self.asan: + tdLog.info("execute script: %s" % self.stopDnodesSigintPath) + os.system(self.stopDnodesSigintPath) + tdLog.info("execute finished") + return + def stopAll(self): tdLog.info("stop all dnodes") + if self.asan: + tdLog.info("execute script: %s" % self.stopDnodesPath) + os.system(self.stopDnodesPath) + tdLog.info("execute finished") + return + if (not self.dnodes[0].remoteIP == ""): self.dnodes[0].remoteExec(self.dnodes[0].cfgDict, "for i in range(len(tdDnodes.dnodes)):\n tdDnodes.dnodes[i].running=1\ntdDnodes.stopAll()") return diff --git a/tests/pytest/util/log.py b/tests/pytest/util/log.py index 55cd42a6eb..a132178308 100644 --- a/tests/pytest/util/log.py +++ b/tests/pytest/util/log.py @@ -33,13 +33,13 @@ class TDLog: print("\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err)) def success(self, info): - printf("\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info)) + print("\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info)) def notice(self, err): - printf("\033[1;33m%s %s\033[0m" % (datetime.datetime.now(), err)) + print("\033[1;33m%s %s\033[0m" % (datetime.datetime.now(), err)) def exit(self, err): - printf("\033[1;31m%s %s\033[0m" % (datetime.datetime.now(), err)) + print("\033[1;31m%s %s\033[0m" % (datetime.datetime.now(), err)) sys.exit(1) def printNoPrefix(self, info): diff --git a/tests/script/sh/sigint_stop_dnodes.sh b/tests/script/sh/sigint_stop_dnodes.sh new file mode 100755 index 0000000000..12b58225de --- /dev/null +++ b/tests/script/sh/sigint_stop_dnodes.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +set +e +#set -x + +export LD_PRELOAD= +UNAME_BIN=`which uname` +OS_TYPE=`$UNAME_BIN` + +PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` +while [ -n "$PID" ]; do + #echo "Killing taosd processes " $PID + kill $PID + + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` +done diff --git a/tests/script/sh/stop_dnodes.sh b/tests/script/sh/stop_dnodes.sh index d30c75022a..38667d9b3f 100755 --- a/tests/script/sh/stop_dnodes.sh +++ b/tests/script/sh/stop_dnodes.sh @@ -1,8 +1,12 @@ #!/bin/sh +set +e +#set -x + UNAME_BIN=`which uname` OS_TYPE=`$UNAME_BIN` +export LD_PRELOAD= PID=`ps -ef|grep /usr/bin/taosd | grep -v grep | awk '{print $2}'` if [ -n "$PID" ]; then echo systemctl stop taosd @@ -22,16 +26,3 @@ while [ -n "$PID" ]; do fi PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` done - -PID=`ps -ef|grep -w tarbitrator | grep -v grep | awk '{print $2}'` -while [ -n "$PID" ]; do - echo kill -9 $PID - pkill -9 tarbitrator - if [ "$OS_TYPE" != "Darwin" ]; then - fuser -k -n tcp 6040 - else - lsof -nti:6040 | xargs kill -9 - fi - PID=`ps -ef|grep -w tarbitrator | grep -v grep | awk '{print $2}'` -done - diff --git a/tests/script/test.sh b/tests/script/test.sh index e1db1c14de..f2eff825e6 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -66,11 +66,8 @@ else fi declare -x BUILD_DIR=$TOP_DIR/$BIN_DIR - declare -x SIM_DIR=$TOP_DIR/sim - PROGRAM=$BUILD_DIR/build/bin/tsim - PRG_DIR=$SIM_DIR/tsim CFG_DIR=$PRG_DIR/cfg LOG_DIR=$PRG_DIR/log diff --git a/tests/system-test/test.py b/tests/system-test/test.py index d942147c03..712b9689ba 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -73,8 +73,9 @@ if __name__ == "__main__": createDnodeNums = 1 restful = False replicaVar = 1 + asan = False independentMnode = True - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RD:n:i:', [ + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RD:n:i:a', [ 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','adaptercfgupdate','replicaVar','independentMnode']) for key, value in opts: if key in ['-h', '--help']: @@ -99,6 +100,7 @@ if __name__ == "__main__": tdLog.printNoPrefix('-D taosadapter update cfg dict ') tdLog.printNoPrefix('-n the number of replicas') tdLog.printNoPrefix('-i independentMnode Mnode') + tdLog.printNoPrefix('-a address sanitizer mode') sys.exit(0) @@ -167,6 +169,9 @@ if __name__ == "__main__": if key in ['-R', '--restful']: restful = True + if key in ['-a', '--asan']: + asan = True + if key in ['-D', '--adaptercfgupdate']: try: adaptercfgupdate = eval(base64.b64decode(value.encode()).decode()) @@ -387,6 +392,7 @@ if __name__ == "__main__": tdDnodes.init(deployPath, masterIp) tdDnodes.setTestCluster(testCluster) tdDnodes.setValgrind(valgrind) + tdDnodes.setAsan(asan) tdDnodes.stopAll() is_test_framework = 0 key_word = 'tdCases.addLinux' @@ -547,4 +553,7 @@ if __name__ == "__main__": if conn is not None: conn.close() + if asan: + tdDnodes.StopAllSigint() + tdLog.info("address sanitizer mode finished") sys.exit(0) diff --git a/tests/system-test/test.sh b/tests/system-test/test.sh new file mode 100755 index 0000000000..39eccc965b --- /dev/null +++ b/tests/system-test/test.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +################################################## +# +# Do simulation test +# +################################################## + +set +e +#set -x + +UNAME_BIN=`which uname` +OS_TYPE=`$UNAME_BIN` + +cd . + +# Get responsible directories +CODE_DIR=`dirname $0` +CODE_DIR=`pwd` + +IN_TDINTERNAL="community" +if [[ "$CODE_DIR" == *"$IN_TDINTERNAL"* ]]; then + cd ../../.. +else + cd ../../ +fi + +TOP_DIR=`pwd` +TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` + +if [[ "$OS_TYPE" != "Darwin" ]]; then + cut_opt="--field=" +else + cut_opt="-f " +fi + +if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` +else + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2` +fi + +declare -x BUILD_DIR=$TOP_DIR/$BIN_DIR +declare -x SIM_DIR=$TOP_DIR/sim +PROGRAM=$BUILD_DIR/build/bin/tsim +PRG_DIR=$SIM_DIR/tsim +ASAN_DIR=$PRG_DIR/asan +SYSTEM_TEST_DIR=$TOP_DIR/tests/system-test + +chmod -R 777 $PRG_DIR +echo "------------------------------------------------------------------------" +echo "Start TDengine Testing Case ..." +echo "BUILD_DIR: $BUILD_DIR" +echo "SYSTEM_TEST_DIR : $SYSTEM_TEST_DIR" +echo "SIM_DIR : $SIM_DIR" +echo "CODE_DIR : $CODE_DIR" +echo "ASAN_DIR : $ASAN_DIR" + +rm -rf $SIM_DIR/* + +mkdir -p $PRG_DIR +mkdir -p $ASAN_DIR + +cd $SYSTEM_TEST_DIR +ulimit -n 600000 +ulimit -c unlimited + +#sudo sysctl -w kernel.core_pattern=$TOP_DIR/core.%p.%e + +echo "ExcuteCmd:" $* +echo "AsanDir:" $ASAN_DIR/psim.asan + +export LD_PRELOAD=libasan.so.5 +$* -a 2> $ASAN_DIR/psim.asan +result=$? +echo "Execute result:" $result + +if [ $result -eq 0 ]; then + $CODE_DIR/sh/checkAsan.sh +else + exit 1 +fi + From 3e29f1978270034973b303ec43bbb6df28000c01 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Fri, 18 Nov 2022 17:23:07 +0800 Subject: [PATCH 72/95] fix:stream state abnormal failure --- include/libs/stream/streamState.h | 11 ++- source/libs/stream/src/streamState.c | 142 +++++++++++++++------------ 2 files changed, 87 insertions(+), 66 deletions(-) diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 7234d306a4..59f030a60c 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -27,8 +27,7 @@ typedef struct SStreamTask SStreamTask; typedef bool (*state_key_cmpr_fn)(void* pKey1, void* pKey2); -// incremental state storage -typedef struct { +typedef struct STdbState { SStreamTask* pOwner; TDB* db; TTB* pStateDb; @@ -37,7 +36,12 @@ typedef struct { TTB* pSessionStateDb; TTB* pParNameDb; TXN txn; - int32_t number; +} STdbState; + +// incremental state storage +typedef struct { + STdbState* pTdbState; + int32_t number; } SStreamState; SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages); @@ -45,6 +49,7 @@ void streamStateClose(SStreamState* pState); int32_t streamStateBegin(SStreamState* pState); int32_t streamStateCommit(SStreamState* pState); int32_t streamStateAbort(SStreamState* pState); +void streamStateDestroy(SStreamState* pState); typedef struct { TBC* pCur; diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 63ec0caa95..0374e22a4a 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -114,6 +114,12 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + pState->pTdbState = taosMemoryCalloc(1, sizeof(STdbState)); + if (pState->pTdbState == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + streamStateDestroy(pState); + return NULL; + } char statePath[1024]; if (!specPath) { @@ -122,31 +128,34 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int memset(statePath, 0, 1024); tstrncpy(statePath, path, 1024); } - if (tdbOpen(statePath, szPage, pages, &pState->db, 0) < 0) { + if (tdbOpen(statePath, szPage, pages, &pState->pTdbState->db, 0) < 0) { goto _err; } // open state storage backend - if (tdbTbOpen("state.db", sizeof(SStateKey), -1, stateKeyCmpr, pState->db, &pState->pStateDb, 0) < 0) { + if (tdbTbOpen("state.db", sizeof(SStateKey), -1, stateKeyCmpr, pState->pTdbState->db, &pState->pTdbState->pStateDb, + 0) < 0) { goto _err; } // todo refactor - if (tdbTbOpen("fill.state.db", sizeof(SWinKey), -1, winKeyCmpr, pState->db, &pState->pFillStateDb, 0) < 0) { + if (tdbTbOpen("fill.state.db", sizeof(SWinKey), -1, winKeyCmpr, pState->pTdbState->db, + &pState->pTdbState->pFillStateDb, 0) < 0) { goto _err; } - if (tdbTbOpen("session.state.db", sizeof(SStateSessionKey), -1, stateSessionKeyCmpr, pState->db, - &pState->pSessionStateDb, 0) < 0) { + if (tdbTbOpen("session.state.db", sizeof(SStateSessionKey), -1, stateSessionKeyCmpr, pState->pTdbState->db, + &pState->pTdbState->pSessionStateDb, 0) < 0) { goto _err; } - if (tdbTbOpen("func.state.db", sizeof(STupleKey), -1, STupleKeyCmpr, pState->db, &pState->pFuncStateDb, 0) < 0) { + if (tdbTbOpen("func.state.db", sizeof(STupleKey), -1, STupleKeyCmpr, pState->pTdbState->db, + &pState->pTdbState->pFuncStateDb, 0) < 0) { goto _err; } - if (tdbTbOpen("parname.state.db", sizeof(int64_t), TSDB_TABLE_NAME_LEN, NULL, pState->db, &pState->pParNameDb, 0) < - 0) { + if (tdbTbOpen("parname.state.db", sizeof(int64_t), TSDB_TABLE_NAME_LEN, NULL, pState->pTdbState->db, + &pState->pTdbState->pParNameDb, 0) < 0) { goto _err; } @@ -154,117 +163,117 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int goto _err; } - pState->pOwner = pTask; + pState->pTdbState->pOwner = pTask; return pState; _err: - tdbTbClose(pState->pStateDb); - tdbTbClose(pState->pFuncStateDb); - tdbTbClose(pState->pFillStateDb); - tdbTbClose(pState->pSessionStateDb); - tdbTbClose(pState->pParNameDb); - tdbClose(pState->db); - taosMemoryFree(pState); + tdbTbClose(pState->pTdbState->pStateDb); + tdbTbClose(pState->pTdbState->pFuncStateDb); + tdbTbClose(pState->pTdbState->pFillStateDb); + tdbTbClose(pState->pTdbState->pSessionStateDb); + tdbTbClose(pState->pTdbState->pParNameDb); + tdbClose(pState->pTdbState->db); + streamStateDestroy(pState); return NULL; } void streamStateClose(SStreamState* pState) { - tdbCommit(pState->db, &pState->txn); - tdbPostCommit(pState->db, &pState->txn); - tdbTbClose(pState->pStateDb); - tdbTbClose(pState->pFuncStateDb); - tdbTbClose(pState->pFillStateDb); - tdbTbClose(pState->pSessionStateDb); - tdbTbClose(pState->pParNameDb); - tdbClose(pState->db); + tdbCommit(pState->pTdbState->db, &pState->pTdbState->txn); + tdbPostCommit(pState->pTdbState->db, &pState->pTdbState->txn); + tdbTbClose(pState->pTdbState->pStateDb); + tdbTbClose(pState->pTdbState->pFuncStateDb); + tdbTbClose(pState->pTdbState->pFillStateDb); + tdbTbClose(pState->pTdbState->pSessionStateDb); + tdbTbClose(pState->pTdbState->pParNameDb); + tdbClose(pState->pTdbState->db); - taosMemoryFree(pState); + streamStateDestroy(pState); } int32_t streamStateBegin(SStreamState* pState) { - if (tdbTxnOpen(&pState->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < - 0) { + if (tdbTxnOpen(&pState->pTdbState->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, + TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { return -1; } - if (tdbBegin(pState->db, &pState->txn) < 0) { - tdbTxnClose(&pState->txn); + if (tdbBegin(pState->pTdbState->db, &pState->pTdbState->txn) < 0) { + tdbTxnClose(&pState->pTdbState->txn); return -1; } return 0; } int32_t streamStateCommit(SStreamState* pState) { - if (tdbCommit(pState->db, &pState->txn) < 0) { + if (tdbCommit(pState->pTdbState->db, &pState->pTdbState->txn) < 0) { return -1; } - if (tdbPostCommit(pState->db, &pState->txn) < 0) { + if (tdbPostCommit(pState->pTdbState->db, &pState->pTdbState->txn) < 0) { return -1; } - memset(&pState->txn, 0, sizeof(TXN)); - if (tdbTxnOpen(&pState->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < - 0) { + memset(&pState->pTdbState->txn, 0, sizeof(TXN)); + if (tdbTxnOpen(&pState->pTdbState->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, + TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { return -1; } - if (tdbBegin(pState->db, &pState->txn) < 0) { + if (tdbBegin(pState->pTdbState->db, &pState->pTdbState->txn) < 0) { return -1; } return 0; } int32_t streamStateAbort(SStreamState* pState) { - if (tdbAbort(pState->db, &pState->txn) < 0) { + if (tdbAbort(pState->pTdbState->db, &pState->pTdbState->txn) < 0) { return -1; } - memset(&pState->txn, 0, sizeof(TXN)); - if (tdbTxnOpen(&pState->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < - 0) { + memset(&pState->pTdbState->txn, 0, sizeof(TXN)); + if (tdbTxnOpen(&pState->pTdbState->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, + TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { return -1; } - if (tdbBegin(pState->db, &pState->txn) < 0) { + if (tdbBegin(pState->pTdbState->db, &pState->pTdbState->txn) < 0) { return -1; } return 0; } int32_t streamStateFuncPut(SStreamState* pState, const STupleKey* key, const void* value, int32_t vLen) { - return tdbTbUpsert(pState->pFuncStateDb, key, sizeof(STupleKey), value, vLen, &pState->txn); + return tdbTbUpsert(pState->pTdbState->pFuncStateDb, key, sizeof(STupleKey), value, vLen, &pState->pTdbState->txn); } int32_t streamStateFuncGet(SStreamState* pState, const STupleKey* key, void** pVal, int32_t* pVLen) { - return tdbTbGet(pState->pFuncStateDb, key, sizeof(STupleKey), pVal, pVLen); + return tdbTbGet(pState->pTdbState->pFuncStateDb, key, sizeof(STupleKey), pVal, pVLen); } int32_t streamStateFuncDel(SStreamState* pState, const STupleKey* key) { - return tdbTbDelete(pState->pFuncStateDb, key, sizeof(STupleKey), &pState->txn); + return tdbTbDelete(pState->pTdbState->pFuncStateDb, key, sizeof(STupleKey), &pState->pTdbState->txn); } // todo refactor int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) { SStateKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbUpsert(pState->pStateDb, &sKey, sizeof(SStateKey), value, vLen, &pState->txn); + return tdbTbUpsert(pState->pTdbState->pStateDb, &sKey, sizeof(SStateKey), value, vLen, &pState->pTdbState->txn); } // todo refactor int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) { - return tdbTbUpsert(pState->pFillStateDb, key, sizeof(SWinKey), value, vLen, &pState->txn); + return tdbTbUpsert(pState->pTdbState->pFillStateDb, key, sizeof(SWinKey), value, vLen, &pState->pTdbState->txn); } // todo refactor int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { SStateKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbGet(pState->pStateDb, &sKey, sizeof(SStateKey), pVal, pVLen); + return tdbTbGet(pState->pTdbState->pStateDb, &sKey, sizeof(SStateKey), pVal, pVLen); } // todo refactor int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { - return tdbTbGet(pState->pFillStateDb, key, sizeof(SWinKey), pVal, pVLen); + return tdbTbGet(pState->pTdbState->pFillStateDb, key, sizeof(SWinKey), pVal, pVLen); } // todo refactor int32_t streamStateDel(SStreamState* pState, const SWinKey* key) { SStateKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbDelete(pState->pStateDb, &sKey, sizeof(SStateKey), &pState->txn); + return tdbTbDelete(pState->pTdbState->pStateDb, &sKey, sizeof(SStateKey), &pState->pTdbState->txn); } int32_t streamStateClear(SStreamState* pState) { @@ -288,7 +297,7 @@ void streamStateSetNumber(SStreamState* pState, int32_t number) { pState->number // todo refactor int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key) { - return tdbTbDelete(pState->pFillStateDb, key, sizeof(SWinKey), &pState->txn); + return tdbTbDelete(pState->pTdbState->pFillStateDb, key, sizeof(SWinKey), &pState->pTdbState->txn); } int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { @@ -314,7 +323,7 @@ int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pV SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key) { SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) return NULL; - tdbTbcOpen(pState->pStateDb, &pCur->pCur, NULL); + tdbTbcOpen(pState->pTdbState->pStateDb, &pCur->pCur, NULL); int32_t c = 0; SStateKey sKey = {.key = *key, .opNum = pState->number}; @@ -330,7 +339,7 @@ SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key) { SStreamStateCur* streamStateFillGetCur(SStreamState* pState, const SWinKey* key) { SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) return NULL; - tdbTbcOpen(pState->pFillStateDb, &pCur->pCur, NULL); + tdbTbcOpen(pState->pTdbState->pFillStateDb, &pCur->pCur, NULL); int32_t c = 0; tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c); @@ -422,7 +431,7 @@ SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key return NULL; } pCur->number = pState->number; - if (tdbTbcOpen(pState->pStateDb, &pCur->pCur, NULL) < 0) { + if (tdbTbcOpen(pState->pTdbState->pStateDb, &pCur->pCur, NULL) < 0) { streamStateFreeCur(pCur); return NULL; } @@ -448,7 +457,7 @@ SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* if (!pCur) { return NULL; } - if (tdbTbcOpen(pState->pFillStateDb, &pCur->pCur, NULL) < 0) { + if (tdbTbcOpen(pState->pTdbState->pFillStateDb, &pCur->pCur, NULL) < 0) { streamStateFreeCur(pCur); return NULL; } @@ -473,7 +482,7 @@ SStreamStateCur* streamStateFillSeekKeyPrev(SStreamState* pState, const SWinKey* if (pCur == NULL) { return NULL; } - if (tdbTbcOpen(pState->pFillStateDb, &pCur->pCur, NULL) < 0) { + if (tdbTbcOpen(pState->pTdbState->pFillStateDb, &pCur->pCur, NULL) < 0) { streamStateFreeCur(pCur); return NULL; } @@ -520,7 +529,8 @@ void streamFreeVal(void* val) { tdbFree(val); } int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen) { SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbUpsert(pState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), value, vLen, &pState->txn); + return tdbTbUpsert(pState->pTdbState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), value, vLen, + &pState->pTdbState->txn); } int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen) { @@ -543,7 +553,7 @@ int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVa int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key) { SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbDelete(pState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), &pState->txn); + return tdbTbDelete(pState->pTdbState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), &pState->pTdbState->txn); } SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key) { @@ -552,7 +562,7 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, cons return NULL; } pCur->number = pState->number; - if (tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL) < 0) { + if (tdbTbcOpen(pState->pTdbState->pSessionStateDb, &pCur->pCur, NULL) < 0) { streamStateFreeCur(pCur); return NULL; } @@ -579,7 +589,7 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, cons return NULL; } pCur->number = pState->number; - if (tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL) < 0) { + if (tdbTbcOpen(pState->pTdbState->pSessionStateDb, &pCur->pCur, NULL) < 0) { streamStateFreeCur(pCur); return NULL; } @@ -607,7 +617,7 @@ SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSess return NULL; } pCur->number = pState->number; - if (tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL) < 0) { + if (tdbTbcOpen(pState->pTdbState->pSessionStateDb, &pCur->pCur, NULL) < 0) { streamStateFreeCur(pCur); return NULL; } @@ -674,7 +684,7 @@ int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey* return -1; } pCur->number = pState->number; - if (tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL) < 0) { + if (tdbTbcOpen(pState->pTdbState->pSessionStateDb, &pCur->pCur, NULL) < 0) { streamStateFreeCur(pCur); return -1; } @@ -821,13 +831,19 @@ _end: } int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]) { - tdbTbUpsert(pState->pParNameDb, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN, &pState->txn); + tdbTbUpsert(pState->pTdbState->pParNameDb, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN, + &pState->pTdbState->txn); return 0; } int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal) { int32_t len; - return tdbTbGet(pState->pParNameDb, &groupId, sizeof(int64_t), pVal, &len); + return tdbTbGet(pState->pTdbState->pParNameDb, &groupId, sizeof(int64_t), pVal, &len); +} + +void streamStateDestroy(SStreamState* pState) { + taosMemoryFreeClear(pState->pTdbState); + taosMemoryFreeClear(pState); } #if 0 @@ -837,7 +853,7 @@ char* streamStateSessionDump(SStreamState* pState) { return NULL; } pCur->number = pState->number; - if (tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL) < 0) { + if (tdbTbcOpen(pState->pTdbState->pSessionStateDb, &pCur->pCur, NULL) < 0) { streamStateFreeCur(pCur); return NULL; } From 5831efd379de5b54cc60cada0d7248018d0ebadd Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 20 Nov 2022 12:23:59 +0800 Subject: [PATCH 73/95] test: adjust asan dir --- tests/pytest/util/dnodes.py | 4 ++-- tests/script/sh/checkAsan.sh | 2 +- tests/script/sh/exec.sh | 2 +- tests/script/test.sh | 2 +- tests/system-test/test.sh | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 0e8ec148b4..edb3d761cb 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -390,7 +390,7 @@ class TDDnode: binPath, self.cfgDir) else: if self.asan: - asanDir = "%s/sim/tsim/asan/dnode%d.asan" % ( + asanDir = "%s/sim/asan/dnode%d.asan" % ( self.path, self.index) cmd = "nohup %s -c %s > /dev/null 2> %s & " % ( binPath, self.cfgDir, asanDir) @@ -457,7 +457,7 @@ class TDDnode: if self.valgrind == 0: if self.asan: - asanDir = "%s/sim/tsim/asan/dnode%d.asan" % ( + asanDir = "%s/sim/asan/dnode%d.asan" % ( self.path, self.index) cmd = "nohup %s -c %s > /dev/null 2> %s & " % ( binPath, self.cfgDir, asanDir) diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 184dc9a88f..0ce74a989e 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -15,7 +15,7 @@ else fi TAOS_DIR=`pwd` -LOG_DIR=$TAOS_DIR/sim/tsim/asan +LOG_DIR=$TAOS_DIR/sim/asan error_num=`cat ${LOG_DIR}/*.asan | grep "ERROR" | wc -l` memory_leak=`cat ${LOG_DIR}/*.asan | grep "Direct leak" | wc -l` diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index 3f2c5d268c..5ef4cca741 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -80,7 +80,7 @@ LOG_DIR=$NODE_DIR/log DATA_DIR=$NODE_DIR/data MGMT_DIR=$NODE_DIR/data/mgmt TSDB_DIR=$NODE_DIR/data/tsdb -ASAN_DIR=$SIM_DIR/tsim/asan +ASAN_DIR=$SIM_DIR/asan TAOS_CFG=$NODE_DIR/cfg/taos.cfg echo ------------ $EXEC_OPTON $NODE_NAME diff --git a/tests/script/test.sh b/tests/script/test.sh index f2eff825e6..9385c73b70 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -72,7 +72,7 @@ PRG_DIR=$SIM_DIR/tsim CFG_DIR=$PRG_DIR/cfg LOG_DIR=$PRG_DIR/log DATA_DIR=$PRG_DIR/data -ASAN_DIR=$PRG_DIR/asan +ASAN_DIR=$SIM_DIR/asan chmod -R 777 $PRG_DIR echo "------------------------------------------------------------------------" diff --git a/tests/system-test/test.sh b/tests/system-test/test.sh index 39eccc965b..2a3187e641 100755 --- a/tests/system-test/test.sh +++ b/tests/system-test/test.sh @@ -44,7 +44,7 @@ declare -x BUILD_DIR=$TOP_DIR/$BIN_DIR declare -x SIM_DIR=$TOP_DIR/sim PROGRAM=$BUILD_DIR/build/bin/tsim PRG_DIR=$SIM_DIR/tsim -ASAN_DIR=$PRG_DIR/asan +ASAN_DIR=$SIM_DIR/asan SYSTEM_TEST_DIR=$TOP_DIR/tests/system-test chmod -R 777 $PRG_DIR From 0b0c27ba631227eb62d750ad4dfb450a1e68faf9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 20 Nov 2022 12:43:23 +0800 Subject: [PATCH 74/95] test: adjust asan script --- tests/script/test.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/script/test.sh b/tests/script/test.sh index 9385c73b70..f4979bfa5a 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -138,11 +138,14 @@ if [ -n "$FILE_NAME" ]; then echo "AsanDir:" $ASAN_DIR/tsim.asan eval $PROGRAM -c $CFG_DIR -f $FILE_NAME 2> $ASAN_DIR/tsim.asan result=$? - echo "Execute result: " $result + echo "Execute result:" $result if [ $result -eq 0 ]; then $CODE_DIR/sh/checkAsan.sh else + echo "TSIM has asan errors" + sleep 1 + $CODE_DIR/sh/checkAsan.sh exit 1 fi fi From 40b2d3673b29081bdbffb036d1069058804edf28 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 20 Nov 2022 15:04:44 +0800 Subject: [PATCH 75/95] test: python asan --- tests/parallel_test/cases.task | 6 +++--- tests/pytest/util/log.py | 2 +- tests/system-test/test.sh | 7 ++++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index a43859ebfa..e1b7f9cb32 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -278,7 +278,7 @@ ,,y,script,./test.sh -f tsim/stable/values.sim ,,y,script,./test.sh -f tsim/stable/vnode3.sim ,,y,script,./test.sh -f tsim/stable/metrics_idx.sim -,,n,script,./test.sh -f tsim/sma/drop_sma.sim +,,y,script,./test.sh -f tsim/sma/drop_sma.sim ,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim @@ -438,11 +438,11 @@ ,,,system-test,python3 ./test.py -f 1-insert/database_pre_suf.py ,,,system-test,python3 ./test.py -f 1-insert/InsertFuturets.py ,,,system-test,python3 ./test.py -f 0-others/show.py -,,,system-test,python3 ./test.py -f 2-query/abs.py +,,,system-test,./test.sh python3 ./test.py -f 2-query/abs.py ,,,system-test,python3 ./test.py -f 2-query/abs.py -R ,,,system-test,python3 ./test.py -f 2-query/and_or_for_byte.py ,,,system-test,python3 ./test.py -f 2-query/and_or_for_byte.py -R -,,,system-test,python3 ./test.py -f 2-query/apercentile.py +,,,system-test,./test.sh python3 ./test.py -f 2-query/apercentile.py ,,,system-test,python3 ./test.py -f 2-query/apercentile.py -R ,,,system-test,python3 ./test.py -f 2-query/arccos.py ,,,system-test,python3 ./test.py -f 2-query/arccos.py -R diff --git a/tests/pytest/util/log.py b/tests/pytest/util/log.py index a132178308..000c907ea4 100644 --- a/tests/pytest/util/log.py +++ b/tests/pytest/util/log.py @@ -33,7 +33,7 @@ class TDLog: print("\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err)) def success(self, info): - print("\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info)) + printf("\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info)) def notice(self, err): print("\033[1;33m%s %s\033[0m" % (datetime.datetime.now(), err)) diff --git a/tests/system-test/test.sh b/tests/system-test/test.sh index 2a3187e641..bba6552fab 100755 --- a/tests/system-test/test.sh +++ b/tests/system-test/test.sh @@ -68,15 +68,16 @@ ulimit -c unlimited #sudo sysctl -w kernel.core_pattern=$TOP_DIR/core.%p.%e echo "ExcuteCmd:" $* -echo "AsanDir:" $ASAN_DIR/psim.asan +echo "AsanDir:" $ASAN_DIR/psim.info export LD_PRELOAD=libasan.so.5 -$* -a 2> $ASAN_DIR/psim.asan +$* -a 2> $ASAN_DIR/psim.info + result=$? echo "Execute result:" $result if [ $result -eq 0 ]; then - $CODE_DIR/sh/checkAsan.sh + $TOP_DIR/tests/script/sh/checkAsan.sh else exit 1 fi From 23f7b67b8a9087df441a7d00de56d74e04062ff5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Nov 2022 15:51:40 +0800 Subject: [PATCH 76/95] fix: binary NONE + NULL commit coredump --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 0aa2c6ab83..b0cf09662c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1698,7 +1698,7 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol size += pBlockCol->szBitmap; // offset - if (IS_VAR_DATA_TYPE(pColData->type)) { + if (IS_VAR_DATA_TYPE(pColData->type) && pColData->flag != (HAS_NULL | HAS_NONE)) { code = tsdbCmprData((uint8_t *)pColData->aOffset, sizeof(int32_t) * pColData->nVal, TSDB_DATA_TYPE_INT, cmprAlg, ppOut, nOut + size, &pBlockCol->szOffset, ppBuf); if (code) goto _exit; From ed7eb1cbf101f35b47eb139a6ae64fce44e0b078 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 20 Nov 2022 16:47:10 +0800 Subject: [PATCH 77/95] test: python asan --- tests/system-test/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/test.sh b/tests/system-test/test.sh index bba6552fab..c6238d37d0 100755 --- a/tests/system-test/test.sh +++ b/tests/system-test/test.sh @@ -45,7 +45,7 @@ declare -x SIM_DIR=$TOP_DIR/sim PROGRAM=$BUILD_DIR/build/bin/tsim PRG_DIR=$SIM_DIR/tsim ASAN_DIR=$SIM_DIR/asan -SYSTEM_TEST_DIR=$TOP_DIR/tests/system-test +SYSTEM_TEST_DIR=$CODE_DIR/tests/system-test chmod -R 777 $PRG_DIR echo "------------------------------------------------------------------------" @@ -77,7 +77,7 @@ result=$? echo "Execute result:" $result if [ $result -eq 0 ]; then - $TOP_DIR/tests/script/sh/checkAsan.sh + $CODE_DIR/tests/script/sh/checkAsan.sh else exit 1 fi From 4dc1b33414e918b90bc68f7b96228e129b088e30 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 20 Nov 2022 17:29:16 +0800 Subject: [PATCH 78/95] test: python asan --- tests/script/sh/checkAsan.sh | 1 + tests/script/sh/exec.sh | 1 + tests/script/sh/stop_dnodes.sh | 2 +- tests/system-test/test.py | 2 +- tests/system-test/test.sh | 28 +++++++++++++++++++--------- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 0ce74a989e..8e5738e911 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -3,6 +3,7 @@ set +e #set -x +export LD_PRELOAD= SCRIPT_DIR=`dirname $0` cd $SCRIPT_DIR/../ SCRIPT_DIR=`pwd` diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index 5ef4cca741..ad671631a9 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -11,6 +11,7 @@ set +e #set -x +export LD_PRELOAD= UNAME_BIN=`which uname` OS_TYPE=`$UNAME_BIN` diff --git a/tests/script/sh/stop_dnodes.sh b/tests/script/sh/stop_dnodes.sh index 38667d9b3f..34777ccb59 100755 --- a/tests/script/sh/stop_dnodes.sh +++ b/tests/script/sh/stop_dnodes.sh @@ -3,10 +3,10 @@ set +e #set -x +export LD_PRELOAD= UNAME_BIN=`which uname` OS_TYPE=`$UNAME_BIN` -export LD_PRELOAD= PID=`ps -ef|grep /usr/bin/taosd | grep -v grep | awk '{print $2}'` if [ -n "$PID" ]; then echo systemctl stop taosd diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 712b9689ba..cf9aba123c 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -555,5 +555,5 @@ if __name__ == "__main__": conn.close() if asan: tdDnodes.StopAllSigint() - tdLog.info("address sanitizer mode finished") + tdLog.info("Address sanitizer mode finished") sys.exit(0) diff --git a/tests/system-test/test.sh b/tests/system-test/test.sh index c6238d37d0..6fbc9961d6 100755 --- a/tests/system-test/test.sh +++ b/tests/system-test/test.sh @@ -45,13 +45,11 @@ declare -x SIM_DIR=$TOP_DIR/sim PROGRAM=$BUILD_DIR/build/bin/tsim PRG_DIR=$SIM_DIR/tsim ASAN_DIR=$SIM_DIR/asan -SYSTEM_TEST_DIR=$CODE_DIR/tests/system-test chmod -R 777 $PRG_DIR echo "------------------------------------------------------------------------" echo "Start TDengine Testing Case ..." echo "BUILD_DIR: $BUILD_DIR" -echo "SYSTEM_TEST_DIR : $SYSTEM_TEST_DIR" echo "SIM_DIR : $SIM_DIR" echo "CODE_DIR : $CODE_DIR" echo "ASAN_DIR : $ASAN_DIR" @@ -61,24 +59,36 @@ rm -rf $SIM_DIR/* mkdir -p $PRG_DIR mkdir -p $ASAN_DIR -cd $SYSTEM_TEST_DIR +cd $CODE_DIR ulimit -n 600000 ulimit -c unlimited #sudo sysctl -w kernel.core_pattern=$TOP_DIR/core.%p.%e echo "ExcuteCmd:" $* -echo "AsanDir:" $ASAN_DIR/psim.info +AsanFile=$ASAN_DIR/psim.asan +echo "AsanFile:" $AsanFile export LD_PRELOAD=libasan.so.5 -$* -a 2> $ASAN_DIR/psim.info +$* -a 2> $AsanFile -result=$? -echo "Execute result:" $result +export LD_PRELOAD= +AsanFileLen=`cat $AsanFile | wc -l` +while [ $AsanFileLen -lt 10 ] +do + sleep 1 + `cat $AsanFile | wc -l` +done +echo "AsanFileLen:" $AsanFileLen -if [ $result -eq 0 ]; then - $CODE_DIR/tests/script/sh/checkAsan.sh +AsanFileSuccessLen=`grep -w successfully $AsanFile | wc -l` +echo "AsanFileSuccessLen:" $AsanFileSuccessLen + +if [ $AsanFileSuccessLen -gt 0 ]; then + echo "Execute script successfully and check asan" + $CODE_DIR/../script/sh/checkAsan.sh else + echo "Execute script failure" exit 1 fi From 15c0bb944613f249ba2911218718ae04acd96187 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 20 Nov 2022 21:26:54 +0800 Subject: [PATCH 79/95] test: python asan --- tests/pytest/util/dnodes.py | 15 ++++++++++++--- tests/script/sh/checkAsan.sh | 2 +- tests/script/sh/exec.sh | 2 +- tests/script/sh/sigint_stop_dnodes.sh | 2 +- tests/script/sh/stop_dnodes.sh | 2 +- tests/system-test/{test.sh => pytest.sh} | 3 ++- 6 files changed, 18 insertions(+), 8 deletions(-) rename tests/system-test/{test.sh => pytest.sh} (98%) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index edb3d761cb..22e6127973 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -162,7 +162,11 @@ class TDDnode: def setAsan(self, value): self.asan = value if value: - self.execPath = os.path.abspath(self.path + "/tests/script/sh/exec.sh") + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + self.execPath = os.path.abspath(self.path + "/community/tests/script/sh/exec.sh") + else: + self.execPath = os.path.abspath(self.path + "/tests/script/sh/exec.sh") def getDataSize(self): totalSize = 0 @@ -670,8 +674,13 @@ class TDDnodes: def setAsan(self, value): self.asan = value if value: - self.stopDnodesPath = os.path.abspath(self.path + "/tests/script/sh/stop_dnodes.sh") - self.stopDnodesSigintPath = os.path.abspath(self.path + "/tests/script/sh/sigint_stop_dnodes.sh") + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + self.stopDnodesPath = os.path.abspath(self.path + "/community/tests/script/sh/stop_dnodes.sh") + self.stopDnodesSigintPath = os.path.abspath(self.path + "/community/tests/script/sh/sigint_stop_dnodes.sh") + else: + self.stopDnodesPath = os.path.abspath(self.path + "/tests/script/sh/stop_dnodes.sh") + self.stopDnodesSigintPath = os.path.abspath(self.path + "/tests/script/sh/sigint_stop_dnodes.sh") tdLog.info("run in address sanitizer mode") def setKillValgrind(self, value): diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 8e5738e911..5dfb19e0ae 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -3,7 +3,7 @@ set +e #set -x -export LD_PRELOAD= +unset LD_PRELOAD SCRIPT_DIR=`dirname $0` cd $SCRIPT_DIR/../ SCRIPT_DIR=`pwd` diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index ad671631a9..c8cb121b8a 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -11,7 +11,7 @@ set +e #set -x -export LD_PRELOAD= +unset LD_PRELOAD UNAME_BIN=`which uname` OS_TYPE=`$UNAME_BIN` diff --git a/tests/script/sh/sigint_stop_dnodes.sh b/tests/script/sh/sigint_stop_dnodes.sh index 12b58225de..a398e00772 100755 --- a/tests/script/sh/sigint_stop_dnodes.sh +++ b/tests/script/sh/sigint_stop_dnodes.sh @@ -3,7 +3,7 @@ set +e #set -x -export LD_PRELOAD= +unset LD_PRELOAD UNAME_BIN=`which uname` OS_TYPE=`$UNAME_BIN` diff --git a/tests/script/sh/stop_dnodes.sh b/tests/script/sh/stop_dnodes.sh index 34777ccb59..ce2d7144f9 100755 --- a/tests/script/sh/stop_dnodes.sh +++ b/tests/script/sh/stop_dnodes.sh @@ -3,7 +3,7 @@ set +e #set -x -export LD_PRELOAD= +unset LD_PRELOAD UNAME_BIN=`which uname` OS_TYPE=`$UNAME_BIN` diff --git a/tests/system-test/test.sh b/tests/system-test/pytest.sh similarity index 98% rename from tests/system-test/test.sh rename to tests/system-test/pytest.sh index 6fbc9961d6..f6d081da2a 100755 --- a/tests/system-test/test.sh +++ b/tests/system-test/pytest.sh @@ -69,10 +69,11 @@ echo "ExcuteCmd:" $* AsanFile=$ASAN_DIR/psim.asan echo "AsanFile:" $AsanFile +unset LD_PRELOAD export LD_PRELOAD=libasan.so.5 $* -a 2> $AsanFile -export LD_PRELOAD= +unset LD_PRELOAD AsanFileLen=`cat $AsanFile | wc -l` while [ $AsanFileLen -lt 10 ] do From c8f6235027c9651208c4537b6d62d2b56f5185d5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 20 Nov 2022 22:09:48 +0800 Subject: [PATCH 80/95] test: python asan --- tests/parallel_test/cases.task | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e1b7f9cb32..12e0a9c89d 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -438,11 +438,11 @@ ,,,system-test,python3 ./test.py -f 1-insert/database_pre_suf.py ,,,system-test,python3 ./test.py -f 1-insert/InsertFuturets.py ,,,system-test,python3 ./test.py -f 0-others/show.py -,,,system-test,./test.sh python3 ./test.py -f 2-query/abs.py +,,,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py ,,,system-test,python3 ./test.py -f 2-query/abs.py -R ,,,system-test,python3 ./test.py -f 2-query/and_or_for_byte.py ,,,system-test,python3 ./test.py -f 2-query/and_or_for_byte.py -R -,,,system-test,./test.sh python3 ./test.py -f 2-query/apercentile.py +,,,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py ,,,system-test,python3 ./test.py -f 2-query/apercentile.py -R ,,,system-test,python3 ./test.py -f 2-query/arccos.py ,,,system-test,python3 ./test.py -f 2-query/arccos.py -R From d17dd17be49a811cbafaf2d0fbb479a9111aad28 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 20 Nov 2022 23:14:33 +0800 Subject: [PATCH 81/95] test: preload libasan --- tests/system-test/pytest.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/system-test/pytest.sh b/tests/system-test/pytest.sh index f6d081da2a..d5ae59b4fe 100755 --- a/tests/system-test/pytest.sh +++ b/tests/system-test/pytest.sh @@ -70,7 +70,14 @@ AsanFile=$ASAN_DIR/psim.asan echo "AsanFile:" $AsanFile unset LD_PRELOAD -export LD_PRELOAD=libasan.so.5 +#export LD_PRELOAD=libasan.so.5 +#echo "export1:" $? +export LD_PRELOAD=`gcc -print-file-name=libasan.so` +echo "export2:" $? + +echo "export4:" "$BUILD_DIR/build/bin/taos" +ldd $BUILD_DIR/build/bin/taos + $* -a 2> $AsanFile unset LD_PRELOAD From 2a497fff7c9ad5a5087943815c82bf6125b99d8e Mon Sep 17 00:00:00 2001 From: Pan YANG Date: Mon, 21 Nov 2022 00:02:50 +0800 Subject: [PATCH 82/95] docs: add TDengine knowledge map --- docs/zh/05-get-started/index.md | 13 ++++++++++++- docs/zh/05-get-started/tdengine-map.md | 7 +++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 docs/zh/05-get-started/tdengine-map.md diff --git a/docs/zh/05-get-started/index.md b/docs/zh/05-get-started/index.md index 6faa7fed01..c9eb94ab28 100644 --- a/docs/zh/05-get-started/index.md +++ b/docs/zh/05-get-started/index.md @@ -18,7 +18,18 @@ import {useCurrentSidebarCategory} from '@docusaurus/theme-common'; ``` -### 加入 TDengine 官方社区 +## 学习 TDengine 知识地图 + +TDengine 知识地图中涵盖了 TDengine 的各种知识点，揭示了各概念实体之间的调用关系和数据流向。学习和了解 TDengine 知识地图有助于你快速掌握 TDengine 的知识体系。 + +
+
+ +
图 1. TDengine 知识地图
+
+
+ +## 加入 TDengine 官方社区 微信扫描以下二维码，学习了解 TDengine 的最新技术，与大家共同交流物联网大数据技术应用、TDengine 使用问题和技巧等话题。 diff --git a/docs/zh/05-get-started/tdengine-map.md b/docs/zh/05-get-started/tdengine-map.md new file mode 100644 index 0000000000..574fa0a4fe --- /dev/null +++ b/docs/zh/05-get-started/tdengine-map.md @@ -0,0 +1,7 @@ +--- +title: TDengine 知识地图 +description: '通过知识地图快速掌握 TDengine 的知识体系' +hide_table_of_contents: true +--- + + From af6b1cf3dcbc846b399a8ba9fff824f3e4a369e3 Mon Sep 17 00:00:00 2001 From: Pan YANG Date: Mon, 21 Nov 2022 00:26:16 +0800 Subject: [PATCH 83/95] docs: link to tdengine-map.svg directly --- docs/zh/05-get-started/index.md | 2 +- docs/zh/05-get-started/tdengine-map.md | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) delete mode 100644 docs/zh/05-get-started/tdengine-map.md diff --git a/docs/zh/05-get-started/index.md b/docs/zh/05-get-started/index.md index c9eb94ab28..f179dea26d 100644 --- a/docs/zh/05-get-started/index.md +++ b/docs/zh/05-get-started/index.md @@ -24,7 +24,7 @@ TDengine 知识地图中涵盖了 TDengine 的各种知识点，揭示了各概
- +
图 1. TDengine 知识地图
diff --git a/docs/zh/05-get-started/tdengine-map.md b/docs/zh/05-get-started/tdengine-map.md deleted file mode 100644 index 574fa0a4fe..0000000000 --- a/docs/zh/05-get-started/tdengine-map.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -title: TDengine 知识地图 -description: '通过知识地图快速掌握 TDengine 的知识体系' -hide_table_of_contents: true ---- - - From ee9f5740cfb0ed0ba830564c3b496b7d4bf184fe Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Nov 2022 08:52:32 +0800 Subject: [PATCH 84/95] test: check asan errors --- tests/parallel_test/cases.task | 2 +- tests/script/sh/checkAsan.sh | 7 ++++++- tests/system-test/2-query/abs.py | 2 ++ tests/system-test/pytest.sh | 8 ++------ 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 12e0a9c89d..a9a02d8380 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -278,7 +278,7 @@ ,,y,script,./test.sh -f tsim/stable/values.sim ,,y,script,./test.sh -f tsim/stable/vnode3.sim ,,y,script,./test.sh -f tsim/stable/metrics_idx.sim -,,y,script,./test.sh -f tsim/sma/drop_sma.sim +,,n,script,./test.sh -f tsim/sma/drop_sma.sim ,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 5dfb19e0ae..c53fde1171 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -22,19 +22,24 @@ error_num=`cat ${LOG_DIR}/*.asan | grep "ERROR" | wc -l` memory_leak=`cat ${LOG_DIR}/*.asan | grep "Direct leak" | wc -l` indirect_leak=`cat ${LOG_DIR}/*.asan | grep "Indirect leak" | wc -l` runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | wc -l` +python_error=`cat ${LOG_DIR}/*.info | grep "stack" | wc -l` echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m" echo -e "\033[44;32;1m"asan memory_leak: $memory_leak"\033[0m" echo -e "\033[44;32;1m"asan indirect_leak: $indirect_leak"\033[0m" echo -e "\033[44;32;1m"asan runtime error: $runtime_error"\033[0m" +echo -e "\033[44;32;1m"asan python error: $python_error"\033[0m" -let "errors=$error_num+$memory_leak+$indirect_leak+$runtime_error" +let "errors=$error_num+$memory_leak+$indirect_leak+$runtime_error+$python_error" if [ $errors -eq 0 ]; then echo -e "\033[44;32;1m"no asan errors"\033[0m" exit 0 else echo -e "\033[44;31;1m"asan total errors: $errors"\033[0m" + if [ $python_error -ne 0 ]; then + cat ${LOG_DIR}/*.info + fi cat ${LOG_DIR}/*.asan exit 1 fi \ No newline at end of file diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index d7478a55a8..dcd9e6d620 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -487,6 +487,8 @@ class TDTestCase: self.check_result_auto( f"select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from {dbname}.sub1_bound;", f"select abs(c1) from {dbname}.sub1_bound") + tdSql.query(f"select t1 xxx") + # check basic elem for table per row tdSql.query( f"select abs(c1) ,abs(c2) , abs(c3) , abs(c4), abs(c5), abs(c6) from {dbname}.sub1_bound ") diff --git a/tests/system-test/pytest.sh b/tests/system-test/pytest.sh index d5ae59b4fe..3f03058342 100755 --- a/tests/system-test/pytest.sh +++ b/tests/system-test/pytest.sh @@ -66,17 +66,13 @@ ulimit -c unlimited #sudo sysctl -w kernel.core_pattern=$TOP_DIR/core.%p.%e echo "ExcuteCmd:" $* -AsanFile=$ASAN_DIR/psim.asan +AsanFile=$ASAN_DIR/psim.info echo "AsanFile:" $AsanFile unset LD_PRELOAD #export LD_PRELOAD=libasan.so.5 -#echo "export1:" $? export LD_PRELOAD=`gcc -print-file-name=libasan.so` -echo "export2:" $? - -echo "export4:" "$BUILD_DIR/build/bin/taos" -ldd $BUILD_DIR/build/bin/taos +echo "Preload AsanSo:" $? $* -a 2> $AsanFile From bb6ca0c0287630b26405402a2ae19fbad4ac8a14 Mon Sep 17 00:00:00 2001 From: wenzhouwww Date: Mon, 21 Nov 2022 09:32:51 +0800 Subject: [PATCH 85/95] Update crash_gen_main.py --- tests/pytest/crash_gen/crash_gen_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 49b68965d0..f8c5f970c5 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -2025,7 +2025,7 @@ class TdSuperTable: conf.set("enable.auto.commit", "true") def tmq_commit_cb_print(tmq, resp, offset, param=None): print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}") - # conf.set_auto_commit_cb(tmq_commit_cb_print, None) + conf.set_auto_commit_cb(tmq_commit_cb_print, None) consumer = conf.new_consumer() topic_list = TaosTmqList() for topic in current_topic_list: From c454f849b21f238af3fe3cb7dcb3fcb476d1d9cc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Nov 2022 09:45:38 +0800 Subject: [PATCH 86/95] test: adjust python asan script --- tests/script/sh/checkAsan.sh | 2 +- tests/system-test/2-query/abs.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index c53fde1171..8759db8722 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -22,7 +22,7 @@ error_num=`cat ${LOG_DIR}/*.asan | grep "ERROR" | wc -l` memory_leak=`cat ${LOG_DIR}/*.asan | grep "Direct leak" | wc -l` indirect_leak=`cat ${LOG_DIR}/*.asan | grep "Indirect leak" | wc -l` runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | wc -l` -python_error=`cat ${LOG_DIR}/*.info | grep "stack" | wc -l` +python_error=`cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l` echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m" echo -e "\033[44;32;1m"asan memory_leak: $memory_leak"\033[0m" diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index dcd9e6d620..d7478a55a8 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -487,8 +487,6 @@ class TDTestCase: self.check_result_auto( f"select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from {dbname}.sub1_bound;", f"select abs(c1) from {dbname}.sub1_bound") - tdSql.query(f"select t1 xxx") - # check basic elem for table per row tdSql.query( f"select abs(c1) ,abs(c2) , abs(c3) , abs(c4), abs(c5), abs(c6) from {dbname}.sub1_bound ") From 9aaf2bea428c4b62af10b128cfe55b59a58b7485 Mon Sep 17 00:00:00 2001 From: Pan YANG Date: Mon, 21 Nov 2022 10:32:22 +0800 Subject: [PATCH 87/95] docs: fix pathname of resources file --- docs/zh/05-get-started/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/05-get-started/index.md b/docs/zh/05-get-started/index.md index f179dea26d..832310aa7c 100644 --- a/docs/zh/05-get-started/index.md +++ b/docs/zh/05-get-started/index.md @@ -24,7 +24,7 @@ TDengine 知识地图中涵盖了 TDengine 的各种知识点，揭示了各概
- +
图 1. TDengine 知识地图
From e5470224cf0e9c4bc5ca9a1662aada971d9ad76b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Nov 2022 11:01:44 +0800 Subject: [PATCH 88/95] test: close all dnodes when tsim execution is complete --- tests/script/sh/sigint_stop_dnodes.sh | 2 +- tests/script/test.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/script/sh/sigint_stop_dnodes.sh b/tests/script/sh/sigint_stop_dnodes.sh index a398e00772..83a4f1c1d5 100755 --- a/tests/script/sh/sigint_stop_dnodes.sh +++ b/tests/script/sh/sigint_stop_dnodes.sh @@ -8,9 +8,9 @@ UNAME_BIN=`which uname` OS_TYPE=`$UNAME_BIN` PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` +echo "Killing taosd processes " $PID while [ -n "$PID" ]; do #echo "Killing taosd processes " $PID kill $PID - PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` done diff --git a/tests/script/test.sh b/tests/script/test.sh index f4979bfa5a..a7a5d34fbe 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -141,6 +141,7 @@ if [ -n "$FILE_NAME" ]; then echo "Execute result:" $result if [ $result -eq 0 ]; then + $CODE_DIR/sh/sigint_stop_dnodes.sh $CODE_DIR/sh/checkAsan.sh else echo "TSIM has asan errors" From 391389e989f7dc7a2f2175b5e75b7671568d9784 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Nov 2022 11:47:55 +0800 Subject: [PATCH 89/95] test: close all dnodes when tsim execution is complete --- tests/parallel_test/cases.task | 7 +- tests/script/tsim/sma/sma_leak.sim | 154 +++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+), 3 deletions(-) create mode 100644 tests/script/tsim/sma/sma_leak.sim diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index a9a02d8380..f9e8cd6271 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -216,7 +216,7 @@ ,,y,script,./test.sh -f tsim/stream/drop_stream.sim ,,y,script,./test.sh -f tsim/stream/fillHistoryBasic1.sim ,,y,script,./test.sh -f tsim/stream/fillHistoryBasic2.sim -,,y,script,./test.sh -f tsim/stream/fillHistoryBasic3.sim +,,n,script,./test.sh -f tsim/stream/fillHistoryBasic3.sim ,,y,script,./test.sh -f tsim/stream/distributeInterval0.sim ,,y,script,./test.sh -f tsim/stream/distributeIntervalRetrive0.sim ,,y,script,./test.sh -f tsim/stream/distributeSession0.sim @@ -227,11 +227,11 @@ ,,y,script,./test.sh -f tsim/stream/triggerSession0.sim ,,y,script,./test.sh -f tsim/stream/partitionby.sim ,,y,script,./test.sh -f tsim/stream/partitionby1.sim -,,y,script,./test.sh -f tsim/stream/schedSnode.sim +,,n,script,./test.sh -f tsim/stream/schedSnode.sim ,,y,script,./test.sh -f tsim/stream/windowClose.sim ,,y,script,./test.sh -f tsim/stream/ignoreExpiredData.sim ,,y,script,./test.sh -f tsim/stream/sliding.sim -,,y,script,./test.sh -f tsim/stream/partitionbyColumnInterval.sim +,,n,script,./test.sh -f tsim/stream/partitionbyColumnInterval.sim ,,y,script,./test.sh -f tsim/stream/partitionbyColumnSession.sim ,,y,script,./test.sh -f tsim/stream/partitionbyColumnState.sim ,,y,script,./test.sh -f tsim/stream/deleteInterval.sim @@ -279,6 +279,7 @@ ,,y,script,./test.sh -f tsim/stable/vnode3.sim ,,y,script,./test.sh -f tsim/stable/metrics_idx.sim ,,n,script,./test.sh -f tsim/sma/drop_sma.sim +,,n,script,./test.sh -f tsim/sma/sma_leak.sim ,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim diff --git a/tests/script/tsim/sma/sma_leak.sim b/tests/script/tsim/sma/sma_leak.sim new file mode 100644 index 0000000000..4f2d1ebeb0 --- /dev/null +++ b/tests/script/tsim/sma/sma_leak.sim @@ -0,0 +1,154 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/cfg.sh -n dnode2 -c supportVnodes -v 4 +system sh/cfg.sh -n dnode3 -c supportVnodes -v 4 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ========== step2 +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start + +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql select * from information_schema.ins_dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 3 then + return -1 +endi +if $data(1)[4] != ready then + goto step2 +endi +if $data(2)[4] != ready then + goto step2 +endi +if $data(3)[4] != ready then + goto step2 +endi + +print ========== step3 +sql create database d1 vgroups 1 +sql use d1; + +print --> create stb +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned); + +print --> create sma +sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) interval(6m,10s) sliding(6m); + +return + +print --> show sma +sql show indexes from stb from d1; +if $rows != 1 then + return -1 +endi +if $data[0][0] != sma_index_name1 then + return -1 +endi +if $data[0][1] != d1 then + return -1 +endi +if $data[0][2] != stb then + return -1 +endi + +print --> drop stb +sql drop table stb; + +print ========== step4 repeat + +print --> create stb +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned); + +print --> create sma +sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) interval(6m,10s) sliding(6m); + +print --> show sma +sql show indexes from stb from d1; +if $rows != 1 then + return -1 +endi +if $data[0][0] != sma_index_name1 then + return -1 +endi +if $data[0][1] != d1 then + return -1 +endi +if $data[0][2] != stb then + return -1 +endi + +print --> drop stb +sql drop table stb; + +print ========== step5 +sql drop database if exists db; +sql create database db duration 300; +sql use db; +sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint, c_float float, c_double double, c_bool bool, c_binary binary(16), c_nchar nchar(32), c_ts timestamp, c_tint_un tinyint unsigned, c_sint_un smallint unsigned, c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int); +sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s; + +print ========== step6 repeat +sql drop database if exists db; +sql create database db duration 300; +sql use db; +sql create table stb1(ts timestamp, c_int int, c_bint bigint ) tags (t_int int); +sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s; + +print ========== step7 +sql drop database if exists db; +sql create database db duration 300; +sql use db; +sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int); + +sql create table ct1 using stb1 tags ( 1 ); +sql create table ct2 using stb1 tags ( 2 ); +sql create table ct3 using stb1 tags ( 3 ); +sql create table ct4 using stb1 tags ( 4 ); + +sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s; +sql CREATE SMA INDEX sma_index_2 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) max_delay 6m; +sql CREATE SMA INDEX sma_index_3 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) watermark 5s max_delay 6m; + +sql DROP INDEX sma_index_1 ; +sql DROP INDEX sma_index_2 ; +sql DROP INDEX sma_index_3 ; + +print ========== step8 +sql drop database if exists db; +sql create database db duration 300; +sql use db; +sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int); + +sql create table ct1 using stb1 tags ( 1 ); +sql create table ct2 using stb1 tags ( 2 ); +sql create table ct3 using stb1 tags ( 3 ); +sql create table ct4 using stb1 tags ( 4 ); + +sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s; +sql CREATE SMA INDEX sma_index_2 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) max_delay 6m; +sql CREATE SMA INDEX sma_index_3 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) watermark 5s max_delay 6m; + +sql DROP INDEX sma_index_1 ; +sql DROP INDEX sma_index_2 ; +sql DROP INDEX sma_index_3 ; + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT From ada76477fc818986a18fe64a81501445b3d23ed3 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 21 Nov 2022 11:54:25 +0800 Subject: [PATCH 90/95] fix: free stream state cursor --- source/libs/stream/src/streamState.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 0374e22a4a..ea31347db5 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -411,6 +411,7 @@ int32_t streamStateGetFirst(SStreamState* pState, SWinKey* key) { streamStatePut(pState, &tmp, NULL, 0); SStreamStateCur* pCur = streamStateSeekKeyNext(pState, &tmp); int32_t code = streamStateGetKVByCur(pCur, key, NULL, 0); + streamStateFreeCur(pCur); streamStateDel(pState, &tmp); return code; } From afe7a19ba0f3affe572927b437dba3d1fff24b16 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Nov 2022 14:09:12 +0800 Subject: [PATCH 91/95] test: add asan case --- tests/parallel_test/cases.task | 24 ++++++++++++------------ tests/script/sh/checkAsan.sh | 9 ++++++++- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index f9e8cd6271..890412f8eb 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -439,31 +439,31 @@ ,,,system-test,python3 ./test.py -f 1-insert/database_pre_suf.py ,,,system-test,python3 ./test.py -f 1-insert/InsertFuturets.py ,,,system-test,python3 ./test.py -f 0-others/show.py -,,,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py ,,,system-test,python3 ./test.py -f 2-query/abs.py -R -,,,system-test,python3 ./test.py -f 2-query/and_or_for_byte.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/and_or_for_byte.py ,,,system-test,python3 ./test.py -f 2-query/and_or_for_byte.py -R -,,,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py ,,,system-test,python3 ./test.py -f 2-query/apercentile.py -R -,,,system-test,python3 ./test.py -f 2-query/arccos.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py ,,,system-test,python3 ./test.py -f 2-query/arccos.py -R -,,,system-test,python3 ./test.py -f 2-query/arcsin.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py ,,,system-test,python3 ./test.py -f 2-query/arcsin.py -R -,,,system-test,python3 ./test.py -f 2-query/arctan.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py ,,,system-test,python3 ./test.py -f 2-query/arctan.py -R ,,,system-test,python3 ./test.py -f 2-query/avg.py ,,,system-test,python3 ./test.py -f 2-query/avg.py -R -,,,system-test,python3 ./test.py -f 2-query/between.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py ,,,system-test,python3 ./test.py -f 2-query/between.py -R -,,,system-test,python3 ./test.py -f 2-query/bottom.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py ,,,system-test,python3 ./test.py -f 2-query/bottom.py -R -,,,system-test,python3 ./test.py -f 2-query/cast.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py ,,,system-test,python3 ./test.py -f 2-query/cast.py -R -,,,system-test,python3 ./test.py -f 2-query/ceil.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py ,,,system-test,python3 ./test.py -f 2-query/ceil.py -R -,,,system-test,python3 ./test.py -f 2-query/char_length.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py ,,,system-test,python3 ./test.py -f 2-query/char_length.py -R -,,,system-test,python3 ./test.py -f 2-query/check_tsdb.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py ,,,system-test,python3 ./test.py -f 2-query/check_tsdb.py -R ,,,system-test,python3 ./test.py -f 2-query/concat.py ,,,system-test,python3 ./test.py -f 2-query/concat.py -R diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 8759db8722..074956534f 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -21,7 +21,14 @@ LOG_DIR=$TAOS_DIR/sim/asan error_num=`cat ${LOG_DIR}/*.asan | grep "ERROR" | wc -l` memory_leak=`cat ${LOG_DIR}/*.asan | grep "Direct leak" | wc -l` indirect_leak=`cat ${LOG_DIR}/*.asan | grep "Indirect leak" | wc -l` -runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | wc -l` + +# ignore +# /root/TDengine/source/libs/scalar/src/sclfunc.c:735:11: runtime error: 4.75783e+11 is outside the range of representable values of type 'signed char' +# /root/TDengine/source/libs/scalar/src/sclfunc.c:790:11: runtime error: 3.4e+38 is outside the range of representable values of type 'long int' +# /root/TDengine/source/libs/scalar/src/sclfunc.c:772:11: runtime error: 3.52344e+09 is outside the range of representable values of type 'int' +# /root/TDengine/source/libs/scalar/src/sclfunc.c:753:11: runtime error: 4.75783e+11 is outside the range of representable values of type 'short int' +runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type" | wc -l` + python_error=`cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l` echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m" From 36a43fc016ae9db8b598d9be86e762e8a3e01003 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Mon, 21 Nov 2022 14:46:46 +0800 Subject: [PATCH 92/95] fix: set sleep time to 10s, set curl timeout --- packaging/docker/run.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packaging/docker/run.sh b/packaging/docker/run.sh index b5d6b011ea..b02b3cbd8a 100755 --- a/packaging/docker/run.sh +++ b/packaging/docker/run.sh @@ -4,7 +4,7 @@ TAOS_RUN_TAOSBENCHMARK_TEST_ONCE=0 #ADMIN_URL=${ADMIN_URL:-http://172.26.10.84:10001} TAOSD_STARTUP_TIMEOUT_SECOND=${TAOSD_STARTUP_TIMEOUT_SECOND:-160} TAOS_TIMEOUT_SECOND=${TAOS_TIMEOUT_SECOND:-5} -BACKUP_CORE_FOLDER=/data/corefile +BACKUP_CORE_FOLDER=/var/log/corefile ALERT_URL=app/system/alert/add echo "ADMIN_URL: ${ADMIN_URL}" @@ -37,7 +37,7 @@ function post_error_msg() { echo "service_state: ${service_state}" echo "`date` service_msg: ${service_msg}" echo "${taos_version}" - curl -X POST -H "Content-Type: application/json" \ + curl --connect-timeout 10 --max-time 20 -X POST -H "Content-Type: application/json" \ -d"{\"appName\":\"${app_name}\",\ \"alertLevel\":\"${service_state}\",\ \"taosVersion\":\"${taos_version}\",\ @@ -77,7 +77,7 @@ function post_disk_error_msg() { echo "disk_state: ${disk_state}" echo "`date` disk_msg: ${disk_msg}" echo "${taos_version}" - curl -X POST -H "Content-Type: application/json" \ + curl --connect-timeout 10 --max-time 20 -X POST -H "Content-Type: application/json" \ -d"{\"appName\":\"${app_name}\",\ \"alertLevel\":\"${disk_state}\",\ \"taosVersion\":\"${taos_version}\",\ @@ -200,5 +200,5 @@ do if [ $? -ne 0 ]; then taosadapter & fi - sleep 30 + sleep 10 done From f7f7b97279d9c87e12d73ab4410174a186fd38c1 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Mon, 21 Nov 2022 15:19:19 +0800 Subject: [PATCH 93/95] fix: mv does not work cross devices --- packaging/docker/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packaging/docker/run.sh b/packaging/docker/run.sh index b02b3cbd8a..3654beadcb 100755 --- a/packaging/docker/run.sh +++ b/packaging/docker/run.sh @@ -59,7 +59,8 @@ function check_taosd_exit_type() { if [ ! -z "$core_files" ]; then # move core files to another folder mkdir -p ${BACKUP_CORE_FOLDER} - mv ${core_folder}/${core_prefix}* ${BACKUP_CORE_FOLDER}/ + cp ${core_folder}/${core_prefix}* ${BACKUP_CORE_FOLDER}/ + rm -f ${core_folder}/${core_prefix}* set_service_state "error" "taosd exit with core file" else set_service_state "error" "taosd exit without core file" From 2df8d3229a10ef28f9ea1758b464e9994fd53176 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Nov 2022 15:38:50 +0800 Subject: [PATCH 94/95] test: add asan case --- tests/parallel_test/cases.task | 56 +++++++++++++++++----------------- tests/system-test/pytest.sh | 11 ++++--- 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 890412f8eb..b1df6975d8 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -471,63 +471,63 @@ ,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -R ,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py ,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -R -,,,system-test,python3 ./test.py -f 2-query/cos.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py ,,,system-test,python3 ./test.py -f 2-query/cos.py -R -,,,system-test,python3 ./test.py -f 2-query/count_partition.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py ,,,system-test,python3 ./test.py -f 2-query/count_partition.py -R -,,,system-test,python3 ./test.py -f 2-query/count.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py ,,,system-test,python3 ./test.py -f 2-query/count.py -R -,,,system-test,python3 ./test.py -f 2-query/countAlwaysReturnValue.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py ,,,system-test,python3 ./test.py -f 2-query/countAlwaysReturnValue.py -R -,,,system-test,python3 ./test.py -f 2-query/db.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/db.py ,,,system-test,python3 ./test.py -f 2-query/db.py -R -,,,system-test,python3 ./test.py -f 2-query/diff.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py ,,,system-test,python3 ./test.py -f 2-query/diff.py -R -,,,system-test,python3 ./test.py -f 2-query/distinct.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py ,,,system-test,python3 ./test.py -f 2-query/distinct.py -R -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_apercentile.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py ,,,system-test,python3 ./test.py -f 2-query/distribute_agg_apercentile.py -R -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_avg.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py ,,,system-test,python3 ./test.py -f 2-query/distribute_agg_avg.py -R -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_count.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py ,,,system-test,python3 ./test.py -f 2-query/distribute_agg_count.py -R -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_max.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py ,,,system-test,python3 ./test.py -f 2-query/distribute_agg_max.py -R -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_min.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py ,,,system-test,python3 ./test.py -f 2-query/distribute_agg_min.py -R -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_spread.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py ,,,system-test,python3 ./test.py -f 2-query/distribute_agg_spread.py -R -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_stddev.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py ,,,system-test,python3 ./test.py -f 2-query/distribute_agg_stddev.py -R -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_sum.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py ,,,system-test,python3 ./test.py -f 2-query/distribute_agg_sum.py -R -,,,system-test,python3 ./test.py -f 2-query/explain.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py ,,,system-test,python3 ./test.py -f 2-query/explain.py -R -,,,system-test,python3 ./test.py -f 2-query/first.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py ,,,system-test,python3 ./test.py -f 2-query/first.py -R -,,,system-test,python3 ./test.py -f 2-query/floor.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py ,,,system-test,python3 ./test.py -f 2-query/floor.py -R -,,,system-test,python3 ./test.py -f 2-query/function_null.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py ,,,system-test,python3 ./test.py -f 2-query/function_null.py -R -,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_stateduration.py ,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -R -,,,system-test,python3 ./test.py -f 2-query/histogram.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/histogram.py ,,,system-test,python3 ./test.py -f 2-query/histogram.py -R -,,,system-test,python3 ./test.py -f 2-query/hyperloglog.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py ,,,system-test,python3 ./test.py -f 2-query/hyperloglog.py -R ,,,system-test,python3 ./test.py -f 2-query/interp.py ,,,system-test,python3 ./test.py -f 2-query/interp.py -R -,,,system-test,python3 ./test.py -f 2-query/irate.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py ,,,system-test,python3 ./test.py -f 2-query/irate.py -R -,,,system-test,python3 ./test.py -f 2-query/join.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py ,,,system-test,python3 ./test.py -f 2-query/join.py -R ,,,system-test,python3 ./test.py -f 2-query/last_row.py ,,,system-test,python3 ./test.py -f 2-query/last_row.py -R -,,,system-test,python3 ./test.py -f 2-query/last.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py ,,,system-test,python3 ./test.py -f 2-query/last.py -R -,,,system-test,python3 ./test.py -f 2-query/leastsquares.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py ,,,system-test,python3 ./test.py -f 2-query/leastsquares.py -R -,,,system-test,python3 ./test.py -f 2-query/length.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py ,,,system-test,python3 ./test.py -f 2-query/length.py -R ,,,system-test,python3 ./test.py -f 2-query/log.py ,,,system-test,python3 ./test.py -f 2-query/log.py -R @@ -535,7 +535,7 @@ ,,,system-test,python3 ./test.py -f 2-query/lower.py -R ,,,system-test,python3 ./test.py -f 2-query/ltrim.py ,,,system-test,python3 ./test.py -f 2-query/ltrim.py -R -,,,system-test,python3 ./test.py -f 2-query/mavg.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py ,,,system-test,python3 ./test.py -f 2-query/mavg.py -R ,,,system-test,python3 ./test.py -f 2-query/max_partition.py ,,,system-test,python3 ./test.py -f 2-query/max_partition.py -R diff --git a/tests/system-test/pytest.sh b/tests/system-test/pytest.sh index 3f03058342..148f80eca3 100755 --- a/tests/system-test/pytest.sh +++ b/tests/system-test/pytest.sh @@ -77,13 +77,14 @@ echo "Preload AsanSo:" $? $* -a 2> $AsanFile unset LD_PRELOAD -AsanFileLen=`cat $AsanFile | wc -l` -while [ $AsanFileLen -lt 10 ] +for ((i=1;i<=20;i++)) do - sleep 1 - `cat $AsanFile | wc -l` + AsanFileLen=`cat $AsanFile | wc -l` + echo "AsanFileLen:" $AsanFileLen + if [ $AsanFileLen -gt 10 ]; then + break + fi done -echo "AsanFileLen:" $AsanFileLen AsanFileSuccessLen=`grep -w successfully $AsanFile | wc -l` echo "AsanFileSuccessLen:" $AsanFileSuccessLen From 3e81e36522f24497fa4420e2518630a2af18df76 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 21 Nov 2022 15:52:46 +0800 Subject: [PATCH 95/95] feat: taosdump for windows (#18298) * feat: taos-tools 2333455 * feat: taosdump for windows --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 82a7052125..b996ffcd17 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 23e2b73 + GIT_TAG e00ebd9 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE