From a7d21c9ee73c24aa7da8430ebe10008963d75afb Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 8 Nov 2022 14:50:03 +0800 Subject: [PATCH 01/59] enh: interp fill linear ignore null values during calculation --- source/libs/executor/inc/executorimpl.h | 1 - source/libs/executor/src/timewindowoperator.c | 166 ++++-------------- 2 files changed, 33 insertions(+), 134 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index dedf1630af..03907a0d03 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -782,7 +782,6 @@ typedef struct STimeSliceOperatorInfo { SArray* pPrevRow; // SArray SArray* pNextRow; // SArray SArray* pLinearInfo; // SArray - bool fillLastPoint; bool isPrevRowSet; bool isNextRowSet; int32_t fillType; // fill type diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 88f3e4cff9..ef99e31f67 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1991,10 +1991,8 @@ static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock pSliceInfo->isNextRowSet = true; } -static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex, - bool isLastRow) { +static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) { int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - bool fillLastPoint = pSliceInfo->fillLastPoint; for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId); @@ -2002,30 +2000,21 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo // null data should not be kept since it can not be used to perform interpolation if (!colDataIsNull_s(pColInfoData, i)) { - if (isLastRow) { + if (pLinearInfo->start.key == INT64_MIN) { pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); memcpy(pLinearInfo->start.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); - } else if (fillLastPoint) { + } else if (pLinearInfo->end.key == INT64_MAX) { pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); } else { - pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); - pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex + 1); - - char* val; - val = colDataGetData(pColInfoData, rowIndex); - memcpy(pLinearInfo->start.val, val, pLinearInfo->bytes); - val = colDataGetData(pColInfoData, rowIndex + 1); - memcpy(pLinearInfo->end.val, val, pLinearInfo->bytes); + pLinearInfo->start.key = pLinearInfo->end.key; + pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); + memcpy(pLinearInfo->start.val, pLinearInfo->end.val, pLinearInfo->bytes); + memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); } - - pLinearInfo->hasNull = false; - } else { - pLinearInfo->hasNull = true; } } - pSliceInfo->fillLastPoint = isLastRow; } static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock) { @@ -2209,15 +2198,9 @@ static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB taosArrayPush(pInfo->pLinearInfo, &linearInfo); } - pInfo->fillLastPoint = false; - return TSDB_CODE_SUCCESS; } -static bool needToFillLastPoint(STimeSliceOperatorInfo* pSliceInfo) { - return (pSliceInfo->fillLastPoint == true && pSliceInfo->fillType == TSDB_FILL_LINEAR); -} - static int32_t initKeeperInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) { int32_t code; code = initPrevRowsKeeper(pInfo, pBlock); @@ -2273,15 +2256,6 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { for (int32_t i = 0; i < pBlock->info.rows; ++i) { int64_t ts = *(int64_t*)colDataGetData(pTsCol, i); - if (i == 0 && needToFillLastPoint(pSliceInfo)) { // first row in current block - doKeepLinearInfo(pSliceInfo, pBlock, i, false); - while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); - pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - } - } - if (pSliceInfo->current > pSliceInfo->win.ekey) { doSetOperatorCompleted(pOperator); break; @@ -2313,94 +2287,44 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { pResBlock->info.rows += 1; doKeepPrevRows(pSliceInfo, pBlock, i); + doKeepLinearInfo(pSliceInfo, pBlock, i); - // for linear interpolation, always fill value between this and next points; - // if its the first point in data block, also fill values between previous(if there's any) and this point; - // if its the last point in data block, no need to fill, but reserve this point as the start value and do - // the interpolation when processing next data block. - if (pSliceInfo->fillType == TSDB_FILL_LINEAR) { - pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - if (i < pBlock->info.rows - 1) { - doKeepLinearInfo(pSliceInfo, pBlock, i, false); - int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); - if (nextTs > pSliceInfo->current) { - while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); - pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, - pInterval->precision); - } - - if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); - break; - } - } - } else { // it is the last row of current block - // store ts value as start, and calculate interp value when processing next block - doKeepLinearInfo(pSliceInfo, pBlock, i, true); - } - } else { // non-linear interpolation - pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); - break; - } + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + if (pSliceInfo->current > pSliceInfo->win.ekey) { + doSetOperatorCompleted(pOperator); + break; } } else if (ts < pSliceInfo->current) { // in case of interpolation window starts and ends between two datapoints, fill(prev) need to interpolate doKeepPrevRows(pSliceInfo, pBlock, i); + doKeepLinearInfo(pSliceInfo, pBlock, i); - if (pSliceInfo->fillType == TSDB_FILL_LINEAR) { - // no need to increate pSliceInfo->current here - // pSliceInfo->current = - // taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - if (i < pBlock->info.rows - 1) { - doKeepLinearInfo(pSliceInfo, pBlock, i, false); - int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); - if (nextTs > pSliceInfo->current) { - while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); - pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, - pInterval->precision); - } + if (i < pBlock->info.rows - 1) { + // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate + doKeepNextRows(pSliceInfo, pBlock, i + 1); + int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); + if (nextTs > pSliceInfo->current) { + while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); + pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, + pInterval->precision); + } - if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); - break; - } + if (pSliceInfo->current > pSliceInfo->win.ekey) { + doSetOperatorCompleted(pOperator); + break; } } else { - // store ts value as start, and calculate interp value when processing next block - doKeepLinearInfo(pSliceInfo, pBlock, i, true); - } - } else { // non-linear interpolation - if (i < pBlock->info.rows - 1) { - // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate - doKeepNextRows(pSliceInfo, pBlock, i + 1); - int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); - if (nextTs > pSliceInfo->current) { - while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); - pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, - pInterval->precision); - } - - if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); - break; - } - } else { - // ignore current row, and do nothing - } - } else { // it is the last row of current block - doKeepPrevRows(pSliceInfo, pBlock, i); + // ignore current row, and do nothing } + } else { // it is the last row of current block + doKeepPrevRows(pSliceInfo, pBlock, i); } } else { // ts > pSliceInfo->current // in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate doKeepNextRows(pSliceInfo, pBlock, i); + doKeepLinearInfo(pSliceInfo, pBlock, i); while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); @@ -2436,32 +2360,8 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { pResBlock->info.rows += 1; doKeepPrevRows(pSliceInfo, pBlock, i); - if (pSliceInfo->fillType == TSDB_FILL_LINEAR) { - pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - if (i < pBlock->info.rows - 1) { - doKeepLinearInfo(pSliceInfo, pBlock, i, false); - int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); - if (nextTs > pSliceInfo->current) { - while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); - pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, - pInterval->precision); - } - - if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); - break; - } - } - } else { // it is the last row of current block - // store ts value as start, and calculate interp value when processing next block - doKeepLinearInfo(pSliceInfo, pBlock, i, true); - } - } else { // non-linear interpolation - pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - } + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); } if (pSliceInfo->current > pSliceInfo->win.ekey) { From 6aca4b8a5cef4c71b4b2f30680f3e2c08fb2a424 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 8 Nov 2022 14:50:03 +0800 Subject: [PATCH 02/59] enh: interp fill linear ignore null values during calculation --- source/libs/executor/inc/tfill.h | 1 - source/libs/executor/src/timewindowoperator.c | 43 +++++++++++-------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index ed019be767..7e0866f545 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -36,7 +36,6 @@ typedef struct SFillColInfo { typedef struct SFillLinearInfo { SPoint start; SPoint end; - bool hasNull; int16_t type; int32_t bytes; } SFillLinearInfo; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index ef99e31f67..0950fcce2f 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1999,7 +1999,7 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, i); // null data should not be kept since it can not be used to perform interpolation - if (!colDataIsNull_s(pColInfoData, i)) { + if (!colDataIsNull_s(pColInfoData, rowIndex)) { if (pLinearInfo->start.key == INT64_MIN) { pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); memcpy(pLinearInfo->start.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); @@ -2017,7 +2017,8 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo } -static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock) { +static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, + SSDataBlock* pSrcBlock, int32_t index, bool beforeRange) { int32_t rows = pResBlock->info.rows; blockDataEnsureCapacity(pResBlock, rows + 1); // todo set the correct primary timestamp column @@ -2036,7 +2037,6 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; - // SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); switch (pSliceInfo->fillType) { case TSDB_FILL_NULL: { colDataAppendNULL(pDst, rows); @@ -2071,17 +2071,17 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp current.val = taosMemoryCalloc(pLinearInfo->bytes, 1); // before interp range, do not fill - if (start.key == INT64_MIN || end.key == INT64_MAX) { + SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot); + if (colDataIsNull_s(pSrc, index) || (!beforeRange && end.key == INT64_MAX)) { hasInterp = false; break; + } else if (start.key == INT64_MIN || (beforeRange && end.key == INT64_MAX)) { + //hasInterp = true; + return true; } - if (pLinearInfo->hasNull) { - colDataAppendNULL(pDst, rows); - } else { - taosGetLinearInterpolationVal(¤t, pLinearInfo->type, &start, &end, pLinearInfo->type); - colDataAppend(pDst, rows, (char*)current.val, false); - } + taosGetLinearInterpolationVal(¤t, pLinearInfo->type, &start, &end, pLinearInfo->type); + colDataAppend(pDst, rows, (char*)current.val, false); taosMemoryFree(current.val); break; @@ -2117,6 +2117,8 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp if (hasInterp) { pResBlock->info.rows += 1; } + + return hasInterp; } static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) { @@ -2192,7 +2194,6 @@ static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB linearInfo.end.key = INT64_MAX; linearInfo.start.val = taosMemoryCalloc(1, pColInfo->info.bytes); linearInfo.end.val = taosMemoryCalloc(1, pColInfo->info.bytes); - linearInfo.hasNull = false; linearInfo.type = pColInfo->info.type; linearInfo.bytes = pColInfo->info.bytes; taosArrayPush(pInfo->pLinearInfo, &linearInfo); @@ -2306,9 +2307,12 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); - pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, - pInterval->precision); + if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i, false) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { + break; + } else { + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + } } if (pSliceInfo->current > pSliceInfo->win.ekey) { @@ -2327,9 +2331,12 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doKeepLinearInfo(pSliceInfo, pBlock, i); while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); - pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i, true) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { + break; + } else { + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + } } // add current row if timestamp match @@ -2376,7 +2383,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { // except for fill(next), fill(linear) while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT && pSliceInfo->fillType != TSDB_FILL_LINEAR) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, NULL, 0, false); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); } From df3ae1341fe69192915a2ac8c30e3aab5a25c8a4 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 8 Nov 2022 14:50:03 +0800 Subject: [PATCH 03/59] add test cases --- tests/system-test/2-query/interp.py | 51 ++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index ce6b85c1cf..4db074011d 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -311,7 +311,7 @@ class TDTestCase: ## {. . .} tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") - tdSql.checkRows(12) + tdSql.checkRows(11) tdSql.checkData(0, 0, 5) tdSql.checkData(1, 0, 6) tdSql.checkData(2, 0, 7) @@ -354,7 +354,7 @@ class TDTestCase: ## ..{.} tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:13', '2020-02-01 00:00:17') every(1s) fill(linear)") - tdSql.checkRows(5) + tdSql.checkRows(3) tdSql.checkData(0, 0, 13) tdSql.checkData(1, 0, 14) tdSql.checkData(2, 0, 15) @@ -555,7 +555,7 @@ class TDTestCase: tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") - tdSql.checkRows(12) + tdSql.checkRows(11) tdSql.checkCols(2) tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') @@ -583,7 +583,7 @@ class TDTestCase: # multiple _irowts tdSql.query(f"select interp(c0),_irowts from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") - tdSql.checkRows(12) + tdSql.checkRows(11) tdSql.checkCols(2) tdSql.checkData(0, 1, '2020-02-01 00:00:05.000') @@ -599,7 +599,7 @@ class TDTestCase: tdSql.checkData(10, 1, '2020-02-01 00:00:15.000') tdSql.query(f"select _irowts, interp(c0), interp(c0), _irowts from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") - tdSql.checkRows(12) + tdSql.checkRows(11) tdSql.checkCols(4) cols = (0, 3) @@ -837,6 +837,47 @@ class TDTestCase: tdSql.checkData(0, 0, 15) tdSql.checkData(1, 0, 15) + # test fill linear + + ## | {. | | .} | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-11 00:00:05') every(1d) fill(linear)") + tdSql.checkRows(11) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 6) + tdSql.checkData(2, 0, 7) + tdSql.checkData(3, 0, 8) + tdSql.checkData(4, 0, 9) + tdSql.checkData(5, 0, 10) + tdSql.checkData(6, 0, 11) + tdSql.checkData(7, 0, 12) + tdSql.checkData(8, 0, 13) + tdSql.checkData(9, 0, 14) + tdSql.checkData(10, 0, 15) + + ## | . | {} | . | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-03 00:00:05', '2020-02-07 00:00:05') every(1d) fill(linear)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 7) + tdSql.checkData(1, 0, 8) + tdSql.checkData(2, 0, 9) + tdSql.checkData(3, 0, 10) + tdSql.checkData(4, 0, 11) + + ## | {. | } | . | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-01-31 00:00:05', '2020-02-05 00:00:05') every(1d) fill(linear)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 6) + tdSql.checkData(2, 0, 7) + tdSql.checkData(3, 0, 8) + tdSql.checkData(4, 0, 9) + + ## | . | { | .} | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(linear)") + tdSql.checkRows(2) + tdSql.checkData(0, 0, 14) + tdSql.checkData(1, 0, 15) + tdLog.printNoPrefix("==========step10:test multi-interp cases") tdSql.query(f"select interp(c0),interp(c1),interp(c2),interp(c3) from {dbname}.{tbname} range('2020-02-09 00:00:05', '2020-02-13 00:00:05') every(1d) fill(null)") tdSql.checkRows(5) From f19f6201366badc90e3ddbdb070b1331b5dd23fb Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 8 Nov 2022 18:25:38 +0800 Subject: [PATCH 04/59] interp fill linear do not output NULL value --- source/libs/executor/src/timewindowoperator.c | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 0950fcce2f..75d7010eb1 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2018,7 +2018,7 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo } static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, - SSDataBlock* pSrcBlock, int32_t index, bool beforeRange) { + SSDataBlock* pSrcBlock, int32_t index, bool beforeTs) { int32_t rows = pResBlock->info.rows; blockDataEnsureCapacity(pResBlock, rows + 1); // todo set the correct primary timestamp column @@ -2072,10 +2072,10 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp // before interp range, do not fill SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot); - if (colDataIsNull_s(pSrc, index) || (!beforeRange && end.key == INT64_MAX)) { + if (colDataIsNull_s(pSrc, index) || (!beforeTs && end.key == INT64_MAX)) { hasInterp = false; break; - } else if (start.key == INT64_MIN || (beforeRange && end.key == INT64_MAX)) { + } else if (start.key == INT64_MIN || (beforeTs && end.key == INT64_MAX)) { //hasInterp = true; return true; } @@ -2264,6 +2264,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { if (ts == pSliceInfo->current) { blockDataEnsureCapacity(pResBlock, pResBlock->info.rows + 1); + bool isNullRow = false; for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; @@ -2277,8 +2278,12 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); if (colDataIsNull_s(pSrc, i)) { - colDataAppendNULL(pDst, pResBlock->info.rows); - continue; + if (pSliceInfo->fillType != TSDB_FILL_LINEAR) { + colDataAppendNULL(pDst, pResBlock->info.rows); + continue; + } else { + isNullRow = true; + } } char* v = colDataGetData(pSrc, i); @@ -2286,7 +2291,9 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } } - pResBlock->info.rows += 1; + if (!isNullRow) { + pResBlock->info.rows += 1; + } doKeepPrevRows(pSliceInfo, pBlock, i); doKeepLinearInfo(pSliceInfo, pBlock, i); @@ -2342,6 +2349,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { // add current row if timestamp match if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) { blockDataEnsureCapacity(pResBlock, pResBlock->info.rows + 1); + bool isNullRow = false; for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; @@ -2355,8 +2363,12 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); if (colDataIsNull_s(pSrc, i)) { - colDataAppendNULL(pDst, pResBlock->info.rows); - continue; + if (pSliceInfo->fillType != TSDB_FILL_LINEAR) { + colDataAppendNULL(pDst, pResBlock->info.rows); + continue; + } else { + isNullRow = true; + } } char* v = colDataGetData(pSrc, i); @@ -2364,7 +2376,9 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } } - pResBlock->info.rows += 1; + if (!isNullRow) { + pResBlock->info.rows += 1; + } doKeepPrevRows(pSliceInfo, pBlock, i); pSliceInfo->current = From 7500f3e27eb52874b278703b6430c3dd72a3c20b Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 8 Nov 2022 18:26:07 +0800 Subject: [PATCH 05/59] add test case --- tests/system-test/2-query/interp.py | 256 +++++++++++++++++++++++++++- 1 file changed, 253 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index 4db074011d..fa5c35f782 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -18,6 +18,7 @@ class TDTestCase: def run(self): dbname = "db" tbname = "tb" + tbname1 = "tb1" stbname = "stb" ctbname1 = "ctb1" ctbname2 = "ctb2" @@ -878,7 +879,256 @@ class TDTestCase: tdSql.checkData(0, 0, 14) tdSql.checkData(1, 0, 15) - tdLog.printNoPrefix("==========step10:test multi-interp cases") + + tdLog.printNoPrefix("==========step10:test interp with null data") + tdSql.execute( + f'''create table if not exists {dbname}.{tbname1} + (ts timestamp, c0 int, c1 int) + ''' + ) + + + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:00', 0, NULL)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:05', NULL, NULL)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:10', 10, 10)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:15', NULL, NULL)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:20', 20, NULL)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:25', NULL, NULL)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:30', 30, 30)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:35', 35, NULL)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:40', 40, 40)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:45', NULL, 45)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:50', 50, NULL)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:55', NULL, NULL)") + tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:01:00', 55, 60)") + + # check c0 + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:00') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:03') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:05') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:08') every(1s) fill(linear)") + tdSql.checkRows(9) + tdSql.checkData(0, 0, 0) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 2) + tdSql.checkData(3, 0, 3) + tdSql.checkData(4, 0, 4) + tdSql.checkData(5, 0, 5) + tdSql.checkData(6, 0, 6) + tdSql.checkData(7, 0, 7) + tdSql.checkData(8, 0, 8) + + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:01', '2020-02-02 00:00:03') every(1s) fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:03', '2020-02-02 00:00:08') every(1s) fill(linear)") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 4) + tdSql.checkData(2, 0, 5) + tdSql.checkData(3, 0, 6) + tdSql.checkData(4, 0, 7) + tdSql.checkData(5, 0, 8) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:05', '2020-02-02 00:00:10') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:05', '2020-02-02 00:00:15') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:05', '2020-02-02 00:00:18') every(1s) fill(linear)") + tdSql.checkRows(9) + tdSql.checkData(0, 0, 10) + tdSql.checkData(1, 0, 11) + tdSql.checkData(2, 0, 12) + tdSql.checkData(3, 0, 13) + tdSql.checkData(4, 0, 14) + tdSql.checkData(5, 0, 15) + tdSql.checkData(6, 0, 16) + tdSql.checkData(7, 0, 17) + tdSql.checkData(8, 0, 18) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:05', '2020-02-02 00:00:20') every(1s) fill(linear)") + tdSql.checkRows(11) + tdSql.checkData(0, 0, 10) + tdSql.checkData(1, 0, 11) + tdSql.checkData(2, 0, 12) + tdSql.checkData(3, 0, 13) + tdSql.checkData(4, 0, 14) + tdSql.checkData(5, 0, 15) + tdSql.checkData(6, 0, 16) + tdSql.checkData(7, 0, 17) + tdSql.checkData(8, 0, 18) + tdSql.checkData(9, 0, 19) + tdSql.checkData(10, 0, 20) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:09', '2020-02-02 00:00:11') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:10', '2020-02-02 00:00:15') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:12', '2020-02-02 00:00:13') every(1s) fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:12', '2020-02-02 00:00:15') every(1s) fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:12', '2020-02-02 00:00:18') every(1s) fill(linear)") + tdSql.checkRows(7) + tdSql.checkData(0, 0, 12) + tdSql.checkData(1, 0, 13) + tdSql.checkData(2, 0, 14) + tdSql.checkData(3, 0, 15) + tdSql.checkData(4, 0, 16) + tdSql.checkData(5, 0, 17) + tdSql.checkData(6, 0, 18) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:30', '2020-02-02 00:00:40') every(1s) fill(linear)") + tdSql.checkRows(11) + tdSql.checkData(0, 0, 30) + tdSql.checkData(10, 0, 40) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:25', '2020-02-02 00:00:45') every(1s) fill(linear)") + tdSql.checkRows(11) + tdSql.checkData(0, 0, 30) + tdSql.checkData(10, 0, 40) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:25', '2020-02-02 00:00:45') every(1s) fill(linear)") + tdSql.checkRows(11) + tdSql.checkData(0, 0, 30) + tdSql.checkData(10, 0, 40) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:20', '2020-02-02 00:00:40') every(1s) fill(linear)") + tdSql.checkRows(21) + tdSql.checkData(0, 0, 20) + tdSql.checkData(20, 0, 40) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:30', '2020-02-02 00:00:50') every(1s) fill(linear)") + tdSql.checkRows(21) + tdSql.checkData(0, 0, 30) + tdSql.checkData(20, 0, 50) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:20', '2020-02-02 00:00:50') every(1s) fill(linear)") + tdSql.checkRows(31) + tdSql.checkData(0, 0, 20) + tdSql.checkData(30, 0, 50) + + # check c1 + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:05') every(1s) fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:05') every(1s) fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:08') every(1s) fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:10') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:15') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:20') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:25') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:30') every(1s) fill(linear)") + tdSql.checkRows(21) + tdSql.checkData(0, 0, 10) + tdSql.checkData(1, 0, 11) + tdSql.checkData(2, 0, 12) + tdSql.checkData(3, 0, 13) + tdSql.checkData(4, 0, 14) + tdSql.checkData(5, 0, 15) + tdSql.checkData(6, 0, 16) + tdSql.checkData(7, 0, 17) + tdSql.checkData(8, 0, 18) + tdSql.checkData(9, 0, 19) + tdSql.checkData(10, 0, 20) + tdSql.checkData(11, 0, 21) + tdSql.checkData(12, 0, 22) + tdSql.checkData(13, 0, 23) + tdSql.checkData(14, 0, 24) + tdSql.checkData(15, 0, 25) + tdSql.checkData(16, 0, 26) + tdSql.checkData(17, 0, 27) + tdSql.checkData(18, 0, 28) + tdSql.checkData(19, 0, 29) + tdSql.checkData(20, 0, 30) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:35') every(1s) fill(linear)") + tdSql.checkRows(21) + tdSql.checkData(0, 0, 10) + tdSql.checkData(20, 0, 30) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:40') every(1s) fill(linear)") + tdSql.checkRows(31) + tdSql.checkData(0, 0, 10) + tdSql.checkData(30, 0, 40) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:45') every(1s) fill(linear)") + tdSql.checkRows(36) + tdSql.checkData(0, 0, 10) + tdSql.checkData(35, 0, 45) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:50') every(1s) fill(linear)") + tdSql.checkRows(36) + tdSql.checkData(0, 0, 10) + tdSql.checkData(35, 0, 45) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:55') every(1s) fill(linear)") + tdSql.checkRows(36) + tdSql.checkData(0, 0, 10) + tdSql.checkData(35, 0, 45) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(linear)") + tdSql.checkRows(51) + tdSql.checkData(0, 0, 10) + tdSql.checkData(50, 0, 60) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:40', '2020-02-02 00:00:45') every(1s) fill(linear)") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 40) + tdSql.checkData(5, 0, 45) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:35', '2020-02-02 00:00:50') every(1s) fill(linear)") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 40) + tdSql.checkData(5, 0, 45) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:35', '2020-02-02 00:00:55') every(1s) fill(linear)") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 40) + tdSql.checkData(5, 0, 45) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:30', '2020-02-02 00:01:00') every(1s) fill(linear)") + tdSql.checkRows(31) + tdSql.checkData(0, 0, 30) + tdSql.checkData(30, 0, 60) + + tdLog.printNoPrefix("==========step11:test multi-interp cases") tdSql.query(f"select interp(c0),interp(c1),interp(c2),interp(c3) from {dbname}.{tbname} range('2020-02-09 00:00:05', '2020-02-13 00:00:05') every(1d) fill(null)") tdSql.checkRows(5) tdSql.checkCols(4) @@ -932,7 +1182,7 @@ class TDTestCase: for i in range (tdSql.queryCols): tdSql.checkData(0, i, 13) - tdLog.printNoPrefix("==========step11:test error cases") + tdLog.printNoPrefix("==========step12:test error cases") tdSql.error(f"select interp(c0) from {dbname}.{tbname}") tdSql.error(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05')") @@ -955,7 +1205,7 @@ class TDTestCase: tdSql.error(f"select interp('abcd') from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") tdSql.error(f"select interp('中文字符') from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") - tdLog.printNoPrefix("==========step12:stable cases") + tdLog.printNoPrefix("==========step13:stable cases") #tdSql.query(f"select interp(c0) from {dbname}.{stbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(null)") #tdSql.checkRows(13) From b902ab0d6184f193bd384c6037b783f876281cf3 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 8 Nov 2022 18:48:22 +0800 Subject: [PATCH 06/59] refactor code --- source/libs/executor/src/timewindowoperator.c | 97 +++++++------------ 1 file changed, 36 insertions(+), 61 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 75d7010eb1..461a4fbe43 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2121,6 +2121,40 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp return hasInterp; } +static void addCurrentRowToResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, + SSDataBlock* pSrcBlock, int32_t index) { + blockDataEnsureCapacity(pResBlock, pResBlock->info.rows + 1); + for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) { + SExprInfo* pExprInfo = &pExprSup->pExprInfo[j]; + + int32_t dstSlot = pExprInfo->base.resSchema.slotId; + SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); + + if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { + colDataAppend(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false); + } else { + int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot); + + if (colDataIsNull_s(pSrc, index)) { + if (pSliceInfo->fillType != TSDB_FILL_LINEAR) { + colDataAppendNULL(pDst, pResBlock->info.rows); + continue; + } else { + return; + } + } + + char* v = colDataGetData(pSrc, index); + colDataAppend(pDst, pResBlock->info.rows, v, false); + } + } + + pResBlock->info.rows += 1; + return; +} + + static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) { if (pInfo->pPrevRow != NULL) { return TSDB_CODE_SUCCESS; @@ -2263,37 +2297,8 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (ts == pSliceInfo->current) { - blockDataEnsureCapacity(pResBlock, pResBlock->info.rows + 1); - bool isNullRow = false; - for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { - SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; + addCurrentRowToResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i); - int32_t dstSlot = pExprInfo->base.resSchema.slotId; - SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); - - if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { - colDataAppend(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false); - } else { - int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; - SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); - - if (colDataIsNull_s(pSrc, i)) { - if (pSliceInfo->fillType != TSDB_FILL_LINEAR) { - colDataAppendNULL(pDst, pResBlock->info.rows); - continue; - } else { - isNullRow = true; - } - } - - char* v = colDataGetData(pSrc, i); - colDataAppend(pDst, pResBlock->info.rows, v, false); - } - } - - if (!isNullRow) { - pResBlock->info.rows += 1; - } doKeepPrevRows(pSliceInfo, pBlock, i); doKeepLinearInfo(pSliceInfo, pBlock, i); @@ -2348,37 +2353,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { // add current row if timestamp match if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) { - blockDataEnsureCapacity(pResBlock, pResBlock->info.rows + 1); - bool isNullRow = false; - for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { - SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; - - int32_t dstSlot = pExprInfo->base.resSchema.slotId; - SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); - - if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { - colDataAppend(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false); - } else { - int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; - SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); - - if (colDataIsNull_s(pSrc, i)) { - if (pSliceInfo->fillType != TSDB_FILL_LINEAR) { - colDataAppendNULL(pDst, pResBlock->info.rows); - continue; - } else { - isNullRow = true; - } - } - - char* v = colDataGetData(pSrc, i); - colDataAppend(pDst, pResBlock->info.rows, v, false); - } - } - - if (!isNullRow) { - pResBlock->info.rows += 1; - } + addCurrentRowToResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i); doKeepPrevRows(pSliceInfo, pBlock, i); pSliceInfo->current = From f6de1a8634444e624803911fef5e7c643a252809 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Wed, 9 Nov 2022 18:42:01 +0800 Subject: [PATCH 07/59] enh crash_gen for 3.0 about [stream processing] , [data subscription] ,[delete data] --- tests/pytest/crash_gen.sh | 2 +- tests/pytest/crash_gen/crash_gen_main.py | 748 ++++++++++++++++++++++- tests/pytest/crash_gen/shared/db.py | 58 +- 3 files changed, 774 insertions(+), 34 deletions(-) diff --git a/tests/pytest/crash_gen.sh b/tests/pytest/crash_gen.sh index 539314dea4..cc2941a52a 100755 --- a/tests/pytest/crash_gen.sh +++ b/tests/pytest/crash_gen.sh @@ -45,7 +45,7 @@ fi # Now getting ready to execute Python # The following is the default of our standard dev env (Ubuntu 20.04), modify/adjust at your own risk -PYTHON_EXEC=python3.8 +PYTHON_EXEC=python3 # First we need to set up a path for Python to find our own TAOS modules, so that "import" can work. # export PYTHONPATH=$(pwd)/../../src/connector/python:$(pwd) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 600c64b8e6..757ddf4af8 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -37,6 +37,7 @@ import requests # from guppy import hpy import gc import taos +from taos.tmq import * from .shared.types import TdColumns, TdTags @@ -254,7 +255,7 @@ class WorkerThread: class ThreadCoordinator: - WORKER_THREAD_TIMEOUT = 120 # Normal: 120 + WORKER_THREAD_TIMEOUT = 1200 # Normal: 120 def __init__(self, pool: ThreadPool, dbManager: DbManager): self._curStep = -1 # first step is 0 @@ -674,9 +675,13 @@ class AnyState: # only "under normal circumstances", as we may override it with the -b option CAN_DROP_DB = 2 CAN_CREATE_FIXED_SUPER_TABLE = 3 + CAN_CREATE_STREAM = 3 # super table must exists + CAN_CREATE_TOPIC = 3 # super table must exists + CAN_CREATE_CONSUMERS = 3 CAN_DROP_FIXED_SUPER_TABLE = 4 CAN_ADD_DATA = 5 CAN_READ_DATA = 6 + CAN_DELETE_DATA = 6 def __init__(self): self._info = self.getInfo() @@ -727,12 +732,24 @@ class AnyState: return False return self._info[self.CAN_DROP_FIXED_SUPER_TABLE] + def canCreateTopic(self): + return self._info[self.CAN_CREATE_TOPIC] + + def canCreateConsumers(self): + return self._info[self.CAN_CREATE_CONSUMERS] + + def canCreateStream(self): + return self._info[self.CAN_CREATE_STREAM] + def canAddData(self): return self._info[self.CAN_ADD_DATA] def canReadData(self): return self._info[self.CAN_READ_DATA] + def canDeleteData(self): + return self._info[self.CAN_DELETE_DATA] + def assertAtMostOneSuccess(self, tasks, cls): sCnt = 0 for task in tasks: @@ -921,7 +938,7 @@ class StateMechine: except taos.error.ProgrammingError as err: Logging.error("Failed to initialized state machine, cannot find current state: {}".format(err)) traceback.print_stack() - raise # re-throw + pass # re-throw # TODO: seems no lnoger used, remove? def getCurrentState(self): @@ -974,14 +991,21 @@ class StateMechine: # did not do this when openning connection, and this is NOT the worker # thread, which does this on their own dbc.use(dbName) + if not dbc.hasTables(): # no tables + Logging.debug("[STT] DB_ONLY found, between {} and {}".format(ts, time.time())) return StateDbOnly() # For sure we have tables, which means we must have the super table. # TODO: are we sure? + sTable = self._db.getFixedSuperTable() - if sTable.hasRegTables(dbc): # no regular tables + + + if sTable.hasRegTables(dbc): # no regular tables + # print("debug=====*\n"*100) Logging.debug("[STT] SUPER_TABLE_ONLY found, between {} and {}".format(ts, time.time())) + return StateSuperTableOnly() else: # has actual tables Logging.debug("[STT] HAS_DATA found, between {} and {}".format(ts, time.time())) @@ -1109,6 +1133,7 @@ class Database: return "fs_table" def getFixedSuperTable(self) -> TdSuperTable: + return TdSuperTable(self.getFixedSuperTableName(), self.getName()) # We aim to create a starting time tick, such that, whenever we run our test here once @@ -1342,7 +1367,13 @@ class Task(): 0x2603, # Table does not exist, replaced by 2662 below 0x260d, # Tags number not matched 0x2662, # Table does not exist #TODO: what about 2603 above? - + 0x032C, # Object is creating + 0x032D, # Object is dropping + 0x03D3, # Conflict transaction not completed + 0x0707, # Query not ready , it always occur at replica 3 + 0x707, # Query not ready + 0x396, # Database in creating status + 0x386, # Database in droping status 1000 # REST catch-all error @@ -1638,9 +1669,12 @@ class TaskCreateDb(StateTransitionTask): # numReplica = Dice.throw(Settings.getConfig().max_replicas) + 1 # 1,2 ... N numReplica = Config.getConfig().num_replicas # fixed, always repStr = "replica {}".format(numReplica) - updatePostfix = "update 1" if Config.getConfig().verify_data else "" # allow update only when "verify data" is active + updatePostfix = "" if Config.getConfig().verify_data else "" # allow update only when "verify data" is active , 3.0 version default is update 1 + vg_nums = random.randint(1,8) + cache_model = Dice.choice(['none' , 'last_row' , 'last_value' , 'both']) + buffer = random.randint(3,128) dbName = self._db.getName() - self.execWtSql(wt, "create database {} {} {} ".format(dbName, repStr, updatePostfix ) ) + self.execWtSql(wt, "create database {} {} {} vgroups {} cachemodel '{}' buffer {} ".format(dbName, repStr, updatePostfix, vg_nums, cache_model,buffer ) ) if dbName == "db_0" and Config.getConfig().use_shadow_db: self.execWtSql(wt, "create database {} {} {} ".format("db_s", repStr, updatePostfix ) ) @@ -1654,9 +1688,237 @@ class TaskDropDb(StateTransitionTask): return state.canDropDb() def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + + # drop topics before drop db + + if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): + + self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) + self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),self._db.getFixedSuperTableName) + self.execWtSql(wt, "drop database {}".format(self._db.getName())) + Logging.debug("[OPS] database dropped at {}".format(time.time())) + +''' +# Streams will generator TD-20237 （it will crash taosd , start this task when this issue fixed ） + +class TaskCreateStream(StateTransitionTask): + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canCreateStream() + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + dbname = self._db.getName() + + sub_stream_name = dbname+ '_sub_stream' + sub_stream_tb_name = 'stream_tb_sub' + super_stream_name = dbname+ '_super_stream' + super_stream_tb_name = 'stream_tb_super' + if not self._db.exists(wt.getDbConn()): + Logging.debug("Skipping task, no DB yet") + return + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + + # create stream + + # CREATE STREAM avg_vol_s INTO avg_vol AS SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); + + stbname =sTable.getName() + sub_tables = sTable.getRegTables(wt.getDbConn()) + aggExpr = Dice.choice([ + 'count(*)', + 'avg(speed)', + # 'twa(speed)', # TODO: this one REQUIRES a where statement, not reasonable + 'sum(speed)', + 'stddev(speed)', + # SELECTOR functions + 'min(speed)', + 'max(speed)', + 'first(speed)', + 'last(speed)', + 'apercentile(speed, 10)', # TODO: TD-1316 + 'last_row(*)', # TODO: commented out per TD-3231, we should re-create + # Transformation Functions + 'twa(speed)' + + ]) # TODO: add more from 'top' + + if sub_tables: + + if sub_tables: # if not empty + sub_tbname = sub_tables[0] + # create stream with query above sub_table + stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(sub_stream_name,dbname,sub_stream_tb_name ,aggExpr,dbname,sub_tbname) + try: + self.execWtSql(wt, stream_sql) + Logging.debug("[OPS] stream is creating at {}".format(time.time())) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x03f0]: # stream already exists + # stream need drop before drop table + pass + + else: + pass + + else: + stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(super_stream_name,dbname,super_stream_tb_name,aggExpr, dbname,stbname) + + try: + self.execWtSql(wt, stream_sql) + Logging.debug("[OPS] stream is creating at {}".format(time.time())) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x03f0]: # stream already exists + # stream need drop before drop table + pass +''' + +class TaskCreateTopic(StateTransitionTask): + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canCreateTopic() + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + dbname = self._db.getName() + + sub_topic_name = dbname+ '_sub_topic' + super_topic_name = dbname+ '_super_topic' + stable_topic = dbname+ '_stable_topic' + db_topic = 'database_' + dbname+ '_topics' + if not self._db.exists(wt.getDbConn()): + Logging.debug("Skipping task, no DB yet") + return + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + + # create topic + + # create topic if not exists topic_ctb_column as select ts, c1, c2, c3 from stb1; + + stbname =sTable.getName() + sub_tables = sTable.getRegTables(wt.getDbConn()) + scalarExpr = Dice.choice([ '*','speed','color', + 'abs(speed)', + 'acos(speed)', + 'asin(speed)', + 'atan(speed)', + 'ceil(speed)', + 'cos(speed)', + 'cos(speed)', + 'floor(speed)', + 'log(speed,2)', + 'pow(speed,2)', + 'round(speed)', + 'sin(speed)', + 'sqrt(speed)', + 'char_length(color)', + 'concat(color,color)', + 'concat_ws(" ", color,color," ")', + 'length(color)', + 'lower(color)', + 'ltrim(color)', + 'substr(color , 2)', + 'upper(color)', + 'cast(speed as double)', + 'cast(ts as bigint)', + + ]) # TODO: add more from 'top' + if Dice.throw(3)==0: + + if sub_tables: + + if sub_tables: # if not empty + sub_tbname = sub_tables[0] + # create stream with query above sub_table + topic_sql = 'create topic {} as select {} FROM {}.{} ; '.format(sub_topic_name,scalarExpr,dbname,sub_tbname) + try: + self.execWtSql(wt, "use {}".format(dbname)) + self.execWtSql(wt, topic_sql) + Logging.debug("[OPS] topic is creating at {}".format(time.time())) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x03f0]: # topic already exists + # topic need drop before drop table + pass + + else: + pass + + else: + topic_sql = 'create topic {} as select {} FROM {}.{} '.format(super_topic_name,scalarExpr, dbname,stbname) + try: + self.execWtSql(wt, "use {}".format(dbname)) + self.execWtSql(wt, topic_sql) + Logging.debug("[OPS] subquery topic is creating at {}".format(time.time())) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x03f0]: # topic already exists + # topic need drop before drop table + pass + elif Dice.throw(3)==1: + topic_sql = 'create topic {} AS STABLE {}.{} '.format(stable_topic,dbname,stbname) + try: + self.execWtSql(wt, "use {}".format(dbname)) + self.execWtSql(wt, topic_sql) + Logging.debug("[OPS] stable topic is creating at {}".format(time.time())) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x03f0]: # topic already exists + # topic need drop before drop table + pass + elif Dice.throw(3)==2: + topic_sql = 'create topic {} AS DATABASE {} '.format(db_topic,dbname) + try: + self.execWtSql(wt, "use {}".format(dbname)) + self.execWtSql(wt, topic_sql) + Logging.debug("[OPS] db topic is creating at {}".format(time.time())) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x03f0]: # topic already exists + # topic need drop before drop table + pass + else: + pass + + +class TaskCreateConsumers(StateTransitionTask): + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canCreateConsumers() + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + dbname = self._db.getName() + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + + # create Consumers + if Dice.throw(50)==0: # because subscribe is cost so much time , Reduce frequency of this task + if sTable.hasTopics(wt.getDbConn()): + sTable.createConsumer(wt.getDbConn(),random.randint(1,10)) + + class TaskCreateSuperTable(StateTransitionTask): @classmethod def getEndState(cls): @@ -1673,7 +1935,7 @@ class TaskCreateSuperTable(StateTransitionTask): sTable = self._db.getFixedSuperTable() # type: TdSuperTable # wt.execSql("use db") # should always be in place - + sTable.create(wt.getDbConn(), {'ts': TdDataType.TIMESTAMP, 'speed': TdDataType.INT, 'color': TdDataType.BINARY16}, { 'b': TdDataType.BINARY200, 'f': TdDataType.FLOAT}, @@ -1688,15 +1950,42 @@ class TdSuperTable: def __init__(self, stName, dbName): self._stName = stName self._dbName = dbName + self._consumerLists = {} + self._ConsumerInsts = [] def getName(self): return self._stName + def drop(self, dbc, skipCheck = False): dbName = self._dbName if self.exists(dbc) : # if myself exists - fullTableName = dbName + '.' + self._stName - dbc.execute("DROP TABLE {}".format(fullTableName)) + fullTableName = dbName + '.' + self._stName + if self.hasStreams(dbc): + self.dropStreams(dbc) + self.dropStreamTables(dbc) + if self.hasTopics(dbc): + self.dropTopics(dbName,None) + self.dropTopics(dbName,self._stName) + try: + dbc.execute("DROP TABLE {}".format(fullTableName)) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [1011,0x3F3,0x03f3,0x2662]: # table doesn't exist + pass + # # stream need drop before drop table + # for stream in self.getStreamName(): + # drop_stream_sql = 'drop stream {}'.format(stream) + # try: + # dbc.execute(drop_stream_sql) + # except taos.error.ProgrammingError as err: + # # correcting for strange error number scheme + # errno3 = Helper.convertErrno(err.errno) + # if errno3 in [1011,0x3F3,0x03f3,0x2662,0x03f1]: # stream not exists + # pass + # dbc.execute("DROP TABLE {}".format(fullTableName)) + # pass + else: if not skipCheck: raise CrashGenError("Cannot drop non-existant super table: {}".format(self._stName)) @@ -1711,10 +2000,24 @@ class TdSuperTable: dbName = self._dbName dbc.execute("USE " + dbName) - fullTableName = dbName + '.' + self._stName + fullTableName = dbName + '.' + self._stName + if dbc.existsSuperTable(self._stName): if dropIfExists: + if self.hasStreams(dbc): + self.dropStreams(dbc) + self.dropStreamTables(dbc) + + # drop topics before drop stables + if self.hasTopics(dbc): + self.dropTopics(dbc,self._dbName,None) + self.dropTopics(dbc,self._dbName,self._stName ) + + dbc.execute("DROP TABLE {}".format(fullTableName)) + + pass + # dbc.execute("DROP TABLE {}".format(fullTableName)) else: # error raise CrashGenError("Cannot create super table, already exists: {}".format(self._stName)) @@ -1728,12 +2031,52 @@ class TdSuperTable: ) else: sql += " TAGS (dummy int) " - dbc.execute(sql) + dbc.execute(sql) + + def createConsumer(self, dbc: DbConn , Consumer_nums): + + def generateConsumer(current_topic_list): + conf = TaosTmqConf() + conf.set("group.id", "tg2") + conf.set("td.connect.user", "root") + conf.set("td.connect.pass", "taosdata") + conf.set("enable.auto.commit", "true") + def tmq_commit_cb_print(tmq, resp, offset, param=None): + print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}") + conf.set_auto_commit_cb(tmq_commit_cb_print, None) + consumer = conf.new_consumer() + topic_list = TaosTmqList() + for topic in current_topic_list: + topic_list.append(topic) + consumer.subscribe(topic_list) + time.sleep(5) # consumer work only 5 sec ,and then it will exit + try: + consumer.unsubscribe() + except TmqError as e : + pass + return + + # mulit Consumer + current_topic_list = self.getTopicLists(dbc) + for i in range(Consumer_nums): + consumer_inst = threading.Thread(target=generateConsumer, args=(current_topic_list,)) + self._ConsumerInsts.append(consumer_inst) + + for ConsumerInst in self._ConsumerInsts: + ConsumerInst.start() + for ConsumerInst in self._ConsumerInsts: + ConsumerInst.join() + + def getTopicLists(self, dbc: DbConn): + dbc.query("show topics ") + topics = dbc.getQueryResult() + topicLists = [v[0] for v in topics] + return topicLists def getRegTables(self, dbc: DbConn): dbName = self._dbName try: - dbc.query("select TBNAME from {}.{}".format(dbName, self._stName)) # TODO: analyze result set later + dbc.query("select distinct TBNAME from {}.{}".format(dbName, self._stName)) # TODO: analyze result set later except taos.error.ProgrammingError as err: errno2 = Helper.convertErrno(err.errno) Logging.debug("[=] Failed to get tables from super table: errno=0x{:X}, msg: {}".format(errno2, err)) @@ -1743,7 +2086,73 @@ class TdSuperTable: return [v[0] for v in qr] # list transformation, ref: https://stackoverflow.com/questions/643823/python-list-transformation def hasRegTables(self, dbc: DbConn): - return dbc.query("SELECT * FROM {}.{}".format(self._dbName, self._stName)) > 0 + + if dbc.existsSuperTable(self._stName): + + return dbc.query("SELECT * FROM {}.{}".format(self._dbName, self._stName)) > 0 + else: + return False + + def hasStreamTables(self,dbc: DbConn): + + return dbc.query("show {}.stables like 'stream_tb%'".format(self._dbName)) > 0 + + def hasStreams(self,dbc: DbConn): + return dbc.query("show streams") > 0 + + def hasTopics(self,dbc: DbConn): + + return dbc.query("show topics") > 0 + + def dropTopics(self,dbc: DbConn , dbname=None,stb_name=None): + dbc.query("show topics ") + topics = dbc.getQueryResult() + + if dbname !=None and stb_name == None : + + for topic in topics: + if dbname in topic[0] and topic[0].startswith("database"): + try: + dbc.execute('drop topic {}'.format(topic[0])) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x03EB]: # Topic subscribed cannot be dropped + pass + # for subsript in subscriptions: + + else: + pass + + pass + return True + elif dbname !=None and stb_name!= None: + for topic in topics: + if topic[0].startswith(self._dbName) and topic[0].endswith('topic'): + dbc.execute('drop topic {}'.format(topic[0])) + return True + else: + return True + pass + + def dropStreams(self,dbc:DbConn): + dbc.query("show streams ") + Streams = dbc.getQueryResult() + for Stream in Streams: + if Stream[0].startswith(self._dbName): + dbc.execute('drop stream {}'.format(Stream[0])) + + return not dbc.query("show streams ") > 0 + + def dropStreamTables(self, dbc: DbConn): + dbc.query("show {}.stables like 'stream_tb%'".format(self._dbName)) + + StreamTables = dbc.getQueryResult() + + for StreamTable in StreamTables: + if self.dropStreams(dbc): + dbc.execute('drop table {}.{}'.format(self._dbName,StreamTable[0])) + + return not dbc.query("show {}.stables like 'stream_tb%'".format(self._dbName)) def ensureRegTable(self, task: Optional[Task], dbc: DbConn, regTableName: str): ''' @@ -1838,10 +2247,46 @@ class TdSuperTable: # Run the query against the regular table first doAggr = (Dice.throw(2) == 0) # 1 in 2 chance if not doAggr: # don't do aggregate query, just simple one + commonExpr = Dice.choice([ + '*', + 'abs(speed)', + 'acos(speed)', + 'asin(speed)', + 'atan(speed)', + 'ceil(speed)', + 'cos(speed)', + 'cos(speed)', + 'floor(speed)', + 'log(speed,2)', + 'pow(speed,2)', + 'round(speed)', + 'sin(speed)', + 'sqrt(speed)', + 'char_length(color)', + 'concat(color,color)', + 'concat_ws(" ", color,color," ")', + 'length(color)', + 'lower(color)', + 'ltrim(color)', + 'substr(color , 2)', + 'upper(color)', + 'cast(speed as double)', + 'cast(ts as bigint)', + # 'TO_ISO8601(color)', + # 'TO_UNIXTIMESTAMP(ts)', + 'now()', + 'timediff(ts,now)', + 'timezone()', + 'TIMETRUNCATE(ts,1s)', + 'TIMEZONE()', + 'TODAY()', + 'distinct(color)' + ] + ) ret.append(SqlQuery( # reg table - "select {} from {}.{}".format('*', self._dbName, rTbName))) + "select {} from {}.{}".format(commonExpr, self._dbName, rTbName))) ret.append(SqlQuery( # super table - "select {} from {}.{}".format('*', self._dbName, self.getName()))) + "select {} from {}.{}".format(commonExpr, self._dbName, self.getName()))) else: # Aggregate query aggExpr = Dice.choice([ 'count(*)', @@ -1857,17 +2302,34 @@ class TdSuperTable: 'top(speed, 50)', # TODO: not supported? 'bottom(speed, 50)', # TODO: not supported? 'apercentile(speed, 10)', # TODO: TD-1316 - # 'last_row(speed)', # TODO: commented out per TD-3231, we should re-create + 'last_row(*)', # TODO: commented out per TD-3231, we should re-create # Transformation Functions # 'diff(speed)', # TODO: no supported?! - 'spread(speed)' + 'spread(speed)', + 'elapsed(ts)', + 'mode(speed)', + 'bottom(speed,1)', + 'top(speed,1)', + 'tail(speed,1)', + 'unique(color)', + 'csum(speed)', + 'DERIVATIVE(speed,1s,1)', + 'diff(speed,1)', + 'irate(speed)', + 'mavg(speed,3)', + 'sample(speed,5)', + 'STATECOUNT(speed,"LT",1)', + 'STATEDURATION(speed,"LT",1)', + 'twa(speed)' + ]) # TODO: add more from 'top' # if aggExpr not in ['stddev(speed)']: # STDDEV not valid for super tables?! (Done in TD-1049) sql = "select {} from {}.{}".format(aggExpr, self._dbName, self.getName()) if Dice.throw(3) == 0: # 1 in X chance - sql = sql + ' GROUP BY color' + partion_expr = Dice.choice(['color','tbname']) + sql = sql + ' partition BY ' + partion_expr + ' order by ' + partion_expr Progress.emit(Progress.QUERY_GROUP_BY) # Logging.info("Executing GROUP-BY query: " + sql) ret.append(SqlQuery(sql)) @@ -1965,15 +2427,12 @@ class TaskDropSuperTable(StateTransitionTask): for i in tblSeq: regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) try: + self.execWtSql(wt, "drop table {}.{}". format(self._db.getName(), regTableName)) # nRows always 0, like MySQL except taos.error.ProgrammingError as err: - # correcting for strange error number scheme - errno2 = Helper.convertErrno(err.errno) - if (errno2 in [0x362]): # mnode invalid table name - isSuccess = False - Logging.debug("[DB] Acceptable error when dropping a table") - continue # try to delete next regular table + pass + if (not tickOutput): tickOutput = True # Print only one time @@ -1984,7 +2443,28 @@ class TaskDropSuperTable(StateTransitionTask): # Drop the super table itself tblName = self._db.getFixedSuperTableName() - self.execWtSql(wt, "drop table {}.{}".format(self._db.getName(), tblName)) + + # drop streams before drop stables + if self._db.getFixedSuperTable().hasStreams(wt.getDbConn()): + self._db.getFixedSuperTable().dropStreams(wt.getDbConn()) + self._db.getFixedSuperTable().dropStreamTables(wt.getDbConn()) + + # drop topics before drop stables + if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): + self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) + self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),tblName) + + try: + self.execWtSql(wt, "drop table {}.{}".format(self._db.getName(), tblName)) + except taos.error.ProgrammingError as err: + # correcting for strange error number scheme + errno2 = Helper.convertErrno(err.errno) + if (errno2 in [0x362]): # mnode invalid table name + isSuccess = False + Logging.debug("[DB] Acceptable error when dropping a table") + elif errno2 in [1011,0x3F3,0x03f3]: # table doesn't exist + + pass class TaskAlterTags(StateTransitionTask): @@ -2234,6 +2714,220 @@ class TaskAddData(StateTransitionTask): self.activeTable.discard(i) # not raising an error, unlike remove +class TaskDeleteData(StateTransitionTask): + # Track which table is being actively worked on + activeTable: Set[int] = set() + + # We use these two files to record operations to DB, useful for power-off tests + fAddLogReady = None # type: Optional[io.TextIOWrapper] + fAddLogDone = None # type: Optional[io.TextIOWrapper] + + @classmethod + def prepToRecordOps(cls): + if Config.getConfig().record_ops: + if (cls.fAddLogReady is None): + Logging.info( + "Recording in a file operations to be performed...") + cls.fAddLogReady = open("add_log_ready.txt", "w") + if (cls.fAddLogDone is None): + Logging.info("Recording in a file operations completed...") + cls.fAddLogDone = open("add_log_done.txt", "w") + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canDeleteData() + + def _lockTableIfNeeded(self, fullTableName, extraMsg = ''): + if Config.getConfig().verify_data: + # Logging.info("Locking table: {}".format(fullTableName)) + self.lockTable(fullTableName) + # Logging.info("Table locked {}: {}".format(extraMsg, fullTableName)) + # print("_w" + str(nextInt % 100), end="", flush=True) # Trace what was written + else: + # Logging.info("Skipping locking table") + pass + + def _unlockTableIfNeeded(self, fullTableName): + if Config.getConfig().verify_data: + # Logging.info("Unlocking table: {}".format(fullTableName)) + self.unlockTable(fullTableName) + # Logging.info("Table unlocked: {}".format(fullTableName)) + else: + pass + # Logging.info("Skipping unlocking table") + + def _deleteData(self, db: Database, dbc, regTableName, te: TaskExecutor): # implied: NOT in batches + numRecords = self.LARGE_NUMBER_OF_RECORDS if Config.getConfig().larger_data else self.SMALL_NUMBER_OF_RECORDS + del_Records = int(numRecords/5) + if Dice.throw(2) == 0: + for j in range(del_Records): # number of records per table + intToWrite = db.getNextInt() + nextTick = db.getNextTick() + # nextColor = db.getNextColor() + if Config.getConfig().record_ops: + self.prepToRecordOps() + if self.fAddLogReady is None: + raise CrashGenError("Unexpected empty fAddLogReady") + self.fAddLogReady.write("Ready to delete {} to {}\n".format(intToWrite, regTableName)) + self.fAddLogReady.flush() + os.fsync(self.fAddLogReady.fileno()) + + # TODO: too ugly trying to lock the table reliably, refactor... + fullTableName = db.getName() + '.' + regTableName + self._lockTableIfNeeded(fullTableName) # so that we are verify read-back. TODO: deal with exceptions before unlock + + try: + sql = "delete from {} where ts = '{}' ;".format( # removed: tags ('{}', {}) + fullTableName, + # ds.getFixedSuperTableName(), + # ds.getNextBinary(), ds.getNextFloat(), + nextTick) + + # print(sql) + # Logging.info("Adding data: {}".format(sql)) + dbc.execute(sql) + # Logging.info("Data added: {}".format(sql)) + intWrote = intToWrite + + # Quick hack, attach an update statement here. TODO: create an "update" task + if (not Config.getConfig().use_shadow_db) and Dice.throw(5) == 0: # 1 in N chance, plus not using shaddow DB + intToUpdate = db.getNextInt() # Updated， but should not succeed + # nextColor = db.getNextColor() + sql = "delete from {} where ts = '{}' ;".format( # "INSERt" means "update" here + fullTableName, + nextTick) + # sql = "UPDATE {} set speed={}, color='{}' WHERE ts='{}'".format( + # fullTableName, db.getNextInt(), db.getNextColor(), nextTick) + dbc.execute(sql) + intWrote = intToUpdate # We updated, seems TDengine non-cluster accepts this. + + except: # Any exception at all + self._unlockTableIfNeeded(fullTableName) + raise + + # Now read it back and verify, we might encounter an error if table is dropped + if Config.getConfig().verify_data: # only if command line asks for it + try: + dbc.query("SELECT * from {}.{} WHERE ts='{}'". + format(db.getName(), regTableName, nextTick)) + result = dbc.getQueryResult() + if len(result)==0: + # means data has been delete + print("D1",end="") # DF means delete failed + else: + print("DF",end="") # DF means delete failed + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + # if errno == CrashGenError.INVALID_EMPTY_RESULT: # empty result + # print("D1",end="") # D1 means delete data success and only 1 record + + if errno in [0x218, 0x362,0x2662]: # table doesn't exist + # do nothing + pass + else: + # Re-throw otherwise + raise + finally: + self._unlockTableIfNeeded(fullTableName) # Quite ugly, refactor lock/unlock + # Done with read-back verification, unlock the table now + # Successfully wrote the data into the DB, let's record it somehow + te.recordDataMark(intWrote) + else: + + # delete all datas and verify datas ,expected table is empty + if Config.getConfig().record_ops: + self.prepToRecordOps() + if self.fAddLogReady is None: + raise CrashGenError("Unexpected empty fAddLogReady") + self.fAddLogReady.write("Ready to delete {} to {}\n".format(intToWrite, regTableName)) + self.fAddLogReady.flush() + os.fsync(self.fAddLogReady.fileno()) + + # TODO: too ugly trying to lock the table reliably, refactor... + fullTableName = db.getName() + '.' + regTableName + self._lockTableIfNeeded(fullTableName) # so that we are verify read-back. TODO: deal with exceptions before unlock + + try: + sql = "delete from {} ;".format( # removed: tags ('{}', {}) + fullTableName) + # Logging.info("Adding data: {}".format(sql)) + dbc.execute(sql) + # Logging.info("Data added: {}".format(sql)) + + # Quick hack, attach an update statement here. TODO: create an "update" task + if (not Config.getConfig().use_shadow_db) and Dice.throw(5) == 0: # 1 in N chance, plus not using shaddow DB + sql = "delete from {} ;".format( # "INSERt" means "update" here + fullTableName) + dbc.execute(sql) + + except: # Any exception at all + self._unlockTableIfNeeded(fullTableName) + raise + + # Now read it back and verify, we might encounter an error if table is dropped + if Config.getConfig().verify_data: # only if command line asks for it + try: + dbc.query("SELECT * from {}.{} WHERE ts='{}'". + format(db.getName(), regTableName, nextTick)) + result = dbc.getQueryResult() + if len(result)==0: + # means data has been delete + print("DA",end="") + else: + print("DF",end="") # DF means delete failed + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + # if errno == CrashGenError.INVALID_EMPTY_RESULT: # empty result + # print("Da",end="") # Da means delete data success and for all datas + + if errno in [0x218, 0x362,0x2662]: # table doesn't exist + # do nothing + pass + else: + # Re-throw otherwise + raise + finally: + self._unlockTableIfNeeded(fullTableName) # Quite ugly, refactor lock/unlock + # Done with read-back verification, unlock the table now + + if Config.getConfig().record_ops: + if self.fAddLogDone is None: + raise CrashGenError("Unexpected empty fAddLogDone") + self.fAddLogDone.write("Wrote {} to {}\n".format(intWrote, regTableName)) + self.fAddLogDone.flush() + os.fsync(self.fAddLogDone.fileno()) + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + # ds = self._dbManager # Quite DANGEROUS here, may result in multi-thread client access + db = self._db + dbc = wt.getDbConn() + numTables = self.LARGE_NUMBER_OF_TABLES if Config.getConfig().larger_data else self.SMALL_NUMBER_OF_TABLES + numRecords = self.LARGE_NUMBER_OF_RECORDS if Config.getConfig().larger_data else self.SMALL_NUMBER_OF_RECORDS + tblSeq = list(range(numTables )) + random.shuffle(tblSeq) # now we have random sequence + for i in tblSeq: + if (i in self.activeTable): # wow already active + # print("x", end="", flush=True) # concurrent insertion + Progress.emit(Progress.CONCURRENT_INSERTION) + else: + self.activeTable.add(i) # marking it active + + dbName = db.getName() + sTable = db.getFixedSuperTable() + regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) + fullTableName = dbName + '.' + regTableName + # self._lockTable(fullTableName) # "create table" below. Stop it if the table is "locked" + sTable.ensureRegTable(self, wt.getDbConn(), regTableName) # Ensure the table exists + # self._unlockTable(fullTableName) + + self._deleteData(db, dbc, regTableName, te) + + self.activeTable.discard(i) # not raising an error, unlike remove + class ThreadStacks: # stack info for all threads def __init__(self): @@ -2259,7 +2953,8 @@ class ThreadStacks: # stack info for all threads # Now print print("\n<----- Thread Info for LWP/ID: {} (most recent call last) <-----".format(shortTid)) lastSqlForThread = DbConn.fetchSqlForThread(shortTid) - print("Last SQL statement attempted from thread {} is: {}".format(shortTid, lastSqlForThread)) + time_cost = DbConn.get_time_cost() + print("Last SQL statement attempted from thread {} ({:.4f} sec ago) is: {}".format(shortTid, time_cost ,lastSqlForThread)) stackFrame = 0 for frame in stack: # was using: reversed(stack) # print(frame) @@ -2631,4 +3326,3 @@ class Container(): return self._verifyValidProperty(name) self._cargo[name] = value - diff --git a/tests/pytest/crash_gen/shared/db.py b/tests/pytest/crash_gen/shared/db.py index 60c830f4f7..35ec9a5da6 100644 --- a/tests/pytest/crash_gen/shared/db.py +++ b/tests/pytest/crash_gen/shared/db.py @@ -26,9 +26,12 @@ class DbConn: TYPE_NATIVE = "native-c" TYPE_REST = "rest-api" TYPE_INVALID = "invalid" + + # class variables lastSqlFromThreads : dict[int, str] = {} # stored by thread id, obtained from threading.current_thread().ident%10000 + spendThreads : dict[int, float] = {} # stored by thread id, obtained from threading.current_thread().ident%10000 @classmethod def saveSqlForCurrentThread(cls, sql: str): @@ -37,15 +40,36 @@ class DbConn: run into a dead-lock situation, we can pick out the deadlocked thread, and use that information to find what what SQL statement is stuck. ''' + th = threading.current_thread() shortTid = th.native_id % 10000 #type: ignore cls.lastSqlFromThreads[shortTid] = sql # Save this for later @classmethod - def fetchSqlForThread(cls, shortTid : int) -> str : + def fetchSqlForThread(cls, shortTid : int) -> str : + + print("=======================") if shortTid not in cls.lastSqlFromThreads: raise CrashGenError("No last-attempted-SQL found for thread id: {}".format(shortTid)) - return cls.lastSqlFromThreads[shortTid] + return cls.lastSqlFromThreads[shortTid] + + + @classmethod + def sql_exec_spend(cls, cost: float): + ''' + Let us save the last SQL statement on a per-thread basis, so that when later we + run into a dead-lock situation, we can pick out the deadlocked thread, and use + that information to find what what SQL statement is stuck. + ''' + th = threading.current_thread() + shortTid = th.native_id % 10000 #type: ignore + cls.spendThreads[shortTid] = cost # Save this for later + + @classmethod + def get_time_cost(cls) ->float: + th = threading.current_thread() + shortTid = th.native_id % 10000 #type: ignore + return cls.spendThreads.get(shortTid) @classmethod def create(cls, connType, dbTarget): @@ -61,6 +85,7 @@ class DbConn: def createNative(cls, dbTarget) -> DbConn: return cls.create(cls.TYPE_NATIVE, dbTarget) + @classmethod def createRest(cls, dbTarget) -> DbConn: return cls.create(cls.TYPE_REST, dbTarget) @@ -75,6 +100,7 @@ class DbConn: return "[DbConn: type={}, target={}]".format(self._type, self._dbTarget) def getLastSql(self): + return self._lastSql def open(self): @@ -184,13 +210,19 @@ class DbConnRest(DbConn): def _doSql(self, sql): self._lastSql = sql # remember this, last SQL attempted self.saveSqlForCurrentThread(sql) # Save in global structure too. #TODO: combine with above - try: + time_cost = -1 + time_start = time.time() + try: r = requests.post(self._url, data = sql, - auth = HTTPBasicAuth('root', 'taosdata')) + auth = HTTPBasicAuth('root', 'taosdata')) except: print("REST API Failure (TODO: more info here)") + self.sql_exec_spend(-2) raise + finally: + time_cost = time.time()- time_start + self.sql_exec_spend(time_cost) rj = r.json() # Sanity check for the "Json Result" if ('status' not in rj): @@ -223,6 +255,8 @@ class DbConnRest(DbConn): "[SQL-REST] Execution Result, nRows = {}, SQL = {}".format(nRows, sql)) return nRows + + def query(self, sql): # return rows affected return self.execute(sql) @@ -336,6 +370,7 @@ class MyTDSql: raise return self.affectedRows + class DbTarget: def __init__(self, cfgPath, hostAddr, port): self.cfgPath = cfgPath @@ -355,6 +390,7 @@ class DbConnNative(DbConn): # _connInfoDisplayed = False # TODO: find another way to display this totalConnections = 0 # Not private totalRequests = 0 + time_cost = -1 def __init__(self, dbTarget): super().__init__(dbTarget) @@ -413,8 +449,19 @@ class DbConnNative(DbConn): "Cannot exec SQL unless db connection is open", CrashGenError.DB_CONNECTION_NOT_OPEN) Logging.debug("[SQL] Executing SQL: {}".format(sql)) self._lastSql = sql + time_cost = -1 + nRows = 0 + time_start = time.time() self.saveSqlForCurrentThread(sql) # Save in global structure too. #TODO: combine with above - nRows = self._tdSql.execute(sql) + try: + nRows= self._tdSql.execute(sql) + except Exception as e: + self.sql_exec_spend(-2) + finally: + time_cost = time.time() - time_start + self.sql_exec_spend(time_cost) + + cls = self.__class__ cls.totalRequests += 1 Logging.debug( @@ -494,4 +541,3 @@ class DbManager(): self._dbConn.close() self._dbConn = None Logging.debug("DbManager closed DB connection...") - From 30b1c576e134277b1af26a4639d9fb9b0de01b26 Mon Sep 17 00:00:00 2001 From: wenzhouwww Date: Wed, 9 Nov 2022 18:45:21 +0800 Subject: [PATCH 08/59] Update crash_gen_main.py --- tests/pytest/crash_gen/crash_gen_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 757ddf4af8..bb8de2ab4a 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -255,7 +255,7 @@ class WorkerThread: class ThreadCoordinator: - WORKER_THREAD_TIMEOUT = 1200 # Normal: 120 + WORKER_THREAD_TIMEOUT = 120 # Normal: 120 def __init__(self, pool: ThreadPool, dbManager: DbManager): self._curStep = -1 # first step is 0 From e1ca6a5e9c81fe7dc611b5780b0f670d33739eb9 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Wed, 9 Nov 2022 18:54:21 +0800 Subject: [PATCH 09/59] update --- tests/pytest/crash_gen/crash_gen_main.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index bb8de2ab4a..31b013d065 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1905,18 +1905,23 @@ class TaskCreateConsumers(StateTransitionTask): @classmethod def canBeginFrom(cls, state: AnyState): - return state.canCreateConsumers() + return state.canCreateConsumers() def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - dbname = self._db.getName() - - sTable = self._db.getFixedSuperTable() # type: TdSuperTable - # wt.execSql("use db") # should always be in place - # create Consumers - if Dice.throw(50)==0: # because subscribe is cost so much time , Reduce frequency of this task - if sTable.hasTopics(wt.getDbConn()): - sTable.createConsumer(wt.getDbConn(),random.randint(1,10)) + if Config.getConfig().connector_type == 'native': + dbname = self._db.getName() + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + + # create Consumers + if Dice.throw(50)==0: # because subscribe is cost so much time , Reduce frequency of this task + if sTable.hasTopics(wt.getDbConn()): + sTable.createConsumer(wt.getDbConn(),random.randint(1,10)) + else: + print(" restful not support tmq consumers") + return class TaskCreateSuperTable(StateTransitionTask): From f4aedc9ec2b4032aa191b88f04d925a94efa4dcd Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 10 Nov 2022 09:31:18 +0800 Subject: [PATCH 10/59] update tmq subscript --- tests/pytest/crash_gen/crash_gen_main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 31b013d065..18c3957630 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1374,6 +1374,7 @@ class Task(): 0x707, # Query not ready 0x396, # Database in creating status 0x386, # Database in droping status + 0x03E1, # failed on tmq_subscribe ,topic not exist 1000 # REST catch-all error @@ -2053,7 +2054,10 @@ class TdSuperTable: topic_list = TaosTmqList() for topic in current_topic_list: topic_list.append(topic) - consumer.subscribe(topic_list) + try: + consumer.subscribe(topic_list) + except TmqError as e : + pass time.sleep(5) # consumer work only 5 sec ,and then it will exit try: consumer.unsubscribe() From 119c4c7cab403af8755e136b81f566b6c5e099d6 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 10 Nov 2022 10:38:53 +0800 Subject: [PATCH 11/59] update --- tests/pytest/crash_gen/crash_gen_main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 18c3957630..2a9966eaf8 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1367,6 +1367,7 @@ class Task(): 0x2603, # Table does not exist, replaced by 2662 below 0x260d, # Tags number not matched 0x2662, # Table does not exist #TODO: what about 2603 above? + 0x2600, # database not specified, SQL: show stables , database droped , and show tables 0x032C, # Object is creating 0x032D, # Object is dropping 0x03D3, # Conflict transaction not completed @@ -1377,6 +1378,7 @@ class Task(): 0x03E1, # failed on tmq_subscribe ,topic not exist + 1000 # REST catch-all error ]: return True # These are the ALWAYS-ACCEPTABLE ones From e48d057f7048c28659b2cb26be91965b73987f16 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 10 Nov 2022 18:36:45 +0800 Subject: [PATCH 12/59] update --- tests/pytest/crash_gen/crash_gen_main.py | 212 ++++++++++++++++++----- tests/pytest/crash_gen/shared/db.py | 23 ++- 2 files changed, 192 insertions(+), 43 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 2a9966eaf8..ce0a306a9b 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -420,10 +420,12 @@ class ThreadCoordinator: except threading.BrokenBarrierError as err: self._execStats.registerFailure("Aborted due to worker thread timeout") Logging.error("\n") + Logging.error("Main loop aborted, caused by worker thread(s) time-out of {} seconds".format( ThreadCoordinator.WORKER_THREAD_TIMEOUT)) Logging.error("TAOS related threads blocked at (stack frames top-to-bottom):") ts = ThreadStacks() + ts.record_current_time(time.time()) # record thread exit time at current moment ts.print(filterInternal=True) workerTimeout = True @@ -678,7 +680,11 @@ class AnyState: CAN_CREATE_STREAM = 3 # super table must exists CAN_CREATE_TOPIC = 3 # super table must exists CAN_CREATE_CONSUMERS = 3 + CAN_CREATE_SMA = 3 + CAN_DROP_SMA = 3 CAN_DROP_FIXED_SUPER_TABLE = 4 + CAN_DROP_TOPIC = 4 + CAN_DROP_STREAM = 4 CAN_ADD_DATA = 5 CAN_READ_DATA = 6 CAN_DELETE_DATA = 6 @@ -734,13 +740,25 @@ class AnyState: def canCreateTopic(self): return self._info[self.CAN_CREATE_TOPIC] + + def canDropTopic(self): + return self._info[self.CAN_DROP_TOPIC] def canCreateConsumers(self): return self._info[self.CAN_CREATE_CONSUMERS] + + def canCreateSma(self): + return self._info[self.CAN_CREATE_SMA] + + def canDropSma(self): + return self._info[self.CAN_DROP_SMA] - def canCreateStream(self): + def canCreateStreams(self): return self._info[self.CAN_CREATE_STREAM] + def canDropStream(self): + return self._info[self.CAN_DROP_STREAM] + def canAddData(self): return self._info[self.CAN_ADD_DATA] @@ -919,7 +937,7 @@ class StateHasData(AnyState): ): # only if we didn't create one # we shouldn't have dropped it self.assertNoTask(tasks, TaskDropDb) - if (not self.hasTask(tasks, TaskCreateSuperTable) + if not( self.hasTask(tasks, TaskCreateSuperTable) ): # if we didn't create the table # we should not have a task that drops it self.assertNoTask(tasks, TaskDropSuperTable) @@ -1376,6 +1394,7 @@ class Task(): 0x396, # Database in creating status 0x386, # Database in droping status 0x03E1, # failed on tmq_subscribe ,topic not exist + 0x03ed , # Topic must be dropped first, SQL: drop database db_0 @@ -1694,19 +1713,20 @@ class TaskDropDb(StateTransitionTask): # drop topics before drop db - if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): + # if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): - self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) - self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),self._db.getFixedSuperTableName) + # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) + # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),self._db.getFixedSuperTableName) - self.execWtSql(wt, "drop database {}".format(self._db.getName())) + self.queryWtSql(wt, "drop database {}".format(self._db.getName())) # drop database maybe failed ,because topic exists Logging.debug("[OPS] database dropped at {}".format(time.time())) -''' + # Streams will generator TD-20237 （it will crash taosd , start this task when this issue fixed ） + class TaskCreateStream(StateTransitionTask): @classmethod @@ -1715,7 +1735,7 @@ class TaskCreateStream(StateTransitionTask): @classmethod def canBeginFrom(cls, state: AnyState): - return state.canCreateStream() + return state.canCreateStreams() def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): dbname = self._db.getName() @@ -1784,7 +1804,6 @@ class TaskCreateStream(StateTransitionTask): if errno in [0x03f0]: # stream already exists # stream need drop before drop table pass -''' class TaskCreateTopic(StateTransitionTask): @@ -1856,7 +1875,7 @@ class TaskCreateTopic(StateTransitionTask): Logging.debug("[OPS] topic is creating at {}".format(time.time())) except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) - if errno in [0x03f0]: # topic already exists + if errno in [0x03f0 ]: # topic already exists # topic need drop before drop table pass @@ -1878,7 +1897,7 @@ class TaskCreateTopic(StateTransitionTask): topic_sql = 'create topic {} AS STABLE {}.{} '.format(stable_topic,dbname,stbname) try: self.execWtSql(wt, "use {}".format(dbname)) - self.execWtSql(wt, topic_sql) + self.queryWtSql(wt, topic_sql) Logging.debug("[OPS] stable topic is creating at {}".format(time.time())) except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) @@ -1898,7 +1917,105 @@ class TaskCreateTopic(StateTransitionTask): pass else: pass - + +class TaskDropTopics(StateTransitionTask): + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canDropTopic() + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + dbname = self._db.getName() + + + if not self._db.exists(wt.getDbConn()): + Logging.debug("Skipping task, no DB yet") + return + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + tblName = sTable.getName() + if sTable.hasTopics(wt.getDbConn()): + sTable.dropTopics(wt.getDbConn(),dbname,None) # drop topics of database + sTable.dropTopics(wt.getDbConn(),dbname,tblName) # drop topics of stable + +class TaskCreateSma(StateTransitionTask): + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canCreateSma() + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + # dbname = self._db.getName() + + if not self._db.exists(wt.getDbConn()): + Logging.debug("Skipping task, no DB yet") + return + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + # tblName = sTable.getName() + if sTable.hasStreams(wt.getDbConn()): + sTable.dropStreams(wt.getDbConn()) # drop stream of database + # sTable.dropStreamTables(wt.getDbConn()) # drop streamtables of stable + +class TaskDropStreams(StateTransitionTask): + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canDropStream() + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + # dbname = self._db.getName() + + + if not self._db.exists(wt.getDbConn()): + Logging.debug("Skipping task, no DB yet") + return + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + # tblName = sTable.getName() + if sTable.hasStreams(wt.getDbConn()): + sTable.dropStreams(wt.getDbConn()) # drop stream of database + # sTable.dropStreamTables(wt.getDbConn()) # drop streamtables of stable + +class TaskDropStreamTables(StateTransitionTask): + + @classmethod + def getEndState(cls): + return StateHasData() + + @classmethod + def canBeginFrom(cls, state: AnyState): + return state.canDropStream() + + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + # dbname = self._db.getName() + + + if not self._db.exists(wt.getDbConn()): + Logging.debug("Skipping task, no DB yet") + return + + sTable = self._db.getFixedSuperTable() # type: TdSuperTable + # wt.execSql("use db") # should always be in place + # tblName = sTable.getName() + if sTable.hasStreamTables(wt.getDbConn()): + # sTable.dropStreams(wt.getDbConn()) + sTable.dropStreamTables(wt.getDbConn()) # drop stream tables class TaskCreateConsumers(StateTransitionTask): @@ -1919,9 +2036,10 @@ class TaskCreateConsumers(StateTransitionTask): # wt.execSql("use db") # should always be in place # create Consumers - if Dice.throw(50)==0: # because subscribe is cost so much time , Reduce frequency of this task - if sTable.hasTopics(wt.getDbConn()): - sTable.createConsumer(wt.getDbConn(),random.randint(1,10)) + # if Dice.throw(50)==0: # because subscribe is cost so much time , Reduce frequency of this task + if sTable.hasTopics(wt.getDbConn()): + sTable.createConsumer(wt.getDbConn(),random.randint(1,10)) + pass else: print(" restful not support tmq consumers") return @@ -1969,17 +2087,17 @@ class TdSuperTable: dbName = self._dbName if self.exists(dbc) : # if myself exists fullTableName = dbName + '.' + self._stName - if self.hasStreams(dbc): - self.dropStreams(dbc) - self.dropStreamTables(dbc) - if self.hasTopics(dbc): - self.dropTopics(dbName,None) - self.dropTopics(dbName,self._stName) + # if self.hasStreams(dbc): + # self.dropStreams(dbc) + # self.dropStreamTables(dbc) + # if self.hasTopics(dbc): + # self.dropTopics(dbName,None) + # self.dropTopics(dbName,self._stName) try: dbc.execute("DROP TABLE {}".format(fullTableName)) except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) - if errno in [1011,0x3F3,0x03f3,0x2662]: # table doesn't exist + if errno in [1011,0x3F3,0x03f3,0x2662]: # table doesn't exist # Stream must be dropped first, SQL: DROP TABLE db_0.fs_table pass # # stream need drop before drop table # for stream in self.getStreamName(): @@ -2012,17 +2130,21 @@ class TdSuperTable: if dbc.existsSuperTable(self._stName): if dropIfExists: - if self.hasStreams(dbc): - self.dropStreams(dbc) - self.dropStreamTables(dbc) + # if self.hasStreams(dbc): + # self.dropStreams(dbc) + # self.dropStreamTables(dbc) - # drop topics before drop stables - if self.hasTopics(dbc): - self.dropTopics(dbc,self._dbName,None) - self.dropTopics(dbc,self._dbName,self._stName ) + # # drop topics before drop stables + # if self.hasTopics(dbc): + # self.dropTopics(dbc,self._dbName,None) + # self.dropTopics(dbc,self._dbName,self._stName ) - - dbc.execute("DROP TABLE {}".format(fullTableName)) + try: + dbc.execute("DROP TABLE {}".format(fullTableName)) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [1011,0x3F3,0x03f3,0x2662]: # table doesn't exist # Stream must be dropped first, SQL: DROP TABLE db_0.fs_table + pass pass # dbc.execute("DROP TABLE {}".format(fullTableName)) @@ -2125,6 +2247,7 @@ class TdSuperTable: if dbname in topic[0] and topic[0].startswith("database"): try: dbc.execute('drop topic {}'.format(topic[0])) + Logging.debug("[OPS] topic {} is droping at {}".format(topic,time.time())) except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) if errno in [0x03EB]: # Topic subscribed cannot be dropped @@ -2140,6 +2263,7 @@ class TdSuperTable: for topic in topics: if topic[0].startswith(self._dbName) and topic[0].endswith('topic'): dbc.execute('drop topic {}'.format(topic[0])) + Logging.debug("[OPS] topic {} is droping at {}".format(topic,time.time())) return True else: return True @@ -2455,15 +2579,16 @@ class TaskDropSuperTable(StateTransitionTask): # Drop the super table itself tblName = self._db.getFixedSuperTableName() - # drop streams before drop stables - if self._db.getFixedSuperTable().hasStreams(wt.getDbConn()): - self._db.getFixedSuperTable().dropStreams(wt.getDbConn()) - self._db.getFixedSuperTable().dropStreamTables(wt.getDbConn()) + # # drop streams before drop stables + # if self._db.getFixedSuperTable().hasStreams(wt.getDbConn()): + # self._db.getFixedSuperTable().dropStreams(wt.getDbConn()) + # self._db.getFixedSuperTable().dropStreamTables(wt.getDbConn()) + + # # drop topics before drop stables + # if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): + # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) + # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),tblName) - # drop topics before drop stables - if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): - self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) - self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),tblName) try: self.execWtSql(wt, "drop table {}.{}".format(self._db.getName(), tblName)) @@ -2949,6 +3074,9 @@ class ThreadStacks: # stack info for all threads shortTid = th.native_id % 10000 #type: ignore self._allStacks[shortTid] = stack # Was using th.native_id + def record_current_time(self,current_time): + self.current_time = current_time + def print(self, filteredEndName = None, filterInternal = False): for shortTid, stack in self._allStacks.items(): # for each thread, stack frames top to bottom lastFrame = stack[-1] @@ -2963,9 +3091,11 @@ class ThreadStacks: # stack info for all threads continue # ignore # Now print print("\n<----- Thread Info for LWP/ID: {} (most recent call last) <-----".format(shortTid)) + lastSqlForThread = DbConn.fetchSqlForThread(shortTid) - time_cost = DbConn.get_time_cost() - print("Last SQL statement attempted from thread {} ({:.4f} sec ago) is: {}".format(shortTid, time_cost ,lastSqlForThread)) + last_sql_commit_time = DbConn.get_save_sql_time(shortTid) + # time_cost = DbConn.get_time_cost() + print("Last SQL statement attempted from thread {} ({:.4f} sec ago) is: {}".format(shortTid, self.current_time-last_sql_commit_time ,lastSqlForThread)) stackFrame = 0 for frame in stack: # was using: reversed(stack) # print(frame) diff --git a/tests/pytest/crash_gen/shared/db.py b/tests/pytest/crash_gen/shared/db.py index 35ec9a5da6..05711efbc6 100644 --- a/tests/pytest/crash_gen/shared/db.py +++ b/tests/pytest/crash_gen/shared/db.py @@ -32,7 +32,7 @@ class DbConn: # class variables lastSqlFromThreads : dict[int, str] = {} # stored by thread id, obtained from threading.current_thread().ident%10000 spendThreads : dict[int, float] = {} # stored by thread id, obtained from threading.current_thread().ident%10000 - + current_time : dict[int, float] = {} # save current time @classmethod def saveSqlForCurrentThread(cls, sql: str): ''' @@ -44,6 +44,7 @@ class DbConn: th = threading.current_thread() shortTid = th.native_id % 10000 #type: ignore cls.lastSqlFromThreads[shortTid] = sql # Save this for later + cls.record_save_sql_time() @classmethod def fetchSqlForThread(cls, shortTid : int) -> str : @@ -53,6 +54,25 @@ class DbConn: raise CrashGenError("No last-attempted-SQL found for thread id: {}".format(shortTid)) return cls.lastSqlFromThreads[shortTid] + @classmethod + def get_save_sql_time(cls, shortTid : int): + ''' + Let us save the last SQL statement on a per-thread basis, so that when later we + run into a dead-lock situation, we can pick out the deadlocked thread, and use + that information to find what what SQL statement is stuck. + ''' + return cls.current_time[shortTid] + + @classmethod + def record_save_sql_time(cls): + ''' + Let us save the last SQL statement on a per-thread basis, so that when later we + run into a dead-lock situation, we can pick out the deadlocked thread, and use + that information to find what what SQL statement is stuck. + ''' + th = threading.current_thread() + shortTid = th.native_id % 10000 #type: ignore + cls.current_time[shortTid] = float(time.time()) # Save this for later @classmethod def sql_exec_spend(cls, cost: float): @@ -460,7 +480,6 @@ class DbConnNative(DbConn): finally: time_cost = time.time() - time_start self.sql_exec_spend(time_cost) - cls = self.__class__ cls.totalRequests += 1 From 0f8bb6faa7b67e360362efa55af32b51d0775ca0 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 10 Nov 2022 20:16:04 +0800 Subject: [PATCH 13/59] update enh crash_gen tasks --- tests/pytest/crash_gen/crash_gen_main.py | 63 +++++++++++------------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index ce0a306a9b..6780f12d96 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -549,7 +549,12 @@ class ThreadCoordinator: # pick a task type for current state db = self.pickDatabase() - taskType = db.getStateMachine().pickTaskType() # dynamic name of class + if Dice.throw(2)==1: + taskType = db.getStateMachine().pickTaskType() # dynamic name of class + else: + taskType = db.getStateMachine().balance_pickTaskType() # and an method can get balance task types + pass + return taskType(self._execStats, db) # create a task from it def resetExecutedTasks(self): @@ -680,8 +685,6 @@ class AnyState: CAN_CREATE_STREAM = 3 # super table must exists CAN_CREATE_TOPIC = 3 # super table must exists CAN_CREATE_CONSUMERS = 3 - CAN_CREATE_SMA = 3 - CAN_DROP_SMA = 3 CAN_DROP_FIXED_SUPER_TABLE = 4 CAN_DROP_TOPIC = 4 CAN_DROP_STREAM = 4 @@ -746,12 +749,6 @@ class AnyState: def canCreateConsumers(self): return self._info[self.CAN_CREATE_CONSUMERS] - - def canCreateSma(self): - return self._info[self.CAN_CREATE_SMA] - - def canDropSma(self): - return self._info[self.CAN_DROP_SMA] def canCreateStreams(self): return self._info[self.CAN_CREATE_STREAM] @@ -1093,6 +1090,28 @@ class StateMechine: # Logging.debug(" (weighted random:{}/{}) ".format(i, len(taskTypes))) return taskTypes[i] + def balance_pickTaskType(self): + # all the task types we can choose from at curent state + BasicTypes = self.getTaskTypes() + weightsTypes = BasicTypes.copy() + + # this matrixs can balance the Frequency of different types of tasks + weight_matrixs = {'TaskDropDb': 5 , 'TaskDropTopics': 20 , 'TaskDropStreams':10 , 'TaskDropStreamTables':10 , + 'TaskReadData':50 , 'TaskDropSuperTable':5 , 'TaskAlterTags':3 , 'TaskAddData':10, + 'TaskDeleteData':10 , 'TaskCreateDb':10 , 'TaskCreateStream': 3, 'TaskCreateTopic' :3, + 'TaskCreateConsumers':10, 'TaskCreateSuperTable': 10 } # task type : weghts matrixs + + for task , weights in weight_matrixs.items(): + + for basicType in BasicTypes: + if basicType.__name__ == task: + for _ in range(weights): + weightsTypes.append(basicType) + + task = random.sample(weightsTypes,1) + return task[0] + + # ref: # https://eli.thegreenplace.net/2010/01/22/weighted-random-generation-in-python/ def _weighted_choice_sub(self, weights) -> int: @@ -1395,6 +1414,8 @@ class Task(): 0x386, # Database in droping status 0x03E1, # failed on tmq_subscribe ,topic not exist 0x03ed , # Topic must be dropped first, SQL: drop database db_0 + 0x0203 , # Invalid value + @@ -1942,30 +1963,6 @@ class TaskDropTopics(StateTransitionTask): if sTable.hasTopics(wt.getDbConn()): sTable.dropTopics(wt.getDbConn(),dbname,None) # drop topics of database sTable.dropTopics(wt.getDbConn(),dbname,tblName) # drop topics of stable - -class TaskCreateSma(StateTransitionTask): - - @classmethod - def getEndState(cls): - return StateHasData() - - @classmethod - def canBeginFrom(cls, state: AnyState): - return state.canCreateSma() - - def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - # dbname = self._db.getName() - - if not self._db.exists(wt.getDbConn()): - Logging.debug("Skipping task, no DB yet") - return - - sTable = self._db.getFixedSuperTable() # type: TdSuperTable - # wt.execSql("use db") # should always be in place - # tblName = sTable.getName() - if sTable.hasStreams(wt.getDbConn()): - sTable.dropStreams(wt.getDbConn()) # drop stream of database - # sTable.dropStreamTables(wt.getDbConn()) # drop streamtables of stable class TaskDropStreams(StateTransitionTask): From 6175929113777d50ded7b7d282424e99e05a9981 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 11 Nov 2022 13:50:37 +0800 Subject: [PATCH 14/59] update --- tests/pytest/crash_gen/crash_gen_main.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 6780f12d96..4fa19f543d 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -953,7 +953,7 @@ class StateMechine: except taos.error.ProgrammingError as err: Logging.error("Failed to initialized state machine, cannot find current state: {}".format(err)) traceback.print_stack() - pass # re-throw + raise # re-throw # TODO: seems no lnoger used, remove? def getCurrentState(self): @@ -1095,13 +1095,13 @@ class StateMechine: BasicTypes = self.getTaskTypes() weightsTypes = BasicTypes.copy() - # this matrixs can balance the Frequency of different types of tasks - weight_matrixs = {'TaskDropDb': 5 , 'TaskDropTopics': 20 , 'TaskDropStreams':10 , 'TaskDropStreamTables':10 , + # this matrixs can balance the Frequency of TaskTypes + balance_TaskType_matrixs = {'TaskDropDb': 5 , 'TaskDropTopics': 20 , 'TaskDropStreams':10 , 'TaskDropStreamTables':10 , 'TaskReadData':50 , 'TaskDropSuperTable':5 , 'TaskAlterTags':3 , 'TaskAddData':10, 'TaskDeleteData':10 , 'TaskCreateDb':10 , 'TaskCreateStream': 3, 'TaskCreateTopic' :3, - 'TaskCreateConsumers':10, 'TaskCreateSuperTable': 10 } # task type : weghts matrixs + 'TaskCreateConsumers':10, 'TaskCreateSuperTable': 10 } # TaskType : balance_matrixs of task - for task , weights in weight_matrixs.items(): + for task , weights in balance_TaskType_matrixs.items(): for basicType in BasicTypes: if basicType.__name__ == task: @@ -1738,8 +1738,12 @@ class TaskDropDb(StateTransitionTask): # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),self._db.getFixedSuperTableName) - - self.queryWtSql(wt, "drop database {}".format(self._db.getName())) # drop database maybe failed ,because topic exists + try: + self.queryWtSql(wt, "drop database {}".format(self._db.getName())) # drop database maybe failed ,because topic exists + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x0203]: # drop maybe failed + pass Logging.debug("[OPS] database dropped at {}".format(time.time())) From b47052b7afecd4bec7c7e46a6e68633c4307e6a9 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Tue, 15 Nov 2022 09:12:42 +0800 Subject: [PATCH 15/59] update --- tests/pytest/crash_gen/crash_gen_main.py | 244 +++++------------------ 1 file changed, 48 insertions(+), 196 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 4fa19f543d..90d27c3b8d 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1415,6 +1415,7 @@ class Task(): 0x03E1, # failed on tmq_subscribe ,topic not exist 0x03ed , # Topic must be dropped first, SQL: drop database db_0 0x0203 , # Invalid value + 0x03f0 , # Stream already exist , topic already exists @@ -1732,12 +1733,6 @@ class TaskDropDb(StateTransitionTask): def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - # drop topics before drop db - - # if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): - - # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) - # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),self._db.getFixedSuperTableName) try: self.queryWtSql(wt, "drop database {}".format(self._db.getName())) # drop database maybe failed ,because topic exists except taos.error.ProgrammingError as err: @@ -1748,10 +1743,6 @@ class TaskDropDb(StateTransitionTask): Logging.debug("[OPS] database dropped at {}".format(time.time())) - -# Streams will generator TD-20237 （it will crash taosd , start this task when this issue fixed ） - - class TaskCreateStream(StateTransitionTask): @classmethod @@ -1775,60 +1766,25 @@ class TaskCreateStream(StateTransitionTask): sTable = self._db.getFixedSuperTable() # type: TdSuperTable # wt.execSql("use db") # should always be in place - - # create stream - - # CREATE STREAM avg_vol_s INTO avg_vol AS SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); - stbname =sTable.getName() sub_tables = sTable.getRegTables(wt.getDbConn()) aggExpr = Dice.choice([ - 'count(*)', - 'avg(speed)', - # 'twa(speed)', # TODO: this one REQUIRES a where statement, not reasonable - 'sum(speed)', - 'stddev(speed)', - # SELECTOR functions - 'min(speed)', - 'max(speed)', - 'first(speed)', - 'last(speed)', - 'apercentile(speed, 10)', # TODO: TD-1316 - 'last_row(*)', # TODO: commented out per TD-3231, we should re-create - # Transformation Functions - 'twa(speed)' - - ]) # TODO: add more from 'top' + 'count(*)', 'avg(speed)', 'sum(speed)', 'stddev(speed)','min(speed)', 'max(speed)', 'first(speed)', 'last(speed)', + 'apercentile(speed, 10)', 'last_row(*)', 'twa(speed)']) + + stream_sql = '' # set default value if sub_tables: - - if sub_tables: # if not empty - sub_tbname = sub_tables[0] - # create stream with query above sub_table - stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(sub_stream_name,dbname,sub_stream_tb_name ,aggExpr,dbname,sub_tbname) - try: - self.execWtSql(wt, stream_sql) - Logging.debug("[OPS] stream is creating at {}".format(time.time())) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [0x03f0]: # stream already exists - # stream need drop before drop table - pass - - else: - pass - + sub_tbname = sub_tables[0] + # create stream with query above sub_table + stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.\ + format(sub_stream_name,dbname,sub_stream_tb_name ,aggExpr,dbname,sub_tbname) else: - stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.format(super_stream_name,dbname,super_stream_tb_name,aggExpr, dbname,stbname) + stream_sql = 'create stream {} into {}.{} as select {}, avg(speed) FROM {}.{} PARTITION BY tbname INTERVAL(5s) SLIDING(3s) '.\ + format(super_stream_name,dbname,super_stream_tb_name,aggExpr, dbname,stbname) + self.execWtSql(wt, stream_sql) + Logging.debug("[OPS] stream is creating at {}".format(time.time())) - try: - self.execWtSql(wt, stream_sql) - Logging.debug("[OPS] stream is creating at {}".format(time.time())) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [0x03f0]: # stream already exists - # stream need drop before drop table - pass class TaskCreateTopic(StateTransitionTask): @@ -1853,96 +1809,37 @@ class TaskCreateTopic(StateTransitionTask): sTable = self._db.getFixedSuperTable() # type: TdSuperTable # wt.execSql("use db") # should always be in place - - # create topic - # create topic if not exists topic_ctb_column as select ts, c1, c2, c3 from stb1; stbname =sTable.getName() sub_tables = sTable.getRegTables(wt.getDbConn()) - scalarExpr = Dice.choice([ '*','speed','color', - 'abs(speed)', - 'acos(speed)', - 'asin(speed)', - 'atan(speed)', - 'ceil(speed)', - 'cos(speed)', - 'cos(speed)', - 'floor(speed)', - 'log(speed,2)', - 'pow(speed,2)', - 'round(speed)', - 'sin(speed)', - 'sqrt(speed)', - 'char_length(color)', - 'concat(color,color)', - 'concat_ws(" ", color,color," ")', - 'length(color)', - 'lower(color)', - 'ltrim(color)', - 'substr(color , 2)', - 'upper(color)', - 'cast(speed as double)', - 'cast(ts as bigint)', - - ]) # TODO: add more from 'top' - if Dice.throw(3)==0: - if sub_tables: - - if sub_tables: # if not empty - sub_tbname = sub_tables[0] - # create stream with query above sub_table - topic_sql = 'create topic {} as select {} FROM {}.{} ; '.format(sub_topic_name,scalarExpr,dbname,sub_tbname) - try: - self.execWtSql(wt, "use {}".format(dbname)) - self.execWtSql(wt, topic_sql) - Logging.debug("[OPS] topic is creating at {}".format(time.time())) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [0x03f0 ]: # topic already exists - # topic need drop before drop table - pass - - else: - pass - - else: + scalarExpr = Dice.choice([ '*','speed','color','abs(speed)','acos(speed)','asin(speed)','atan(speed)','ceil(speed)','cos(speed)','cos(speed)', + 'floor(speed)','log(speed,2)','pow(speed,2)','round(speed)','sin(speed)','sqrt(speed)','char_length(color)','concat(color,color)', + 'concat_ws(" ", color,color," ")','length(color)', 'lower(color)', 'ltrim(color)','substr(color , 2)','upper(color)','cast(speed as double)', + 'cast(ts as bigint)']) + topic_sql = '' # set default value + if Dice.throw(3)==0: # create topic : source data from sub query + if sub_tables: # if not empty + sub_tbname = sub_tables[0] + # create topic : source data from sub query of sub stable + topic_sql = 'create topic {} as select {} FROM {}.{} ; '.format(sub_topic_name,scalarExpr,dbname,sub_tbname) + + else: # create topic : source data from sub query of stable topic_sql = 'create topic {} as select {} FROM {}.{} '.format(super_topic_name,scalarExpr, dbname,stbname) - try: - self.execWtSql(wt, "use {}".format(dbname)) - self.execWtSql(wt, topic_sql) - Logging.debug("[OPS] subquery topic is creating at {}".format(time.time())) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [0x03f0]: # topic already exists - # topic need drop before drop table - pass - elif Dice.throw(3)==1: + elif Dice.throw(3)==1: # create topic : source data from super table topic_sql = 'create topic {} AS STABLE {}.{} '.format(stable_topic,dbname,stbname) - try: - self.execWtSql(wt, "use {}".format(dbname)) - self.queryWtSql(wt, topic_sql) - Logging.debug("[OPS] stable topic is creating at {}".format(time.time())) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [0x03f0]: # topic already exists - # topic need drop before drop table - pass - elif Dice.throw(3)==2: - topic_sql = 'create topic {} AS DATABASE {} '.format(db_topic,dbname) - try: - self.execWtSql(wt, "use {}".format(dbname)) - self.execWtSql(wt, topic_sql) - Logging.debug("[OPS] db topic is creating at {}".format(time.time())) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [0x03f0]: # topic already exists - # topic need drop before drop table - pass + + elif Dice.throw(3)==2: # create topic : source data from whole database + topic_sql = 'create topic {} AS DATABASE {} '.format(db_topic,dbname) else: pass + # exec create topics + self.execWtSql(wt, "use {}".format(dbname)) + self.execWtSql(wt, topic_sql) + Logging.debug("[OPS] db topic is creating at {}".format(time.time())) + class TaskDropTopics(StateTransitionTask): @classmethod @@ -1991,7 +1888,6 @@ class TaskDropStreams(StateTransitionTask): # tblName = sTable.getName() if sTable.hasStreams(wt.getDbConn()): sTable.dropStreams(wt.getDbConn()) # drop stream of database - # sTable.dropStreamTables(wt.getDbConn()) # drop streamtables of stable class TaskDropStreamTables(StateTransitionTask): @@ -2012,10 +1908,9 @@ class TaskDropStreamTables(StateTransitionTask): return sTable = self._db.getFixedSuperTable() # type: TdSuperTable - # wt.execSql("use db") # should always be in place + wt.execSql("use db") # should always be in place # tblName = sTable.getName() if sTable.hasStreamTables(wt.getDbConn()): - # sTable.dropStreams(wt.getDbConn()) sTable.dropStreamTables(wt.getDbConn()) # drop stream tables class TaskCreateConsumers(StateTransitionTask): @@ -2031,13 +1926,9 @@ class TaskCreateConsumers(StateTransitionTask): def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): if Config.getConfig().connector_type == 'native': - dbname = self._db.getName() - + sTable = self._db.getFixedSuperTable() # type: TdSuperTable # wt.execSql("use db") # should always be in place - - # create Consumers - # if Dice.throw(50)==0: # because subscribe is cost so much time , Reduce frequency of this task if sTable.hasTopics(wt.getDbConn()): sTable.createConsumer(wt.getDbConn(),random.randint(1,10)) pass @@ -2088,30 +1979,13 @@ class TdSuperTable: dbName = self._dbName if self.exists(dbc) : # if myself exists fullTableName = dbName + '.' + self._stName - # if self.hasStreams(dbc): - # self.dropStreams(dbc) - # self.dropStreamTables(dbc) - # if self.hasTopics(dbc): - # self.dropTopics(dbName,None) - # self.dropTopics(dbName,self._stName) + try: dbc.execute("DROP TABLE {}".format(fullTableName)) except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) if errno in [1011,0x3F3,0x03f3,0x2662]: # table doesn't exist # Stream must be dropped first, SQL: DROP TABLE db_0.fs_table pass - # # stream need drop before drop table - # for stream in self.getStreamName(): - # drop_stream_sql = 'drop stream {}'.format(stream) - # try: - # dbc.execute(drop_stream_sql) - # except taos.error.ProgrammingError as err: - # # correcting for strange error number scheme - # errno3 = Helper.convertErrno(err.errno) - # if errno3 in [1011,0x3F3,0x03f3,0x2662,0x03f1]: # stream not exists - # pass - # dbc.execute("DROP TABLE {}".format(fullTableName)) - # pass else: if not skipCheck: @@ -2131,15 +2005,6 @@ class TdSuperTable: if dbc.existsSuperTable(self._stName): if dropIfExists: - # if self.hasStreams(dbc): - # self.dropStreams(dbc) - # self.dropStreamTables(dbc) - - # # drop topics before drop stables - # if self.hasTopics(dbc): - # self.dropTopics(dbc,self._dbName,None) - # self.dropTopics(dbc,self._dbName,self._stName ) - try: dbc.execute("DROP TABLE {}".format(fullTableName)) except taos.error.ProgrammingError as err: @@ -2164,7 +2029,7 @@ class TdSuperTable: sql += " TAGS (dummy int) " dbc.execute(sql) - def createConsumer(self, dbc: DbConn , Consumer_nums): + def createConsumer(self, dbc,Consumer_nums): def generateConsumer(current_topic_list): conf = TaosTmqConf() @@ -2183,7 +2048,14 @@ class TdSuperTable: consumer.subscribe(topic_list) except TmqError as e : pass - time.sleep(5) # consumer work only 5 sec ,and then it will exit + + # consumer work only 30 sec + time_start = time.time() + while 1: + res = consumer.poll(1000) + if time.time() - time_start >5 : + break + # time.sleep(10) try: consumer.unsubscribe() except TmqError as e : @@ -2579,29 +2451,9 @@ class TaskDropSuperTable(StateTransitionTask): # Drop the super table itself tblName = self._db.getFixedSuperTableName() - - # # drop streams before drop stables - # if self._db.getFixedSuperTable().hasStreams(wt.getDbConn()): - # self._db.getFixedSuperTable().dropStreams(wt.getDbConn()) - # self._db.getFixedSuperTable().dropStreamTables(wt.getDbConn()) + self.execWtSql(wt, "drop table {}.{}".format(self._db.getName(), tblName)) + - # # drop topics before drop stables - # if self._db.getFixedSuperTable().hasTopics(wt.getDbConn()): - # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),None) - # self._db.getFixedSuperTable().dropTopics(wt.getDbConn(),self._db.getName(),tblName) - - - try: - self.execWtSql(wt, "drop table {}.{}".format(self._db.getName(), tblName)) - except taos.error.ProgrammingError as err: - # correcting for strange error number scheme - errno2 = Helper.convertErrno(err.errno) - if (errno2 in [0x362]): # mnode invalid table name - isSuccess = False - Logging.debug("[DB] Acceptable error when dropping a table") - elif errno2 in [1011,0x3F3,0x03f3]: # table doesn't exist - - pass class TaskAlterTags(StateTransitionTask): From 1b0c98433e4e493e8bd273994e69689210215ec9 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 17 Nov 2022 18:04:57 +0800 Subject: [PATCH 16/59] enh: refact query message --- include/common/tmsg.h | 18 +- source/common/src/tmsg.c | 135 +++++++++++++++ source/libs/qworker/src/qwMsg.c | 193 ++++++++++++---------- source/libs/qworker/test/qworkerTests.cpp | 29 +++- source/libs/scheduler/src/schRemote.c | 76 +++++---- 5 files changed, 326 insertions(+), 125 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 8066694709..9b7f3bff4c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1615,15 +1615,21 @@ typedef struct SSubQueryMsg { uint64_t taskId; int64_t refId; int32_t execId; + int32_t msgMask; int8_t taskType; int8_t explain; int8_t needFetch; - uint32_t sqlLen; // the query sql, - uint32_t phyLen; - int32_t msgMask; - char msg[]; + uint32_t sqlLen; + char *sql; + uint32_t msgLen; + char *msg; } SSubQueryMsg; +int32_t tSerializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq); +int32_t tDeserializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq); +void tFreeSSubQueryMsg(SSubQueryMsg *pReq); + + typedef struct { SMsgHead header; uint64_t sId; @@ -1732,6 +1738,10 @@ typedef struct { int32_t execId; } STaskDropReq; +int32_t tSerializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq); +int32_t tDeserializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq); + + typedef struct { int32_t code; } STaskDropRsp; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 63a2b712fc..8ddb60655e 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4643,6 +4643,141 @@ int32_t tDeserializeSMqHbReq(void *buf, int32_t bufLen, SMqHbReq *pReq) { return 0; } +int32_t tSerializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq) { + int32_t headLen = sizeof(SMsgHead); + if (buf != NULL) { + buf = (char *)buf + headLen; + bufLen -= headLen; + } + + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + if (tStartEncode(&encoder) < 0) return -1; + + if (tEncodeU64(&encoder, pReq->sId) < 0) return -1; + if (tEncodeU64(&encoder, pReq->queryId) < 0) return -1; + if (tEncodeU64(&encoder, pReq->taskId) < 0) return -1; + if (tEncodeI64(&encoder, pReq->refId) < 0) return -1; + if (tEncodeI32(&encoder, pReq->execId) < 0) return -1; + if (tEncodeI32(&encoder, pReq->msgMask) < 0) return -1; + if (tEncodeI8(&encoder, pReq->taskType) < 0) return -1; + if (tEncodeI8(&encoder, pReq->explain) < 0) return -1; + if (tEncodeI8(&encoder, pReq->needFetch) < 0) return -1; + if (tEncodeU32(&encoder, pReq->sqlLen) < 0) return -1; + if (tEncodeCStrWithLen(&encoder, pReq->sql, pReq->sqlLen) < 0) return -1; + if (tEncodeU32(&encoder, pReq->msgLen) < 0) return -1; + if (tEncodeCStrWithLen(&encoder, pReq->msg, pReq->msgLen) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + + if (buf != NULL) { + SMsgHead *pHead = (SMsgHead *)((char *)buf - headLen); + pHead->vgId = htonl(pReq->header.vgId); + pHead->contLen = htonl(tlen + headLen); + } + + return tlen + headLen; +} + +int32_t tDeserializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq) { + int32_t headLen = sizeof(SMsgHead); + + SMsgHead *pHead = buf; + pHead->vgId = pReq->header.vgId; + pHead->contLen = pReq->header.contLen; + + SDecoder decoder = {0}; + tDecoderInit(&decoder, (char *)buf + headLen, bufLen - headLen); + + if (tStartDecode(&decoder) < 0) return -1; + + if (tDecodeU64(&decoder, &pReq->sId) < 0) return -1; + if (tDecodeU64(&decoder, &pReq->queryId) < 0) return -1; + if (tDecodeU64(&decoder, &pReq->taskId) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->refId) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->execId) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->msgMask) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->taskType) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->explain) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->needFetch) < 0) return -1; + if (tDecodeU32(&decoder, &pReq->sqlLen) < 0) return -1; + if (tDecodeCStrAlloc(&decoder, &pReq->sql) < 0) return -1; + if (tDecodeU32(&decoder, &pReq->msgLen) < 0) return -1; + if (tDecodeCStrAlloc(&decoder, &pReq->msg) < 0) return -1; + + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + +void tFreeSSubQueryMsg(SSubQueryMsg *pReq) { + if (NULL == pReq) { + return; + } + + taosMemoryFreeClear(pReq->sql); + taosMemoryFreeClear(pReq->msg); +} + +int32_t tSerializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq) { + int32_t headLen = sizeof(SMsgHead); + if (buf != NULL) { + buf = (char *)buf + headLen; + bufLen -= headLen; + } + + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + if (tStartEncode(&encoder) < 0) return -1; + + if (tEncodeU64(&encoder, pReq->sId) < 0) return -1; + if (tEncodeU64(&encoder, pReq->queryId) < 0) return -1; + if (tEncodeU64(&encoder, pReq->taskId) < 0) return -1; + if (tEncodeI64(&encoder, pReq->refId) < 0) return -1; + if (tEncodeI32(&encoder, pReq->execId) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + + if (buf != NULL) { + SMsgHead *pHead = (SMsgHead *)((char *)buf - headLen); + pHead->vgId = htonl(pReq->header.vgId); + pHead->contLen = htonl(tlen + headLen); + } + + return tlen + headLen; +} + +int32_t tDeserializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq) { + int32_t headLen = sizeof(SMsgHead); + + SMsgHead *pHead = buf; + pHead->vgId = pReq->header.vgId; + pHead->contLen = pReq->header.contLen; + + SDecoder decoder = {0}; + tDecoderInit(&decoder, (char *)buf + headLen, bufLen - headLen); + + if (tStartDecode(&decoder) < 0) return -1; + + if (tDecodeU64(&decoder, &pReq->sId) < 0) return -1; + if (tDecodeU64(&decoder, &pReq->queryId) < 0) return -1; + if (tDecodeU64(&decoder, &pReq->taskId) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->refId) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->execId) < 0) return -1; + + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + int32_t tSerializeSSchedulerHbReq(void *buf, int32_t bufLen, SSchedulerHbReq *pReq) { int32_t headLen = sizeof(SMsgHead); diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 7e7f71b176..82e71bc8d0 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -182,23 +182,37 @@ int32_t qwBuildAndSendDropRsp(SRpcHandleInfo *pConn, int32_t code) { #endif int32_t qwBuildAndSendDropMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { - STaskDropReq *req = (STaskDropReq *)rpcMallocCont(sizeof(STaskDropReq)); - if (NULL == req) { - QW_SCH_TASK_ELOG("rpcMallocCont %d failed", (int32_t)sizeof(STaskDropReq)); + STaskDropReq qMsg; + qMsg.header.vgId = mgmt->nodeId; + qMsg.header.contLen = 0; + qMsg.sId = sId; + qMsg.queryId = qId; + qMsg.taskId = tId; + qMsg.refId = rId; + qMsg.execId = eId; + + int32_t msgSize = tSerializeSTaskDropReq(NULL, 0, &qMsg); + if (msgSize < 0) { + QW_SCH_TASK_ELOG("tSerializeSTaskDropReq get size, msgSize:%d", msgSize); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + void *msg = rpcMallocCont(msgSize); + if (NULL == msg) { + QW_SCH_TASK_ELOG("rpcMallocCont %d failed", msgSize); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + if (tSerializeSTaskDropReq(msg, msgSize, &qMsg) < 0) { + QW_SCH_TASK_ELOG("tSerializeSTaskDropReq failed, msgSize:%d", msgSize); + rpcFreeCont(msg); QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - req->header.vgId = mgmt->nodeId; - req->sId = sId; - req->queryId = qId; - req->taskId = tId; - req->refId = rId; - req->execId = eId; - SRpcMsg pNewMsg = { .msgType = TDMT_SCH_DROP_TASK, - .pCont = req, - .contLen = sizeof(STaskDropReq), + .pCont = msg, + .contLen = msgSize, .code = 0, .info = *pConn, }; @@ -247,22 +261,37 @@ int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { } int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { - STaskDropReq *req = (STaskDropReq *)rpcMallocCont(sizeof(STaskDropReq)); - if (NULL == req) { - QW_SCH_TASK_ELOG("rpcMallocCont %d failed", (int32_t)sizeof(STaskDropReq)); + STaskDropReq qMsg; + qMsg.header.vgId = mgmt->nodeId; + qMsg.header.contLen = 0; + qMsg.sId = sId; + qMsg.queryId = qId; + qMsg.taskId = tId; + qMsg.refId = rId; + qMsg.execId = eId; + + int32_t msgSize = tSerializeSTaskDropReq(NULL, 0, &qMsg); + if (msgSize < 0) { + QW_SCH_TASK_ELOG("tSerializeSTaskDropReq get size, msgSize:%d", msgSize); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + void *msg = rpcMallocCont(msgSize); + if (NULL == msg) { + QW_SCH_TASK_ELOG("rpcMallocCont %d failed", msgSize); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + if (tSerializeSTaskDropReq(msg, msgSize, &qMsg) < 0) { + QW_SCH_TASK_ELOG("tSerializeSTaskDropReq failed, msgSize:%d", msgSize); + rpcFreeCont(msg); QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - req->header.vgId = htonl(mgmt->nodeId); - req->sId = htobe64(sId); - req->queryId = htobe64(qId); - req->taskId = htobe64(tId); - req->refId = htobe64(rId); - SRpcMsg brokenMsg = { .msgType = TDMT_SCH_DROP_TASK, - .pCont = req, - .contLen = sizeof(STaskDropReq), + .pCont = msg, + .contLen = msgSize, .code = TSDB_CODE_RPC_BROKEN_LINK, .info = *pConn, }; @@ -312,40 +341,31 @@ int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg, bool chkGran } int32_t code = 0; - SSubQueryMsg *msg = pMsg->pCont; SQWorker *mgmt = (SQWorker *)qWorkerMgmt; - - if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { - QW_ELOG("invalid query msg, msg:%p, msgLen:%d", msg, pMsg->contLen); + SSubQueryMsg msg = {0}; + if (tDeserializeSSubQueryMsg(pMsg->pCont, pMsg->contLen, &msg) < 0) { + QW_ELOG("tDeserializeSSubQueryMsg failed, contLen:%d", pMsg->contLen); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - msg->sId = be64toh(msg->sId); - msg->queryId = be64toh(msg->queryId); - msg->taskId = be64toh(msg->taskId); - msg->refId = be64toh(msg->refId); - msg->execId = ntohl(msg->execId); - msg->phyLen = ntohl(msg->phyLen); - msg->sqlLen = ntohl(msg->sqlLen); - msg->msgMask = ntohl(msg->msgMask); - - if (chkGrant && (!TEST_SHOW_REWRITE_MASK(msg->msgMask)) && (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS)) { - QW_ELOG("query failed cause of grant expired, msgMask:%d", msg->msgMask); + if (chkGrant && (!TEST_SHOW_REWRITE_MASK(msg.msgMask)) && (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS)) { + QW_ELOG("query failed cause of grant expired, msgMask:%d", msg.msgMask); + tFreeSSubQueryMsg(&msg); QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); } - uint64_t sId = msg->sId; - uint64_t qId = msg->queryId; - uint64_t tId = msg->taskId; - int64_t rId = msg->refId; - int32_t eId = msg->execId; + uint64_t sId = msg.sId; + uint64_t qId = msg.queryId; + uint64_t tId = msg.taskId; + int64_t rId = msg.refId; + int32_t eId = msg.execId; SQWMsg qwMsg = { - .msgType = pMsg->msgType, .msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen, .connInfo = pMsg->info}; + .msgType = pMsg->msgType, .msg = msg.msg, .msgLen = msg.msgLen, .connInfo = pMsg->info}; - QW_SCH_TASK_DLOG("prerocessQuery start, handle:%p", pMsg->info.handle); - QW_ERR_RET(qwPreprocessQuery(QW_FPARAMS(), &qwMsg)); - QW_SCH_TASK_DLOG("prerocessQuery end, handle:%p", pMsg->info.handle); + QW_SCH_TASK_DLOG("prerocessQuery start, handle:%p, SQL:%s", pMsg->info.handle, msg.sql); + code = qwPreprocessQuery(QW_FPARAMS(), &qwMsg); + QW_SCH_TASK_DLOG("prerocessQuery end, handle:%p, code:%x", pMsg->info.handle, code); return TSDB_CODE_SUCCESS; } @@ -355,19 +375,25 @@ int32_t qWorkerAbortPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg) { QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SSubQueryMsg *msg = pMsg->pCont; SQWorker *mgmt = (SQWorker *)qWorkerMgmt; + SSubQueryMsg msg = {0}; + if (tDeserializeSSubQueryMsg(pMsg->pCont, pMsg->contLen, &msg) < 0) { + QW_ELOG("tDeserializeSSubQueryMsg failed, contLen:%d", pMsg->contLen); + QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } - uint64_t sId = msg->sId; - uint64_t qId = msg->queryId; - uint64_t tId = msg->taskId; - int64_t rId = msg->refId; - int32_t eId = msg->execId; + uint64_t sId = msg.sId; + uint64_t qId = msg.queryId; + uint64_t tId = msg.taskId; + int64_t rId = msg.refId; + int32_t eId = msg.execId; QW_SCH_TASK_DLOG("Abort prerocessQuery start, handle:%p", pMsg->info.handle); qwAbortPrerocessQuery(QW_FPARAMS()); QW_SCH_TASK_DLOG("Abort prerocessQuery end, handle:%p", pMsg->info.handle); + tFreeSSubQueryMsg(&msg); + return TSDB_CODE_SUCCESS; } @@ -377,42 +403,41 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int } int32_t code = 0; - SSubQueryMsg *msg = pMsg->pCont; SQWorker *mgmt = (SQWorker *)qWorkerMgmt; qwUpdateTimeInQueue(mgmt, ts, QUERY_QUEUE); QW_STAT_INC(mgmt->stat.msgStat.queryProcessed, 1); - if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { - QW_ELOG("invalid query msg, msg:%p, msgLen:%d", msg, pMsg->contLen); + SSubQueryMsg msg = {0}; + if (tDeserializeSSubQueryMsg(pMsg->pCont, pMsg->contLen, &msg) < 0) { + QW_ELOG("tDeserializeSSubQueryMsg failed, contLen:%d", pMsg->contLen); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - uint64_t sId = msg->sId; - uint64_t qId = msg->queryId; - uint64_t tId = msg->taskId; - int64_t rId = msg->refId; - int32_t eId = msg->execId; + uint64_t sId = msg.sId; + uint64_t qId = msg.queryId; + uint64_t tId = msg.taskId; + int64_t rId = msg.refId; + int32_t eId = msg.execId; SQWMsg qwMsg = {.node = node, - .msg = msg->msg + msg->sqlLen, - .msgLen = msg->phyLen, + .msg = msg.msg, + .msgLen = msg.msgLen, .connInfo = pMsg->info, .msgType = pMsg->msgType}; - qwMsg.msgInfo.explain = msg->explain; - qwMsg.msgInfo.taskType = msg->taskType; - qwMsg.msgInfo.needFetch = msg->needFetch; + qwMsg.msgInfo.explain = msg.explain; + qwMsg.msgInfo.taskType = msg.taskType; + qwMsg.msgInfo.needFetch = msg.needFetch; - char *sql = strndup(msg->msg, msg->sqlLen); QW_SCH_TASK_DLOG("processQuery start, node:%p, type:%s, handle:%p, SQL:%s", node, TMSG_INFO(pMsg->msgType), - pMsg->info.handle, sql); - QW_ERR_JRET(qwProcessQuery(QW_FPARAMS(), &qwMsg, sql)); + pMsg->info.handle, msg.sql); + code = qwProcessQuery(QW_FPARAMS(), &qwMsg, msg.sql); + msg.sql = NULL; + QW_SCH_TASK_DLOG("processQuery end, node:%p, code:%x", node, code); -_return: + tFreeSSubQueryMsg(&msg); - QW_SCH_TASK_DLOG("processQuery end, node:%p, code:%d", node, code); - - return code; + return TSDB_CODE_SUCCESS; } int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts) { @@ -548,28 +573,22 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int6 } int32_t code = 0; - STaskDropReq *msg = pMsg->pCont; SQWorker *mgmt = (SQWorker *)qWorkerMgmt; qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE); QW_STAT_INC(mgmt->stat.msgStat.dropProcessed, 1); - if (NULL == msg || pMsg->contLen < sizeof(*msg)) { - QW_ELOG("invalid task drop msg, msg:%p, msgLen:%d", msg, pMsg->contLen); + STaskDropReq msg = {0}; + if (tDeserializeSTaskDropReq(pMsg->pCont, pMsg->contLen, &msg) < 0) { + QW_ELOG("tDeserializeSTaskDropReq failed, contLen:%d", pMsg->contLen); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - msg->sId = be64toh(msg->sId); - msg->queryId = be64toh(msg->queryId); - msg->taskId = be64toh(msg->taskId); - msg->refId = be64toh(msg->refId); - msg->execId = ntohl(msg->execId); - - uint64_t sId = msg->sId; - uint64_t qId = msg->queryId; - uint64_t tId = msg->taskId; - int64_t rId = msg->refId; - int32_t eId = msg->execId; + uint64_t sId = msg.sId; + uint64_t qId = msg.queryId; + uint64_t tId = msg.taskId; + int64_t rId = msg.refId; + int32_t eId = msg.execId; SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .code = pMsg->code, .connInfo = pMsg->info}; diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 6078a2a3ac..8c5e802ef9 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -114,7 +114,7 @@ void qwtBuildQueryReqMsg(SRpcMsg *queryRpc) { qwtqueryMsg.queryId = htobe64(atomic_add_fetch_64(&qwtTestQueryId, 1)); qwtqueryMsg.sId = htobe64(1); qwtqueryMsg.taskId = htobe64(1); - qwtqueryMsg.phyLen = htonl(100); + qwtqueryMsg.msgLen = htonl(100); qwtqueryMsg.sqlLen = 0; queryRpc->msgType = TDMT_SCH_QUERY; queryRpc->pCont = &qwtqueryMsg; @@ -131,12 +131,29 @@ void qwtBuildFetchReqMsg(SResFetchReq *fetchMsg, SRpcMsg *fetchRpc) { } void qwtBuildDropReqMsg(STaskDropReq *dropMsg, SRpcMsg *dropRpc) { - dropMsg->sId = htobe64(1); - dropMsg->queryId = htobe64(atomic_load_64(&qwtTestQueryId)); - dropMsg->taskId = htobe64(1); + dropMsg->sId = 1; + dropMsg->queryId = atomic_load_64(&qwtTestQueryId); + dropMsg->taskId = 1; + + int32_t msgSize = tSerializeSTaskDropReq(NULL, 0, dropMsg); + if (msgSize < 0) { + return; + } + + char *msg = taosMemoryCalloc(1, msgSize); + if (NULL == msg) { + return; + } + + if (tSerializeSTaskDropReq(msg, msgSize, dropMsg) < 0) { + taosMemoryFree(msg); + return; + } + + dropRpc->msgType = TDMT_SCH_DROP_TASK; - dropRpc->pCont = dropMsg; - dropRpc->contLen = sizeof(STaskDropReq); + dropRpc->pCont = msg; + dropRpc->contLen = msgSize; } int32_t qwtStringToPlan(const char *str, SSubplan **subplan) { diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index a6a2a6c301..47a34ede7e 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1042,30 +1042,40 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, case TDMT_SCH_MERGE_QUERY: { SCH_ERR_RET(schMakeQueryRpcCtx(pJob, pTask, &rpcCtx)); - uint32_t len = strlen(pJob->sql); - msgSize = sizeof(SSubQueryMsg) + pTask->msgLen + len; + SSubQueryMsg qMsg; + qMsg.header.vgId = addr->nodeId; + qMsg.header.contLen = 0; + qMsg.sId = schMgmt.sId; + qMsg.queryId = pJob->queryId; + qMsg.taskId = pTask->taskId; + qMsg.refId = pJob->refId; + qMsg.execId = pTask->execId; + qMsg.msgMask = (pTask->plan->showRewrite) ? QUERY_MSG_MASK_SHOW_REWRITE() : 0; + qMsg.taskType = TASK_TYPE_TEMP; + qMsg.explain = SCH_IS_EXPLAIN_JOB(pJob); + qMsg.needFetch = SCH_TASK_NEED_FETCH(pTask); + qMsg.sqlLen = strlen(pJob->sql); + qMsg.sql = pJob->sql; + qMsg.msgLen = pTask->msgLen; + qMsg.msg = pTask->msg; + + msgSize = tSerializeSSubQueryMsg(NULL, 0, &qMsg); + if (msgSize < 0) { + SCH_TASK_ELOG("tSerializeSSubQueryMsg get size, msgSize:%d", msgSize); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SSubQueryMsg *pMsg = msg; - pMsg->header.vgId = htonl(addr->nodeId); - pMsg->sId = htobe64(schMgmt.sId); - pMsg->queryId = htobe64(pJob->queryId); - pMsg->taskId = htobe64(pTask->taskId); - pMsg->refId = htobe64(pJob->refId); - pMsg->execId = htonl(pTask->execId); - pMsg->taskType = TASK_TYPE_TEMP; - pMsg->explain = SCH_IS_EXPLAIN_JOB(pJob); - pMsg->needFetch = SCH_TASK_NEED_FETCH(pTask); - pMsg->phyLen = htonl(pTask->msgLen); - pMsg->sqlLen = htonl(len); - pMsg->msgMask = htonl((pTask->plan->showRewrite) ? QUERY_MSG_MASK_SHOW_REWRITE() : 0); - - memcpy(pMsg->msg, pJob->sql, len); - memcpy(pMsg->msg + len, pTask->msg, pTask->msgLen); + if (tSerializeSSubQueryMsg(msg, msgSize, &qMsg) < 0) { + SCH_TASK_ELOG("tSerializeSSubQueryMsg failed, msgSize:%d", msgSize); + taosMemoryFree(msg); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } persistHandle = true; SCH_SET_TASK_HANDLE(pTask, rpcAllocHandle()); @@ -1092,22 +1102,32 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, break; } case TDMT_SCH_DROP_TASK: { - msgSize = sizeof(STaskDropReq); + STaskDropReq qMsg; + qMsg.header.vgId = addr->nodeId; + qMsg.header.contLen = 0; + qMsg.sId = schMgmt.sId; + qMsg.queryId = pJob->queryId; + qMsg.taskId = pTask->taskId; + qMsg.refId = pJob->refId; + qMsg.execId = pTask->execId; + + msgSize = tSerializeSTaskDropReq(NULL, 0, &qMsg); + if (msgSize < 0) { + SCH_TASK_ELOG("tSerializeSTaskDropReq get size, msgSize:%d", msgSize); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - STaskDropReq *pMsg = msg; - - pMsg->header.vgId = htonl(addr->nodeId); - - pMsg->sId = htobe64(schMgmt.sId); - pMsg->queryId = htobe64(pJob->queryId); - pMsg->taskId = htobe64(pTask->taskId); - pMsg->refId = htobe64(pJob->refId); - pMsg->execId = htonl(pTask->execId); + if (tSerializeSTaskDropReq(msg, msgSize, &qMsg) < 0) { + SCH_TASK_ELOG("tSerializeSTaskDropReq failed, msgSize:%d", msgSize); + taosMemoryFree(msg); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } break; } case TDMT_SCH_QUERY_HEARTBEAT: { From c8f069c6478fea167e8f77f8c7b3403b283cb4d3 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 18 Nov 2022 09:43:40 +0800 Subject: [PATCH 17/59] change interp linear behavior --- source/libs/executor/inc/tfill.h | 2 + source/libs/executor/src/timewindowoperator.c | 61 +++++++++++-------- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index 7e0866f545..2d8df81dbd 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -36,6 +36,8 @@ typedef struct SFillColInfo { typedef struct SFillLinearInfo { SPoint start; SPoint end; + bool isStartSet; + bool isEndSet; int16_t type; int32_t bytes; } SFillLinearInfo; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 991a509d49..7582107c93 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1991,27 +1991,36 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId); SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, i); - // null data should not be kept since it can not be used to perform interpolation - if (!colDataIsNull_s(pColInfoData, rowIndex)) { - if (pLinearInfo->start.key == INT64_MIN) { + // null value is represented by using key = INT64_MIN for now. + // TODO: optimize to ignore null values for linear interpolation. + if (!pLinearInfo->isStartSet) { + if (!colDataIsNull_s(pColInfoData, rowIndex)) { pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); memcpy(pLinearInfo->start.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); - } else if (pLinearInfo->end.key == INT64_MAX) { + } + pLinearInfo->isStartSet = true; + } else if (!pLinearInfo->isEndSet) { + if (!colDataIsNull_s(pColInfoData, rowIndex)) { + pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); + memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); + } + pLinearInfo->isEndSet = true; + } else { + pLinearInfo->start.key = pLinearInfo->end.key; + memcpy(pLinearInfo->start.val, pLinearInfo->end.val, pLinearInfo->bytes); + + if (!colDataIsNull_s(pColInfoData, rowIndex)) { pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); } else { - pLinearInfo->start.key = pLinearInfo->end.key; - pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); - memcpy(pLinearInfo->start.val, pLinearInfo->end.val, pLinearInfo->bytes); - memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); + pLinearInfo->end.key = INT64_MIN; } } } } -static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, - SSDataBlock* pSrcBlock, int32_t index, bool beforeTs) { +static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock) { int32_t rows = pResBlock->info.rows; blockDataEnsureCapacity(pResBlock, rows + 1); // todo set the correct primary timestamp column @@ -2061,18 +2070,18 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp SPoint start = pLinearInfo->start; SPoint end = pLinearInfo->end; SPoint current = {.key = pSliceInfo->current}; - current.val = taosMemoryCalloc(pLinearInfo->bytes, 1); - // before interp range, do not fill - SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot); - if (colDataIsNull_s(pSrc, index) || (!beforeTs && end.key == INT64_MAX)) { + if (!pLinearInfo->isStartSet || !pLinearInfo->isEndSet) { hasInterp = false; break; - } else if (start.key == INT64_MIN || (beforeTs && end.key == INT64_MAX)) { - //hasInterp = true; - return true; } + if (start.key == INT64_MIN || end.key == INT64_MIN) { + colDataAppendNULL(pDst, rows); + break; + } + + current.val = taosMemoryCalloc(pLinearInfo->bytes, 1); taosGetLinearInterpolationVal(¤t, pLinearInfo->type, &start, &end, pLinearInfo->type); colDataAppend(pDst, rows, (char*)current.val, false); @@ -2130,12 +2139,8 @@ static void addCurrentRowToResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot); if (colDataIsNull_s(pSrc, index)) { - if (pSliceInfo->fillType != TSDB_FILL_LINEAR) { - colDataAppendNULL(pDst, pResBlock->info.rows); - continue; - } else { - return; - } + colDataAppendNULL(pDst, pResBlock->info.rows); + continue; } char* v = colDataGetData(pSrc, index); @@ -2218,9 +2223,11 @@ static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB SFillLinearInfo linearInfo = {0}; linearInfo.start.key = INT64_MIN; - linearInfo.end.key = INT64_MAX; + linearInfo.end.key = INT64_MIN; linearInfo.start.val = taosMemoryCalloc(1, pColInfo->info.bytes); linearInfo.end.val = taosMemoryCalloc(1, pColInfo->info.bytes); + linearInfo.isStartSet = false; + linearInfo.isEndSet = false; linearInfo.type = pColInfo->info.type; linearInfo.bytes = pColInfo->info.bytes; taosArrayPush(pInfo->pLinearInfo, &linearInfo); @@ -2312,7 +2319,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i, false) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { + if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { break; } else { pSliceInfo->current = @@ -2336,7 +2343,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doKeepLinearInfo(pSliceInfo, pBlock, i); while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { - if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i, true) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { + if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { break; } else { pSliceInfo->current = @@ -2365,7 +2372,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { // except for fill(next), fill(linear) while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT && pSliceInfo->fillType != TSDB_FILL_LINEAR) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, NULL, 0, false); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); } From 3f88dbe256c610c7fbac22864b0637de67ae7a1b Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 18 Nov 2022 11:43:03 +0800 Subject: [PATCH 18/59] enh: refact client server messages --- include/common/tmsg.h | 3 +++ source/client/src/clientTmq.c | 2 +- source/common/src/tmsg.c | 37 +++++++++++++++++++++++++++++++++ source/libs/qworker/src/qwMsg.c | 32 +++++++++++++++++++++------- 4 files changed, 66 insertions(+), 8 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9b7f3bff4c..69d272386e 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1741,6 +1741,9 @@ typedef struct { int32_t tSerializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq); int32_t tDeserializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq); +int32_t tSerializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp); +int32_t tDeserializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp); + typedef struct { int32_t code; diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 1fe89bcafd..28b41b97c5 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -756,7 +756,7 @@ void tmqSendHbReq(void* param, void* tmrId) { } sendInfo->msgInfo = (SDataBuf){ .pData = pReq, - .len = sizeof(SMqHbReq), + .len = tlen, .handle = NULL, }; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 8ddb60655e..1164c88477 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4778,6 +4778,43 @@ int32_t tDeserializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq) return 0; } +int32_t tSerializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + if (tStartEncode(&encoder) < 0) return -1; + + if (tEncodeI32(&encoder, pRsp->code) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->tbFName) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->sversion) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->tversion) < 0) return -1; + if (tEncodeI64(&encoder, pRsp->affectedRows) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + + return tlen; +} + +int32_t tDeserializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, (char *)buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + + if (tDecodeI32(&decoder, &pRsp->code) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->tbFName) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->sversion) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->tversion) < 0) return -1; + if (tDecodeI64(&decoder, &pRsp->affectedRows) < 0) return -1; + + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + int32_t tSerializeSSchedulerHbReq(void *buf, int32_t bufLen, SSchedulerHbReq *pReq) { int32_t headLen = sizeof(SMsgHead); diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 82e71bc8d0..71f24f8acd 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -65,19 +65,37 @@ int32_t qwBuildAndSendErrorRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t c int32_t qwBuildAndSendQueryRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code, SQWTaskCtx *ctx) { STbVerInfo *tbInfo = ctx ? &ctx->tbInfo : NULL; int64_t affectedRows = ctx ? ctx->affectedRows : 0; - SQueryTableRsp *pRsp = (SQueryTableRsp *)rpcMallocCont(sizeof(SQueryTableRsp)); - pRsp->code = htonl(code); - pRsp->affectedRows = htobe64(affectedRows); + SQueryTableRsp rsp = {0}; + rsp.code = code; + rsp.affectedRows = affectedRows; + if (tbInfo) { - strcpy(pRsp->tbFName, tbInfo->tbFName); - pRsp->sversion = htonl(tbInfo->sversion); - pRsp->tversion = htonl(tbInfo->tversion); + strcpy(rsp.tbFName, tbInfo->tbFName); + rsp.sversion = tbInfo->sversion; + rsp.tversion = tbInfo->tversion; + } + + int32_t msgSize = tSerializeSQueryTableRsp(NULL, 0, &rsp); + if (msgSize < 0) { + qError("tSerializeSQueryTableRsp failed"); + QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + void *pRsp = rpcMallocCont(msgSize); + if (NULL == pRsp) { + qError("rpcMallocCont %d failed", msgSize); + QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + if (tSerializeSQueryTableRsp(pRsp, msgSize, &rsp) < 0) { + qError("tSerializeSQueryTableRsp %d failed", msgSize); + QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } SRpcMsg rpcRsp = { .msgType = rspType, .pCont = pRsp, - .contLen = sizeof(*pRsp), + .contLen = msgSize, .code = code, .info = *pConn, }; From 70eb1f445e0a33a9815249aa3999570953ead33d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 18 Nov 2022 13:18:23 +0800 Subject: [PATCH 19/59] fix: fix compile error --- source/libs/qworker/test/qworkerTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 8c5e802ef9..8a48977c77 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -140,7 +140,7 @@ void qwtBuildDropReqMsg(STaskDropReq *dropMsg, SRpcMsg *dropRpc) { return; } - char *msg = taosMemoryCalloc(1, msgSize); + char *msg = (char*)taosMemoryCalloc(1, msgSize); if (NULL == msg) { return; } From 7291f2bd5adb979f7a8bfa449c4765000a037340 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 18 Nov 2022 09:43:40 +0800 Subject: [PATCH 20/59] change interp linear behavior --- source/libs/executor/src/timewindowoperator.c | 13 +- tests/system-test/2-query/interp.py | 482 +++++++++++++----- 2 files changed, 370 insertions(+), 125 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 7582107c93..5e1a5d1ec4 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2020,7 +2020,7 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo } -static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock) { +static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, bool beforeTs) { int32_t rows = pResBlock->info.rows; blockDataEnsureCapacity(pResBlock, rows + 1); // todo set the correct primary timestamp column @@ -2071,6 +2071,11 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp SPoint end = pLinearInfo->end; SPoint current = {.key = pSliceInfo->current}; + // do not interpolate before ts range, only increate pSliceInfo->current + if (beforeTs && !pLinearInfo->isEndSet) { + return true; + } + if (!pLinearInfo->isStartSet || !pLinearInfo->isEndSet) { hasInterp = false; break; @@ -2319,7 +2324,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { + if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, false) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { break; } else { pSliceInfo->current = @@ -2343,7 +2348,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doKeepLinearInfo(pSliceInfo, pBlock, i); while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { - if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { + if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, true) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { break; } else { pSliceInfo->current = @@ -2372,7 +2377,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { // except for fill(next), fill(linear) while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT && pSliceInfo->fillType != TSDB_FILL_LINEAR) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, false); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); } diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index fa5c35f782..b5bf6f9dbe 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -908,224 +908,464 @@ class TDTestCase: tdSql.checkData(0, 0, 0) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:03') every(1s) fill(linear)") - tdSql.checkRows(1) + tdSql.checkRows(4) tdSql.checkData(0, 0, 0) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:05') every(1s) fill(linear)") - tdSql.checkRows(1) + tdSql.checkRows(6) tdSql.checkData(0, 0, 0) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:08') every(1s) fill(linear)") tdSql.checkRows(9) - tdSql.checkData(0, 0, 0) - tdSql.checkData(1, 0, 1) - tdSql.checkData(2, 0, 2) - tdSql.checkData(3, 0, 3) - tdSql.checkData(4, 0, 4) - tdSql.checkData(5, 0, 5) - tdSql.checkData(6, 0, 6) - tdSql.checkData(7, 0, 7) - tdSql.checkData(8, 0, 8) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:01', '2020-02-02 00:00:03') every(1s) fill(linear)") - tdSql.checkRows(0) + tdSql.checkRows(3) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:03', '2020-02-02 00:00:08') every(1s) fill(linear)") tdSql.checkRows(6) - tdSql.checkData(0, 0, 3) - tdSql.checkData(1, 0, 4) - tdSql.checkData(2, 0, 5) - tdSql.checkData(3, 0, 6) - tdSql.checkData(4, 0, 7) - tdSql.checkData(5, 0, 8) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:05', '2020-02-02 00:00:10') every(1s) fill(linear)") - tdSql.checkRows(1) - tdSql.checkData(0, 0, 10) + tdSql.checkRows(6) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, 10) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:05', '2020-02-02 00:00:15') every(1s) fill(linear)") - tdSql.checkRows(1) - tdSql.checkData(0, 0, 10) + tdSql.checkRows(11) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, 10) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:05', '2020-02-02 00:00:18') every(1s) fill(linear)") - tdSql.checkRows(9) - tdSql.checkData(0, 0, 10) - tdSql.checkData(1, 0, 11) - tdSql.checkData(2, 0, 12) - tdSql.checkData(3, 0, 13) - tdSql.checkData(4, 0, 14) - tdSql.checkData(5, 0, 15) - tdSql.checkData(6, 0, 16) - tdSql.checkData(7, 0, 17) - tdSql.checkData(8, 0, 18) + tdSql.checkRows(14) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, 10) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, None) + tdSql.checkData(11, 0, None) + tdSql.checkData(12, 0, None) + tdSql.checkData(13, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:05', '2020-02-02 00:00:20') every(1s) fill(linear)") - tdSql.checkRows(11) - tdSql.checkData(0, 0, 10) - tdSql.checkData(1, 0, 11) - tdSql.checkData(2, 0, 12) - tdSql.checkData(3, 0, 13) - tdSql.checkData(4, 0, 14) - tdSql.checkData(5, 0, 15) - tdSql.checkData(6, 0, 16) - tdSql.checkData(7, 0, 17) - tdSql.checkData(8, 0, 18) - tdSql.checkData(9, 0, 19) - tdSql.checkData(10, 0, 20) + tdSql.checkRows(16) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, 10) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, None) + tdSql.checkData(11, 0, None) + tdSql.checkData(12, 0, None) + tdSql.checkData(13, 0, None) + tdSql.checkData(14, 0, None) + tdSql.checkData(15, 0, 20) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:09', '2020-02-02 00:00:11') every(1s) fill(linear)") - tdSql.checkRows(1) - tdSql.checkData(0, 0, 10) + tdSql.checkRows(3) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 10) + tdSql.checkData(2, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:10', '2020-02-02 00:00:15') every(1s) fill(linear)") - tdSql.checkRows(1) + tdSql.checkRows(6) tdSql.checkData(0, 0, 10) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:12', '2020-02-02 00:00:13') every(1s) fill(linear)") - tdSql.checkRows(0) + tdSql.checkRows(2) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:12', '2020-02-02 00:00:15') every(1s) fill(linear)") - tdSql.checkRows(0) + tdSql.checkRows(4) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:12', '2020-02-02 00:00:18') every(1s) fill(linear)") tdSql.checkRows(7) - tdSql.checkData(0, 0, 12) - tdSql.checkData(1, 0, 13) - tdSql.checkData(2, 0, 14) - tdSql.checkData(3, 0, 15) - tdSql.checkData(4, 0, 16) - tdSql.checkData(5, 0, 17) - tdSql.checkData(6, 0, 18) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, None) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:30', '2020-02-02 00:00:40') every(1s) fill(linear)") tdSql.checkRows(11) tdSql.checkData(0, 0, 30) + tdSql.checkData(1, 0, 31) + tdSql.checkData(2, 0, 32) + tdSql.checkData(3, 0, 33) + tdSql.checkData(4, 0, 34) + tdSql.checkData(5, 0, 35) + tdSql.checkData(6, 0, 36) + tdSql.checkData(7, 0, 37) + tdSql.checkData(8, 0, 38) + tdSql.checkData(9, 0, 39) tdSql.checkData(10, 0, 40) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:25', '2020-02-02 00:00:45') every(1s) fill(linear)") - tdSql.checkRows(11) - tdSql.checkData(0, 0, 30) - tdSql.checkData(10, 0, 40) - - tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:25', '2020-02-02 00:00:45') every(1s) fill(linear)") - tdSql.checkRows(11) - tdSql.checkData(0, 0, 30) - tdSql.checkData(10, 0, 40) + tdSql.checkRows(21) + tdSql.checkData(5, 0, 30) + tdSql.checkData(6, 0, 31) + tdSql.checkData(7, 0, 32) + tdSql.checkData(8, 0, 33) + tdSql.checkData(9, 0, 34) + tdSql.checkData(10, 0, 35) + tdSql.checkData(11, 0, 36) + tdSql.checkData(12, 0, 37) + tdSql.checkData(13, 0, 38) + tdSql.checkData(14, 0, 39) + tdSql.checkData(15, 0, 40) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:20', '2020-02-02 00:00:40') every(1s) fill(linear)") tdSql.checkRows(21) tdSql.checkData(0, 0, 20) + tdSql.checkData(10, 0, 30) + tdSql.checkData(11, 0, 31) + tdSql.checkData(12, 0, 32) + tdSql.checkData(13, 0, 33) + tdSql.checkData(14, 0, 34) + tdSql.checkData(15, 0, 35) + tdSql.checkData(16, 0, 36) + tdSql.checkData(17, 0, 37) + tdSql.checkData(18, 0, 38) + tdSql.checkData(19, 0, 39) tdSql.checkData(20, 0, 40) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:30', '2020-02-02 00:00:50') every(1s) fill(linear)") tdSql.checkRows(21) tdSql.checkData(0, 0, 30) + tdSql.checkData(1, 0, 31) + tdSql.checkData(2, 0, 32) + tdSql.checkData(3, 0, 33) + tdSql.checkData(4, 0, 34) + tdSql.checkData(5, 0, 35) + tdSql.checkData(6, 0, 36) + tdSql.checkData(7, 0, 37) + tdSql.checkData(8, 0, 38) + tdSql.checkData(9, 0, 39) + tdSql.checkData(10, 0, 40) tdSql.checkData(20, 0, 50) tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-02 00:00:20', '2020-02-02 00:00:50') every(1s) fill(linear)") tdSql.checkRows(31) tdSql.checkData(0, 0, 20) + tdSql.checkData(10, 0, 30) + tdSql.checkData(11, 0, 31) + tdSql.checkData(12, 0, 32) + tdSql.checkData(13, 0, 33) + tdSql.checkData(14, 0, 34) + tdSql.checkData(15, 0, 35) + tdSql.checkData(16, 0, 36) + tdSql.checkData(17, 0, 37) + tdSql.checkData(18, 0, 38) + tdSql.checkData(19, 0, 39) + tdSql.checkData(20, 0, 40) tdSql.checkData(30, 0, 50) # check c1 tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:05') every(1s) fill(linear)") - tdSql.checkRows(0) + tdSql.checkRows(6) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:05') every(1s) fill(linear)") - tdSql.checkRows(0) + tdSql.checkRows(6) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:08') every(1s) fill(linear)") - tdSql.checkRows(0) + tdSql.checkRows(9) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:10') every(1s) fill(linear)") - tdSql.checkRows(1) - tdSql.checkData(0, 0, 10) + tdSql.checkRows(11) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, 10) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:15') every(1s) fill(linear)") - tdSql.checkRows(1) - tdSql.checkData(0, 0, 10) + tdSql.checkRows(16) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, 10) + tdSql.checkData(11, 0, None) + tdSql.checkData(12, 0, None) + tdSql.checkData(13, 0, None) + tdSql.checkData(14, 0, None) + tdSql.checkData(15, 0, None) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:20') every(1s) fill(linear)") - tdSql.checkRows(1) - tdSql.checkData(0, 0, 10) + tdSql.checkRows(21) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, 10) + tdSql.checkData(11, 0, None) + tdSql.checkData(12, 0, None) + tdSql.checkData(13, 0, None) + tdSql.checkData(14, 0, None) + tdSql.checkData(15, 0, None) + tdSql.checkData(16, 0, None) + tdSql.checkData(17, 0, None) + tdSql.checkData(18, 0, None) + tdSql.checkData(19, 0, None) + tdSql.checkData(20, 0, None) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:25') every(1s) fill(linear)") - tdSql.checkRows(1) - tdSql.checkData(0, 0, 10) + tdSql.checkRows(26) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, 10) + tdSql.checkData(11, 0, None) + tdSql.checkData(12, 0, None) + tdSql.checkData(13, 0, None) + tdSql.checkData(14, 0, None) + tdSql.checkData(15, 0, None) + tdSql.checkData(16, 0, None) + tdSql.checkData(17, 0, None) + tdSql.checkData(18, 0, None) + tdSql.checkData(19, 0, None) + tdSql.checkData(20, 0, None) + tdSql.checkData(21, 0, None) + tdSql.checkData(22, 0, None) + tdSql.checkData(23, 0, None) + tdSql.checkData(24, 0, None) + tdSql.checkData(25, 0, None) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:30') every(1s) fill(linear)") - tdSql.checkRows(21) - tdSql.checkData(0, 0, 10) - tdSql.checkData(1, 0, 11) - tdSql.checkData(2, 0, 12) - tdSql.checkData(3, 0, 13) - tdSql.checkData(4, 0, 14) - tdSql.checkData(5, 0, 15) - tdSql.checkData(6, 0, 16) - tdSql.checkData(7, 0, 17) - tdSql.checkData(8, 0, 18) - tdSql.checkData(9, 0, 19) - tdSql.checkData(10, 0, 20) - tdSql.checkData(11, 0, 21) - tdSql.checkData(12, 0, 22) - tdSql.checkData(13, 0, 23) - tdSql.checkData(14, 0, 24) - tdSql.checkData(15, 0, 25) - tdSql.checkData(16, 0, 26) - tdSql.checkData(17, 0, 27) - tdSql.checkData(18, 0, 28) - tdSql.checkData(19, 0, 29) - tdSql.checkData(20, 0, 30) + tdSql.checkRows(31) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, 10) + tdSql.checkData(11, 0, None) + tdSql.checkData(12, 0, None) + tdSql.checkData(13, 0, None) + tdSql.checkData(14, 0, None) + tdSql.checkData(15, 0, None) + tdSql.checkData(16, 0, None) + tdSql.checkData(17, 0, None) + tdSql.checkData(18, 0, None) + tdSql.checkData(19, 0, None) + tdSql.checkData(20, 0, None) + tdSql.checkData(21, 0, None) + tdSql.checkData(22, 0, None) + tdSql.checkData(23, 0, None) + tdSql.checkData(24, 0, None) + tdSql.checkData(25, 0, None) + tdSql.checkData(26, 0, None) + tdSql.checkData(27, 0, None) + tdSql.checkData(28, 0, None) + tdSql.checkData(29, 0, None) + tdSql.checkData(30, 0, 30) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:35') every(1s) fill(linear)") - tdSql.checkRows(21) - tdSql.checkData(0, 0, 10) - tdSql.checkData(20, 0, 30) + tdSql.checkRows(36) + tdSql.checkData(10, 0, 10) + tdSql.checkData(30, 0, 30) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:40') every(1s) fill(linear)") - tdSql.checkRows(31) + tdSql.checkRows(41) tdSql.checkData(0, 0, 10) - tdSql.checkData(30, 0, 40) + tdSql.checkData(30, 0, 30) + tdSql.checkData(40, 0, 40) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:45') every(1s) fill(linear)") - tdSql.checkRows(36) + tdSql.checkRows(46) tdSql.checkData(0, 0, 10) - tdSql.checkData(35, 0, 45) + tdSql.checkData(30, 0, 30) + tdSql.checkData(40, 0, 40) + tdSql.checkData(41, 0, 41) + tdSql.checkData(42, 0, 42) + tdSql.checkData(43, 0, 43) + tdSql.checkData(44, 0, 44) + tdSql.checkData(45, 0, 45) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:50') every(1s) fill(linear)") - tdSql.checkRows(36) - tdSql.checkData(0, 0, 10) - tdSql.checkData(35, 0, 45) - - tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:55') every(1s) fill(linear)") - tdSql.checkRows(36) - tdSql.checkData(0, 0, 10) - tdSql.checkData(35, 0, 45) - - tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(linear)") tdSql.checkRows(51) tdSql.checkData(0, 0, 10) - tdSql.checkData(50, 0, 60) + tdSql.checkData(30, 0, 30) + tdSql.checkData(40, 0, 40) + tdSql.checkData(41, 0, 41) + tdSql.checkData(42, 0, 42) + tdSql.checkData(43, 0, 43) + tdSql.checkData(44, 0, 44) + tdSql.checkData(45, 0, 45) + + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:55') every(1s) fill(linear)") + tdSql.checkRows(56) + tdSql.checkData(0, 0, 10) + tdSql.checkData(30, 0, 30) + tdSql.checkData(40, 0, 40) + tdSql.checkData(41, 0, 41) + tdSql.checkData(42, 0, 42) + tdSql.checkData(43, 0, 43) + tdSql.checkData(44, 0, 44) + tdSql.checkData(45, 0, 45) + + + tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(linear)") + tdSql.checkRows(60) + tdSql.checkData(0, 0, 10) + tdSql.checkData(30, 0, 30) + tdSql.checkData(40, 0, 40) + tdSql.checkData(41, 0, 41) + tdSql.checkData(42, 0, 42) + tdSql.checkData(43, 0, 43) + tdSql.checkData(44, 0, 44) + tdSql.checkData(45, 0, 45) + tdSql.checkData(60, 0, 60) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:40', '2020-02-02 00:00:45') every(1s) fill(linear)") tdSql.checkRows(6) tdSql.checkData(0, 0, 40) + tdSql.checkData(1, 0, 41) + tdSql.checkData(2, 0, 42) + tdSql.checkData(3, 0, 43) + tdSql.checkData(4, 0, 44) tdSql.checkData(5, 0, 45) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:35', '2020-02-02 00:00:50') every(1s) fill(linear)") - tdSql.checkRows(6) - tdSql.checkData(0, 0, 40) - tdSql.checkData(5, 0, 45) + tdSql.checkRows(16) + tdSql.checkData(5, 0, 40) + tdSql.checkData(6, 0, 41) + tdSql.checkData(7, 0, 42) + tdSql.checkData(8, 0, 43) + tdSql.checkData(9, 0, 44) + tdSql.checkData(10, 0, 45) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:35', '2020-02-02 00:00:55') every(1s) fill(linear)") - tdSql.checkRows(6) - tdSql.checkData(0, 0, 40) - tdSql.checkData(5, 0, 45) + tdSql.checkRows(21) + tdSql.checkData(5, 0, 40) + tdSql.checkData(6, 0, 41) + tdSql.checkData(7, 0, 42) + tdSql.checkData(8, 0, 43) + tdSql.checkData(9, 0, 44) + tdSql.checkData(10, 0, 45) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:30', '2020-02-02 00:01:00') every(1s) fill(linear)") tdSql.checkRows(31) tdSql.checkData(0, 0, 30) + tdSql.checkData(10, 0, 40) + tdSql.checkData(11, 0, 41) + tdSql.checkData(12, 0, 42) + tdSql.checkData(13, 0, 43) + tdSql.checkData(14, 0, 44) + tdSql.checkData(15, 0, 45) tdSql.checkData(30, 0, 60) tdLog.printNoPrefix("==========step11:test multi-interp cases") From 9f5a406441d8dba1bc12a762fe62bfa6e4e4203b Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 18 Nov 2022 14:59:42 +0800 Subject: [PATCH 21/59] update --- tests/pytest/crash_gen/crash_gen_main.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 90d27c3b8d..2aa499285f 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -2039,7 +2039,7 @@ class TdSuperTable: conf.set("enable.auto.commit", "true") def tmq_commit_cb_print(tmq, resp, offset, param=None): print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}") - conf.set_auto_commit_cb(tmq_commit_cb_print, None) + # conf.set_auto_commit_cb(tmq_commit_cb_print, None) consumer = conf.new_consumer() topic_list = TaosTmqList() for topic in current_topic_list: @@ -2049,13 +2049,12 @@ class TdSuperTable: except TmqError as e : pass - # consumer work only 30 sec + # consumer with random work life time_start = time.time() while 1: res = consumer.poll(1000) - if time.time() - time_start >5 : + if time.time() - time_start >random.randint(5,50) : break - # time.sleep(10) try: consumer.unsubscribe() except TmqError as e : From 0e6481a4a0b108a34715a2a3aef6b479a9b6af5e Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 18 Nov 2022 15:10:23 +0800 Subject: [PATCH 22/59] update --- tests/pytest/crash_gen/crash_gen_main.py | 34 +++++++++--------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 2aa499285f..49b68965d0 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1978,15 +1978,8 @@ class TdSuperTable: def drop(self, dbc, skipCheck = False): dbName = self._dbName if self.exists(dbc) : # if myself exists - fullTableName = dbName + '.' + self._stName - - try: - dbc.execute("DROP TABLE {}".format(fullTableName)) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [1011,0x3F3,0x03f3,0x2662]: # table doesn't exist # Stream must be dropped first, SQL: DROP TABLE db_0.fs_table - pass - + fullTableName = dbName + '.' + self._stName + dbc.execute("DROP TABLE {}".format(fullTableName)) else: if not skipCheck: raise CrashGenError("Cannot drop non-existant super table: {}".format(self._stName)) @@ -2004,16 +1997,9 @@ class TdSuperTable: fullTableName = dbName + '.' + self._stName if dbc.existsSuperTable(self._stName): - if dropIfExists: - try: - dbc.execute("DROP TABLE {}".format(fullTableName)) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [1011,0x3F3,0x03f3,0x2662]: # table doesn't exist # Stream must be dropped first, SQL: DROP TABLE db_0.fs_table - pass - - pass - # dbc.execute("DROP TABLE {}".format(fullTableName)) + if dropIfExists: + dbc.execute("DROP TABLE {}".format(fullTableName)) + else: # error raise CrashGenError("Cannot create super table, already exists: {}".format(self._stName)) @@ -2434,11 +2420,15 @@ class TaskDropSuperTable(StateTransitionTask): for i in tblSeq: regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) try: - self.execWtSql(wt, "drop table {}.{}". format(self._db.getName(), regTableName)) # nRows always 0, like MySQL except taos.error.ProgrammingError as err: - pass + # correcting for strange error number scheme + errno2 = Helper.convertErrno(err.errno) + if (errno2 in [0x362]): # mnode invalid table name + isSuccess = False + Logging.debug("[DB] Acceptable error when dropping a table") + continue # try to delete next regular table if (not tickOutput): @@ -2956,6 +2946,8 @@ class ThreadStacks: # stack info for all threads print(" {}".format(frame.line)) stackFrame += 1 print("-----> End of Thread Info ----->\n") + if self.current_time-last_sql_commit_time >100: # dead lock occured + print("maybe dead locked of thread {} ".format(shortTid)) class ClientManager: def __init__(self): From 4703ccc4bcc4b8c967948b0f4d698079915c4706 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 18 Nov 2022 15:53:58 +0800 Subject: [PATCH 23/59] fix fill prev/next error --- source/libs/executor/src/timewindowoperator.c | 28 +++++---- tests/system-test/2-query/interp.py | 60 ++++++++++++++----- 2 files changed, 64 insertions(+), 24 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 5e1a5d1ec4..0dd3251b0c 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1945,10 +1945,8 @@ static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - // null data should not be kept since it can not be used to perform interpolation - if (!colDataIsNull_s(pColInfoData, i)) { - SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, i); - + SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, i); + if (!colDataIsNull_s(pColInfoData, rowIndex)) { pkey->isNull = false; char* val = colDataGetData(pColInfoData, rowIndex); if (!IS_VAR_DATA_TYPE(pkey->type)) { @@ -1956,6 +1954,8 @@ static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock } else { memcpy(pkey->pData, val, varDataLen(val)); } + } else { + pkey->isNull = true; } } @@ -1967,10 +1967,8 @@ static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - // null data should not be kept since it can not be used to perform interpolation - if (!colDataIsNull_s(pColInfoData, i)) { - SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, i); - + SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, i); + if (!colDataIsNull_s(pColInfoData, rowIndex)) { pkey->isNull = false; char* val = colDataGetData(pColInfoData, rowIndex); if (!IS_VAR_DATA_TYPE(pkey->type)) { @@ -1978,6 +1976,8 @@ static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock } else { memcpy(pkey->pData, val, varDataLen(val)); } + } else { + pkey->isNull = true; } } @@ -2100,7 +2100,11 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot); - colDataAppend(pDst, rows, pkey->pData, false); + if (pkey->isNull == false) { + colDataAppend(pDst, rows, pkey->pData, false); + } else { + colDataAppendNULL(pDst, rows); + } break; } @@ -2111,7 +2115,11 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, srcSlot); - colDataAppend(pDst, rows, pkey->pData, false); + if (pkey->isNull == false) { + colDataAppend(pDst, rows, pkey->pData, false); + } else { + colDataAppendNULL(pDst, rows); + } break; } diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index b5bf6f9dbe..a0ac4ab03b 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -39,8 +39,6 @@ class TDTestCase: tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:10', 10, 10, 10, 10, 10.0, 10.0, true, 'varchar', 'nchar')") tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:15', 15, 15, 15, 15, 15.0, 15.0, true, 'varchar', 'nchar')") - tdSql.execute(f"insert into {dbname}.{tbname} (ts) values (now)") - tdLog.printNoPrefix("==========step3:fill null") ## {. . .} @@ -248,7 +246,7 @@ class TDTestCase: ## {. . .} tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(next)") - tdSql.checkRows(13) + tdSql.checkRows(12) tdSql.checkData(0, 0, 5) tdSql.checkData(1, 0, 5) tdSql.checkData(2, 0, 10) @@ -298,14 +296,14 @@ class TDTestCase: ## ..{.} tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:13', '2020-02-01 00:00:17') every(1s) fill(next)") - tdSql.checkRows(5) + tdSql.checkRows(3) tdSql.checkData(0, 0, 15) tdSql.checkData(1, 0, 15) tdSql.checkData(2, 0, 15) ## ... {} tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(next)") - tdSql.checkRows(4) + tdSql.checkRows(0) tdLog.printNoPrefix("==========step7:fill linear") @@ -513,7 +511,7 @@ class TDTestCase: tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(next)") - tdSql.checkRows(13) + tdSql.checkRows(12) tdSql.checkCols(2) tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') @@ -925,7 +923,7 @@ class TDTestCase: tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:08') every(1s) fill(linear)") tdSql.checkRows(9) - tdSql.checkData(0, 0, None) + tdSql.checkData(0, 0, 0) tdSql.checkData(1, 0, None) tdSql.checkData(2, 0, None) tdSql.checkData(3, 0, None) @@ -1274,18 +1272,18 @@ class TDTestCase: tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:35') every(1s) fill(linear)") tdSql.checkRows(36) - tdSql.checkData(10, 0, 10) + tdSql.checkData(10, 0, 10) tdSql.checkData(30, 0, 30) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:40') every(1s) fill(linear)") tdSql.checkRows(41) - tdSql.checkData(0, 0, 10) + tdSql.checkData(10, 0, 10) tdSql.checkData(30, 0, 30) tdSql.checkData(40, 0, 40) tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:45') every(1s) fill(linear)") tdSql.checkRows(46) - tdSql.checkData(0, 0, 10) + tdSql.checkData(10, 0, 10) tdSql.checkData(30, 0, 30) tdSql.checkData(40, 0, 40) tdSql.checkData(41, 0, 41) @@ -1296,7 +1294,7 @@ class TDTestCase: tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:50') every(1s) fill(linear)") tdSql.checkRows(51) - tdSql.checkData(0, 0, 10) + tdSql.checkData(10, 0, 10) tdSql.checkData(30, 0, 30) tdSql.checkData(40, 0, 40) tdSql.checkData(41, 0, 41) @@ -1308,7 +1306,7 @@ class TDTestCase: tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:00:55') every(1s) fill(linear)") tdSql.checkRows(56) - tdSql.checkData(0, 0, 10) + tdSql.checkData(10, 0, 10) tdSql.checkData(30, 0, 30) tdSql.checkData(40, 0, 40) tdSql.checkData(41, 0, 41) @@ -1319,8 +1317,8 @@ class TDTestCase: tdSql.query(f"select interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(linear)") - tdSql.checkRows(60) - tdSql.checkData(0, 0, 10) + tdSql.checkRows(61) + tdSql.checkData(10, 0, 10) tdSql.checkData(30, 0, 30) tdSql.checkData(40, 0, 40) tdSql.checkData(41, 0, 41) @@ -1368,6 +1366,40 @@ class TDTestCase: tdSql.checkData(15, 0, 45) tdSql.checkData(30, 0, 60) + # two interps + tdSql.query(f"select interp(c0),interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(linear)") + tdSql.checkRows(61) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 0) + tdSql.checkData(10, 0, 10) + tdSql.checkData(20, 0, 20) + tdSql.checkData(30, 0, 30) + tdSql.checkData(31, 0, 31) + tdSql.checkData(32, 0, 32) + tdSql.checkData(33, 0, 33) + tdSql.checkData(34, 0, 34) + tdSql.checkData(35, 0, 35) + tdSql.checkData(36, 0, 36) + tdSql.checkData(37, 0, 37) + tdSql.checkData(38, 0, 38) + tdSql.checkData(39, 0, 39) + tdSql.checkData(40, 0, 40) + tdSql.checkData(50, 0, 50) + tdSql.checkData(60, 0, 55) + + tdSql.checkData(0, 1, None) + tdSql.checkData(10, 1, 10) + tdSql.checkData(20, 1, None) + tdSql.checkData(30, 1, 30) + tdSql.checkData(40, 1, 40) + tdSql.checkData(41, 1, 41) + tdSql.checkData(42, 1, 42) + tdSql.checkData(43, 1, 43) + tdSql.checkData(44, 1, 44) + tdSql.checkData(45, 1, 45) + tdSql.checkData(50, 1, None) + tdSql.checkData(60, 1, 60) + tdLog.printNoPrefix("==========step11:test multi-interp cases") tdSql.query(f"select interp(c0),interp(c1),interp(c2),interp(c3) from {dbname}.{tbname} range('2020-02-09 00:00:05', '2020-02-13 00:00:05') every(1d) fill(null)") tdSql.checkRows(5) From 7ff95b426e3ea3448b6c124116a7e0e6f8b4068c Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Fri, 18 Nov 2022 15:56:08 +0800 Subject: [PATCH 24/59] feat: send alert in script for cloud version --- packaging/docker/DockerfileCloud | 3 + packaging/docker/run.sh | 182 +++++++++++++++++++++++++++++-- 2 files changed, 178 insertions(+), 7 deletions(-) mode change 100644 => 100755 packaging/docker/run.sh diff --git a/packaging/docker/DockerfileCloud b/packaging/docker/DockerfileCloud index 2b060c1b91..21e387bab3 100644 --- a/packaging/docker/DockerfileCloud +++ b/packaging/docker/DockerfileCloud @@ -7,6 +7,9 @@ ARG dirName ARG cpuType RUN echo ${pkgFile} && echo ${dirName} +RUN apt update +RUN apt install -y curl + COPY ${pkgFile} /root/ ENV TINI_VERSION v0.19.0 ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${cpuType} /tini diff --git a/packaging/docker/run.sh b/packaging/docker/run.sh old mode 100644 new mode 100755 index 2700b0b060..b5d6b011ea --- a/packaging/docker/run.sh +++ b/packaging/docker/run.sh @@ -1,16 +1,160 @@ #!/bin/bash + TAOS_RUN_TAOSBENCHMARK_TEST_ONCE=0 +#ADMIN_URL=${ADMIN_URL:-http://172.26.10.84:10001} +TAOSD_STARTUP_TIMEOUT_SECOND=${TAOSD_STARTUP_TIMEOUT_SECOND:-160} +TAOS_TIMEOUT_SECOND=${TAOS_TIMEOUT_SECOND:-5} +BACKUP_CORE_FOLDER=/data/corefile +ALERT_URL=app/system/alert/add + +echo "ADMIN_URL: ${ADMIN_URL}" +echo "TAOS_TIMEOUT_SECOND: ${TAOS_TIMEOUT_SECOND}" + +function set_service_state() { + #echo "set service state: $1, $2" + service_state="$1" + service_msg="$2" +} +set_service_state "init" "ok" +app_name=`hostname |cut -d\- -f1` + +function check_taosd() { + timeout $TAOS_TIMEOUT_SECOND taos -s "show databases;" >/dev/null + local ret=$? + if [ $ret -ne 0 ]; then + echo "`date` check taosd error $ret" + if [ "x$1" != "xignore" ]; then + set_service_state "error" "taos check failed $ret" + fi + else + set_service_state "ready" "ok" + fi +} +function post_error_msg() { + if [ ! -z "${ADMIN_URL}" ]; then + taos_version=`taos --version` + echo "app_name: ${app_name}" + echo "service_state: ${service_state}" + echo "`date` service_msg: ${service_msg}" + echo "${taos_version}" + curl -X POST -H "Content-Type: application/json" \ + -d"{\"appName\":\"${app_name}\",\ + \"alertLevel\":\"${service_state}\",\ + \"taosVersion\":\"${taos_version}\",\ + \"alertMsg\":\"${service_msg}\"}" \ + ${ADMIN_URL}/${ALERT_URL} + fi +} +function check_taosd_exit_type() { + local core_pattern=`cat /proc/sys/kernel/core_pattern` + echo "$core_pattern" | grep -q "^/" + if [ $? -eq 0 ]; then + core_folder=`dirname $core_pattern` + core_prefix=`basename $core_pattern | sed "s/%.*//"` + else + core_folder=`pwd` + core_prefix="$core_pattern" + fi + local core_files=`ls $core_folder | grep "^${core_prefix}"` + if [ ! -z "$core_files" ]; then + # move core files to another folder + mkdir -p ${BACKUP_CORE_FOLDER} + mv ${core_folder}/${core_prefix}* ${BACKUP_CORE_FOLDER}/ + set_service_state "error" "taosd exit with core file" + else + set_service_state "error" "taosd exit without core file" + fi +} +disk_usage_level=(60 80 99) +current_disk_level=0 +disk_state="ok" +disk_msg="ok" +get_usage_ok="yes" +function post_disk_error_msg() { + if [ ! -z "${ADMIN_URL}" ]; then + taos_version=`taos --version` + echo "app_name: ${app_name}" + echo "disk_state: ${disk_state}" + echo "`date` disk_msg: ${disk_msg}" + echo "${taos_version}" + curl -X POST -H "Content-Type: application/json" \ + -d"{\"appName\":\"${app_name}\",\ + \"alertLevel\":\"${disk_state}\",\ + \"taosVersion\":\"${taos_version}\",\ + \"alertMsg\":\"${disk_msg}\"}" \ + ${ADMIN_URL}/${ALERT_URL} + fi +} +function check_disk() { + local folder=`cat /etc/taos/taos.cfg|grep -v "^#"|grep dataDir|awk '{print $NF}'` + if [ -z "$folder" ]; then + folder="/var/lib/taos" + fi + local mount_point="$folder" + local usage="" + while [ -z "$usage" ]; do + usage=`df -h|grep -w "${mount_point}"|awk '{print $5}'|grep -v Use|sed "s/%$//"` + if [ "x${mount_point}" = "x/" ]; then + break + fi + mount_point=`dirname ${mount_point}` + done + if [ -z "$usage" ]; then + disk_state="error" + disk_msg="cannot get disk usage" + if [ "$get_usage_ok" = "yes" ]; then + post_disk_error_msg + get_usage_ok="no" + fi + else + get_usage_ok="yes" + local current_level=0 + for level in ${disk_usage_level[*]}; do + if [ ${usage} -ge ${level} ]; then + disk_state="error" + disk_msg="disk usage over ${level}%" + current_level=${level} + fi + done + if [ ${current_level} -gt ${current_disk_level} ]; then + post_disk_error_msg + elif [ ${current_level} -lt ${current_disk_level} ]; then + echo "disk usage reduced from ${current_disk_level} to ${current_level}" + fi + current_disk_level=${current_level} + fi +} +function run_taosd() { + taosd + set_service_state "error" "taosd exit" + # post error msg + # check crash or OOM + check_taosd_exit_type + post_error_msg +} +function print_service_state_change() { + if [ "x$1" != "x${service_state}" ]; then + echo "`date` service state: ${service_state}, ${service_msg}" + fi +} +taosd_start_time=`date +%s` while ((1)) do + check_disk # echo "outer loop: $a" - sleep 10 - output=`taos -k` - status=${output:0:1} + output=`timeout $TAOS_TIMEOUT_SECOND taos -k` + if [ -z "${output}" ]; then + echo "`date` taos -k error" + status="" + else + status=${output:0:1} + fi # echo $output # echo $status if [ "$status"x = "0"x ] then - taosd & + # taosd_start_time=`date +%s` + run_taosd & fi # echo "$status"x "$TAOS_RUN_TAOSBENCHMARK_TEST"x "$TAOS_RUN_TAOSBENCHMARK_TEST_ONCE"x if [ "$status"x = "2"x ] && [ "$TAOS_RUN_TAOSBENCHMARK_TEST"x = "1"x ] && [ "$TAOS_RUN_TAOSBENCHMARK_TEST_ONCE"x = "0"x ] @@ -24,13 +168,37 @@ do taos -s "select stable_name from information_schema.ins_stables where db_name = 'test';"|grep -q -w meters if [ $? -ne 0 ]; then taosBenchmark -y -t 1000 -n 1000 -S 900000 - taos -s "create user admin_user pass 'NDS65R6t' sysinfo 0;" - taos -s "GRANT ALL on test.* to admin_user;" + taos -s "create user admin_user pass 'NDS65R6t' sysinfo 0;" + taos -s "GRANT ALL on test.* to admin_user;" fi fi + # check taosd status + if [ "$service_state" = "ready" ]; then + # check taosd status + check_taosd + print_service_state_change "ready" + if [ "$service_state" = "error" ]; then + post_error_msg + fi + elif [ "$service_state" = "init" ]; then + check_taosd "ignore" + # check timeout + current_time=`date +%s` + time_elapsed=$(( current_time - taosd_start_time )) + if [ ${time_elapsed} -gt ${TAOSD_STARTUP_TIMEOUT_SECOND} ]; then + set_service_state "error" "taosd startup timeout" + post_error_msg + fi + print_service_state_change "init" + elif [ "$service_state" = "error" ]; then + # check taosd status + check_taosd + print_service_state_change "error" + fi # check taosadapter nc -z localhost 6041 if [ $? -ne 0 ]; then - taosadapter & + taosadapter & fi + sleep 30 done From 54bc2fc2ae1ee6a35ab30256de3882832fe1e36e Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 18 Nov 2022 16:17:29 +0800 Subject: [PATCH 25/59] update case for restful mode run --- tests/system-test/2-query/abs.py | 14 ++++---------- tests/system-test/2-query/arccos.py | 17 ++++------------- tests/system-test/2-query/arcsin.py | 18 +++++------------- tests/system-test/2-query/arctan.py | 16 +++------------- tests/system-test/2-query/ceil.py | 11 +++-------- tests/system-test/2-query/cos.py | 20 ++------------------ tests/system-test/2-query/floor.py | 11 +++-------- tests/system-test/2-query/log.py | 1 - tests/system-test/2-query/round.py | 12 ++++-------- 9 files changed, 28 insertions(+), 92 deletions(-) diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index d7478a55a8..485abf6605 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -204,18 +204,12 @@ class TDTestCase: row_check.append(elem) auto_result.append(row_check) - check_status = True + + tdSql.query(abs_query) for row_index, row in enumerate(abs_result): for col_index, elem in enumerate(row): - if auto_result[row_index][col_index] != elem: - check_status = False - if not check_status: - tdLog.notice( - "abs function value has not as expected , sql is \"%s\" " % abs_query) - sys.exit(1) - else: - tdLog.info( - "abs value check pass , it work as expected ,sql is \"%s\" " % abs_query) + tdSql.checkData(row_index,col_index,auto_result[row_index][col_index]) + def test_errors(self): dbname = "testdb" diff --git a/tests/system-test/2-query/arccos.py b/tests/system-test/2-query/arccos.py index ed717741c5..df3061a0e0 100644 --- a/tests/system-test/2-query/arccos.py +++ b/tests/system-test/2-query/arccos.py @@ -86,21 +86,12 @@ class TDTestCase: row_check.append(elem) auto_result.append(row_check) - check_status = True - + tdSql.query(pow_query) for row_index , row in enumerate(pow_result): for col_index , elem in enumerate(row): - if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): - check_status = False - elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): - check_status = False - else: - pass - if not check_status: - tdLog.notice("acos function value has not as expected , sql is \"%s\" "%pow_query ) - sys.exit(1) - else: - tdLog.info("acos value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + tdSql.checkData(row_index,col_index,auto_result[row_index][col_index]) + def test_errors(self, dbname="db"): error_sql_lists = [ diff --git a/tests/system-test/2-query/arcsin.py b/tests/system-test/2-query/arcsin.py index 71de088979..73985005ba 100644 --- a/tests/system-test/2-query/arcsin.py +++ b/tests/system-test/2-query/arcsin.py @@ -86,21 +86,13 @@ class TDTestCase: row_check.append(elem) auto_result.append(row_check) - check_status = True - + + tdSql.query(pow_query) for row_index , row in enumerate(pow_result): for col_index , elem in enumerate(row): - if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): - check_status = False - elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): - check_status = False - else: - pass - if not check_status: - tdLog.notice("asin function value has not as expected , sql is \"%s\" "%pow_query ) - sys.exit(1) - else: - tdLog.info("asin value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + tdSql.checkData(row_index,col_index,auto_result[row_index][col_index]) + + def test_errors(self, dbname="db"): error_sql_lists = [ diff --git a/tests/system-test/2-query/arctan.py b/tests/system-test/2-query/arctan.py index 9780f9855b..5964a2955a 100644 --- a/tests/system-test/2-query/arctan.py +++ b/tests/system-test/2-query/arctan.py @@ -84,22 +84,12 @@ class TDTestCase: row_check.append(elem) auto_result.append(row_check) - check_status = True - + tdSql.query(pow_query) for row_index , row in enumerate(pow_result): for col_index , elem in enumerate(row): - if auto_result[row_index][col_index] == None and elem: - check_status = False - elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): - check_status = False - else: - pass - if not check_status: - tdLog.notice("atan function value has not as expected , sql is \"%s\" "%pow_query ) - sys.exit(1) - else: - tdLog.info("atan value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + tdSql.checkData(row_index,col_index,auto_result[row_index][col_index]) + def test_errors(self, dbname="db"): error_sql_lists = [ f"select atan from {dbname}.t1", diff --git a/tests/system-test/2-query/ceil.py b/tests/system-test/2-query/ceil.py index fffd484720..afc70d23a5 100644 --- a/tests/system-test/2-query/ceil.py +++ b/tests/system-test/2-query/ceil.py @@ -85,16 +85,11 @@ class TDTestCase: row_check.append(elem) auto_result.append(row_check) - check_status = True + tdSql.query(ceil_query) for row_index , row in enumerate(ceil_result): for col_index , elem in enumerate(row): - if auto_result[row_index][col_index] != elem: - check_status = False - if not check_status: - tdLog.notice("ceil function value has not as expected , sql is \"%s\" "%ceil_query ) - sys.exit(1) - else: - tdLog.info("ceil value check pass , it work as expected ,sql is \"%s\" "%ceil_query ) + tdSql.checkData(row_index,col_index,auto_result[row_index][col_index]) + def test_errors(self, dbname="db"): error_sql_lists = [ diff --git a/tests/system-test/2-query/cos.py b/tests/system-test/2-query/cos.py index d6bddc4e84..40b6257a5f 100644 --- a/tests/system-test/2-query/cos.py +++ b/tests/system-test/2-query/cos.py @@ -84,26 +84,10 @@ class TDTestCase: row_check.append(elem) auto_result.append(row_check) - check_status = True - print("========",pow_query, origin_query ) - + tdSql.query(pow_query) for row_index , row in enumerate(pow_result): for col_index , elem in enumerate(row): - if auto_result[row_index][col_index] == None and elem: - check_status = False - elif auto_result[row_index][col_index] != None and ((auto_result[row_index][col_index] != elem) and (str(auto_result[row_index][col_index])[:6] != str(elem)[:6] )): - # elif auto_result[row_index][col_index] != None and (abs(auto_result[row_index][col_index] - elem) > 0.000001): - print("=====") - print(row_index, col_index) - print(auto_result[row_index][col_index], elem, origin_result[row_index][col_index]) - check_status = False - else: - pass - if not check_status: - tdLog.notice("cos function value has not as expected , sql is \"%s\" "%pow_query ) - sys.exit(1) - else: - tdLog.info("cos value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + tdSql.checkData(row_index,col_index,auto_result[row_index][col_index]) def test_errors(self, dbname="db"): error_sql_lists = [ diff --git a/tests/system-test/2-query/floor.py b/tests/system-test/2-query/floor.py index 6a75872bcf..2b53868b1a 100644 --- a/tests/system-test/2-query/floor.py +++ b/tests/system-test/2-query/floor.py @@ -85,16 +85,11 @@ class TDTestCase: row_check.append(elem) auto_result.append(row_check) - check_status = True + tdSql.query(floor_query) for row_index , row in enumerate(floor_result): for col_index , elem in enumerate(row): - if auto_result[row_index][col_index] != elem: - check_status = False - if not check_status: - tdLog.notice("floor function value has not as expected , sql is \"%s\" "%floor_query ) - sys.exit(1) - else: - tdLog.info("floor value check pass , it work as expected ,sql is \"%s\" "%floor_query ) + tdSql.checkData(row_index,col_index,auto_result[row_index][col_index]) + def test_errors(self, dbname=DBNAME): error_sql_lists = [ diff --git a/tests/system-test/2-query/log.py b/tests/system-test/2-query/log.py index 7305a44f56..ba4fb8cc96 100644 --- a/tests/system-test/2-query/log.py +++ b/tests/system-test/2-query/log.py @@ -91,7 +91,6 @@ class TDTestCase: elem = math.log(elem , base) elif elem <=0: elem = None - row_check.append(elem) auto_result.append(row_check) diff --git a/tests/system-test/2-query/round.py b/tests/system-test/2-query/round.py index e3d98d6986..1ad96b93d7 100644 --- a/tests/system-test/2-query/round.py +++ b/tests/system-test/2-query/round.py @@ -81,16 +81,12 @@ class TDTestCase: row_check.append(elem) auto_result.append(row_check) - check_status = True + tdSql.query(round_query) for row_index , row in enumerate(round_result): for col_index , elem in enumerate(row): - if auto_result[row_index][col_index] != elem: - check_status = False - if not check_status: - tdLog.notice("round function value has not as expected , sql is \"%s\" "%round_query ) - sys.exit(1) - else: - tdLog.info("round value check pass , it work as expected ,sql is \"%s\" "%round_query ) + tdSql.checkData(row_index , col_index ,auto_result[row_index][col_index]) + + def test_errors(self, dbname="db"): error_sql_lists = [ From 53fbc7b2567889558c088bfef7a39a396f0eebc7 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 18 Nov 2022 16:31:22 +0800 Subject: [PATCH 26/59] add more test cases --- tests/system-test/2-query/interp.py | 405 ++++++++++++++++++++++++++-- 1 file changed, 389 insertions(+), 16 deletions(-) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index a0ac4ab03b..ce57357abd 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -900,6 +900,8 @@ class TDTestCase: tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:00:55', NULL, NULL)") tdSql.execute(f"insert into {dbname}.{tbname1} values ('2020-02-02 00:01:00', 55, 60)") + # test fill linear + # check c0 tdSql.query(f"select interp(c0) from {dbname}.{tbname1} range('2020-02-01 23:59:59', '2020-02-02 00:00:00') every(1s) fill(linear)") tdSql.checkRows(1) @@ -1370,35 +1372,406 @@ class TDTestCase: tdSql.query(f"select interp(c0),interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(linear)") tdSql.checkRows(61) tdSql.checkCols(2) - tdSql.checkData(0, 0, 0) - tdSql.checkData(10, 0, 10) - tdSql.checkData(20, 0, 20) - tdSql.checkData(30, 0, 30) + tdSql.checkData(0, 0, 0) # + tdSql.checkData(1, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) # + tdSql.checkData(6, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, 10) # + tdSql.checkData(11, 0, None) + tdSql.checkData(14, 0, None) + tdSql.checkData(15, 0, None) # + tdSql.checkData(16, 0, None) + tdSql.checkData(19, 0, None) + tdSql.checkData(20, 0, 20) # + tdSql.checkData(21, 0, None) + tdSql.checkData(24, 0, None) + tdSql.checkData(25, 0, None) # + tdSql.checkData(26, 0, None) + tdSql.checkData(29, 0, None) + tdSql.checkData(30, 0, 30) # tdSql.checkData(31, 0, 31) tdSql.checkData(32, 0, 32) tdSql.checkData(33, 0, 33) tdSql.checkData(34, 0, 34) - tdSql.checkData(35, 0, 35) + tdSql.checkData(35, 0, 35) # tdSql.checkData(36, 0, 36) tdSql.checkData(37, 0, 37) tdSql.checkData(38, 0, 38) tdSql.checkData(39, 0, 39) - tdSql.checkData(40, 0, 40) - tdSql.checkData(50, 0, 50) - tdSql.checkData(60, 0, 55) + tdSql.checkData(40, 0, 40) # + tdSql.checkData(41, 0, None) + tdSql.checkData(44, 0, None) + tdSql.checkData(45, 0, None) # + tdSql.checkData(46, 0, None) + tdSql.checkData(49, 0, None) + tdSql.checkData(50, 0, 50) # + tdSql.checkData(51, 0, None) + tdSql.checkData(54, 0, None) + tdSql.checkData(55, 0, None) # + tdSql.checkData(56, 0, None) + tdSql.checkData(59, 0, None) + tdSql.checkData(60, 0, 55) # - tdSql.checkData(0, 1, None) - tdSql.checkData(10, 1, 10) - tdSql.checkData(20, 1, None) - tdSql.checkData(30, 1, 30) - tdSql.checkData(40, 1, 40) + tdSql.checkData(0, 1, None) # + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, None) + tdSql.checkData(5, 1, None) # + tdSql.checkData(6, 1, None) + tdSql.checkData(9, 1, None) + tdSql.checkData(10, 1, 10) # + tdSql.checkData(11, 1, None) + tdSql.checkData(14, 1, None) + tdSql.checkData(15, 1, None) # + tdSql.checkData(16, 1, None) + tdSql.checkData(19, 1, None) + tdSql.checkData(20, 1, None) # + tdSql.checkData(21, 1, None) + tdSql.checkData(24, 1, None) + tdSql.checkData(25, 1, None) # + tdSql.checkData(26, 1, None) + tdSql.checkData(29, 1, None) + tdSql.checkData(30, 1, 30) # + tdSql.checkData(31, 1, None) + tdSql.checkData(34, 1, None) + tdSql.checkData(35, 1, None) # + tdSql.checkData(36, 1, None) + tdSql.checkData(39, 1, None) + tdSql.checkData(40, 1, 40) # tdSql.checkData(41, 1, 41) tdSql.checkData(42, 1, 42) tdSql.checkData(43, 1, 43) tdSql.checkData(44, 1, 44) - tdSql.checkData(45, 1, 45) - tdSql.checkData(50, 1, None) - tdSql.checkData(60, 1, 60) + tdSql.checkData(45, 1, 45) # + tdSql.checkData(46, 1, None) + tdSql.checkData(49, 1, None) + tdSql.checkData(50, 1, None) # + tdSql.checkData(51, 1, None) + tdSql.checkData(54, 1, None) + tdSql.checkData(55, 1, None) # + tdSql.checkData(56, 1, None) + tdSql.checkData(59, 1, None) + tdSql.checkData(60, 1, 60) # + + # test fill null + tdSql.query(f"select interp(c0),interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(null)") + tdSql.checkRows(61) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 0) # + tdSql.checkData(1, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) # + tdSql.checkData(6, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, 10) # + tdSql.checkData(11, 0, None) + tdSql.checkData(14, 0, None) + tdSql.checkData(15, 0, None) # + tdSql.checkData(16, 0, None) + tdSql.checkData(19, 0, None) + tdSql.checkData(20, 0, 20) # + tdSql.checkData(21, 0, None) + tdSql.checkData(24, 0, None) + tdSql.checkData(25, 0, None) # + tdSql.checkData(26, 0, None) + tdSql.checkData(29, 0, None) + tdSql.checkData(30, 0, 30) # + tdSql.checkData(31, 0, None) + tdSql.checkData(34, 0, None) + tdSql.checkData(35, 0, 35) # + tdSql.checkData(36, 0, None) + tdSql.checkData(39, 0, None) + tdSql.checkData(40, 0, 40) # + tdSql.checkData(41, 0, None) + tdSql.checkData(44, 0, None) + tdSql.checkData(45, 0, None) # + tdSql.checkData(46, 0, None) + tdSql.checkData(49, 0, None) + tdSql.checkData(50, 0, 50) # + tdSql.checkData(51, 0, None) + tdSql.checkData(54, 0, None) + tdSql.checkData(55, 0, None) # + tdSql.checkData(56, 0, None) + tdSql.checkData(59, 0, None) + tdSql.checkData(60, 0, 55) # + + tdSql.checkData(0, 1, None) # + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, None) + tdSql.checkData(5, 1, None) # + tdSql.checkData(6, 1, None) + tdSql.checkData(9, 1, None) + tdSql.checkData(10, 1, 10) # + tdSql.checkData(11, 1, None) + tdSql.checkData(14, 1, None) + tdSql.checkData(15, 1, None) # + tdSql.checkData(16, 1, None) + tdSql.checkData(19, 1, None) + tdSql.checkData(20, 1, None) # + tdSql.checkData(21, 1, None) + tdSql.checkData(24, 1, None) + tdSql.checkData(25, 1, None) # + tdSql.checkData(26, 1, None) + tdSql.checkData(29, 1, None) + tdSql.checkData(30, 1, 30) # + tdSql.checkData(31, 1, None) + tdSql.checkData(34, 1, None) + tdSql.checkData(35, 1, None) # + tdSql.checkData(36, 1, None) + tdSql.checkData(39, 1, None) + tdSql.checkData(40, 1, 40) # + tdSql.checkData(41, 1, None) + tdSql.checkData(44, 1, None) + tdSql.checkData(45, 1, 45) # + tdSql.checkData(46, 1, None) + tdSql.checkData(49, 1, None) + tdSql.checkData(50, 1, None) # + tdSql.checkData(51, 1, None) + tdSql.checkData(54, 1, None) + tdSql.checkData(55, 1, None) # + tdSql.checkData(56, 1, None) + tdSql.checkData(59, 1, None) + tdSql.checkData(60, 1, 60) # + + # test fill value + tdSql.query(f"select interp(c0),interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(value, 123)") + tdSql.checkRows(61) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 0) # + tdSql.checkData(1, 0, 123) + tdSql.checkData(4, 0, 123) + tdSql.checkData(5, 0, None) # + tdSql.checkData(6, 0, 123) + tdSql.checkData(9, 0, 123) + tdSql.checkData(10, 0, 10) # + tdSql.checkData(11, 0, 123) + tdSql.checkData(14, 0, 123) + tdSql.checkData(15, 0, None) # + tdSql.checkData(16, 0, 123) + tdSql.checkData(19, 0, 123) + tdSql.checkData(20, 0, 20) # + tdSql.checkData(21, 0, 123) + tdSql.checkData(24, 0, 123) + tdSql.checkData(25, 0, None) # + tdSql.checkData(26, 0, 123) + tdSql.checkData(29, 0, 123) + tdSql.checkData(30, 0, 30) # + tdSql.checkData(31, 0, 123) + tdSql.checkData(34, 0, 123) + tdSql.checkData(35, 0, 35) # + tdSql.checkData(36, 0, 123) + tdSql.checkData(39, 0, 123) + tdSql.checkData(40, 0, 40) # + tdSql.checkData(41, 0, 123) + tdSql.checkData(44, 0, 123) + tdSql.checkData(45, 0, None) # + tdSql.checkData(46, 0, 123) + tdSql.checkData(49, 0, 123) + tdSql.checkData(50, 0, 50) # + tdSql.checkData(51, 0, 123) + tdSql.checkData(54, 0, 123) + tdSql.checkData(55, 0, None) # + tdSql.checkData(59, 0, 123) + tdSql.checkData(60, 0, 55) # + + tdSql.checkData(0, 1, None) # + tdSql.checkData(1, 1, 123) + tdSql.checkData(4, 1, 123) + tdSql.checkData(5, 1, None) # + tdSql.checkData(6, 1, 123) + tdSql.checkData(9, 1, 123) + tdSql.checkData(10, 1, 10) # + tdSql.checkData(11, 1, 123) + tdSql.checkData(14, 1, 123) + tdSql.checkData(15, 1, None) # + tdSql.checkData(16, 1, 123) + tdSql.checkData(19, 1, 123) + tdSql.checkData(20, 1, None) # + tdSql.checkData(21, 1, 123) + tdSql.checkData(24, 1, 123) + tdSql.checkData(25, 1, None) # + tdSql.checkData(26, 1, 123) + tdSql.checkData(29, 1, 123) + tdSql.checkData(30, 1, 30) # + tdSql.checkData(31, 1, 123) + tdSql.checkData(34, 1, 123) + tdSql.checkData(35, 1, None) # + tdSql.checkData(36, 1, 123) + tdSql.checkData(39, 1, 123) + tdSql.checkData(40, 1, 40) # + tdSql.checkData(41, 1, 123) + tdSql.checkData(44, 1, 123) + tdSql.checkData(45, 1, 45) # + tdSql.checkData(46, 1, 123) + tdSql.checkData(49, 1, 123) + tdSql.checkData(50, 1, None) # + tdSql.checkData(51, 1, 123) + tdSql.checkData(54, 1, 123) + tdSql.checkData(55, 1, None) # + tdSql.checkData(56, 1, 123) + tdSql.checkData(59, 1, 123) + tdSql.checkData(60, 1, 60) # + + # test fill prev + tdSql.query(f"select interp(c0),interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(prev)") + tdSql.checkRows(61) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 0) # + tdSql.checkData(1, 0, 0) + tdSql.checkData(4, 0, 0) + tdSql.checkData(5, 0, None) # + tdSql.checkData(6, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, 10) # + tdSql.checkData(11, 0, 10) + tdSql.checkData(14, 0, 10) + tdSql.checkData(15, 0, None) # + tdSql.checkData(16, 0, None) + tdSql.checkData(19, 0, None) + tdSql.checkData(20, 0, 20) # + tdSql.checkData(21, 0, 20) + tdSql.checkData(24, 0, 20) + tdSql.checkData(25, 0, None) # + tdSql.checkData(26, 0, None) + tdSql.checkData(29, 0, None) + tdSql.checkData(30, 0, 30) # + tdSql.checkData(31, 0, 30) + tdSql.checkData(34, 0, 30) + tdSql.checkData(35, 0, 35) # + tdSql.checkData(36, 0, 35) + tdSql.checkData(39, 0, 35) + tdSql.checkData(40, 0, 40) # + tdSql.checkData(41, 0, 40) + tdSql.checkData(44, 0, 40) + tdSql.checkData(45, 0, None) # + tdSql.checkData(46, 0, None) + tdSql.checkData(49, 0, None) + tdSql.checkData(50, 0, 50) # + tdSql.checkData(51, 0, 50) + tdSql.checkData(54, 0, 50) + tdSql.checkData(55, 0, None) # + tdSql.checkData(56, 0, None) + tdSql.checkData(59, 0, None) + tdSql.checkData(60, 0, 55) # + + tdSql.checkData(0, 1, None) # + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, None) + tdSql.checkData(5, 1, None) # + tdSql.checkData(6, 1, None) + tdSql.checkData(9, 1, None) + tdSql.checkData(10, 1, 10) # + tdSql.checkData(11, 1, 10) + tdSql.checkData(14, 1, 10) + tdSql.checkData(15, 1, None) # + tdSql.checkData(16, 1, None) + tdSql.checkData(19, 1, None) + tdSql.checkData(20, 1, None) # + tdSql.checkData(21, 1, None) + tdSql.checkData(24, 1, None) + tdSql.checkData(25, 1, None) # + tdSql.checkData(26, 1, None) + tdSql.checkData(29, 1, None) + tdSql.checkData(30, 1, 30) # + tdSql.checkData(31, 1, 30) + tdSql.checkData(34, 1, 30) + tdSql.checkData(35, 1, None) # + tdSql.checkData(36, 1, None) + tdSql.checkData(39, 1, None) + tdSql.checkData(40, 1, 40) # + tdSql.checkData(41, 1, 40) + tdSql.checkData(44, 1, 40) + tdSql.checkData(45, 1, 45) # + tdSql.checkData(46, 1, 45) + tdSql.checkData(49, 1, 45) + tdSql.checkData(50, 1, None) # + tdSql.checkData(51, 1, None) + tdSql.checkData(54, 1, None) + tdSql.checkData(55, 1, None) # + tdSql.checkData(56, 1, None) + tdSql.checkData(59, 1, None) + tdSql.checkData(60, 1, 60) # + + # test fill next + tdSql.query(f"select interp(c0),interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(next)") + tdSql.checkRows(61) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 0) # + tdSql.checkData(1, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) # + tdSql.checkData(6, 0, 10) + tdSql.checkData(9, 0, 10) + tdSql.checkData(10, 0, 10) # + tdSql.checkData(11, 0, None) + tdSql.checkData(14, 0, None) + tdSql.checkData(15, 0, None) # + tdSql.checkData(16, 0, 20) + tdSql.checkData(19, 0, 20) + tdSql.checkData(20, 0, 20) # + tdSql.checkData(21, 0, None) + tdSql.checkData(24, 0, None) + tdSql.checkData(25, 0, None) # + tdSql.checkData(26, 0, 30) + tdSql.checkData(29, 0, 30) + tdSql.checkData(30, 0, 30) # + tdSql.checkData(31, 0, 35) + tdSql.checkData(34, 0, 35) + tdSql.checkData(35, 0, 35) # + tdSql.checkData(36, 0, 40) + tdSql.checkData(39, 0, 40) + tdSql.checkData(40, 0, 40) # + tdSql.checkData(41, 0, None) + tdSql.checkData(44, 0, None) + tdSql.checkData(45, 0, None) # + tdSql.checkData(46, 0, 50) + tdSql.checkData(49, 0, 50) + tdSql.checkData(50, 0, 50) # + tdSql.checkData(51, 0, None) + tdSql.checkData(54, 0, None) + tdSql.checkData(55, 0, None) # + tdSql.checkData(56, 0, 55) + tdSql.checkData(59, 0, 55) + tdSql.checkData(60, 0, 55) # + + tdSql.checkData(0, 1, None) # + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, None) + tdSql.checkData(5, 1, None) # + tdSql.checkData(6, 1, 10) + tdSql.checkData(9, 1, 10) + tdSql.checkData(10, 1, 10) # + tdSql.checkData(11, 1, None) + tdSql.checkData(14, 1, None) + tdSql.checkData(15, 1, None) # + tdSql.checkData(16, 1, None) + tdSql.checkData(19, 1, None) + tdSql.checkData(20, 1, None) # + tdSql.checkData(21, 1, None) + tdSql.checkData(24, 1, None) + tdSql.checkData(25, 1, None) # + tdSql.checkData(26, 1, 30) + tdSql.checkData(29, 1, 30) + tdSql.checkData(30, 1, 30) # + tdSql.checkData(31, 1, None) + tdSql.checkData(34, 1, None) + tdSql.checkData(35, 1, None) # + tdSql.checkData(36, 1, 40) + tdSql.checkData(39, 1, 40) + tdSql.checkData(40, 1, 40) # + tdSql.checkData(41, 1, 45) + tdSql.checkData(44, 1, 45) + tdSql.checkData(45, 1, 45) # + tdSql.checkData(46, 1, None) + tdSql.checkData(49, 1, None) + tdSql.checkData(50, 1, None) # + tdSql.checkData(51, 1, None) + tdSql.checkData(54, 1, None) + tdSql.checkData(55, 1, None) # + tdSql.checkData(56, 1, 60) + tdSql.checkData(59, 1, 60) + tdSql.checkData(60, 1, 60) # tdLog.printNoPrefix("==========step11:test multi-interp cases") tdSql.query(f"select interp(c0),interp(c1),interp(c2),interp(c3) from {dbname}.{tbname} range('2020-02-09 00:00:05', '2020-02-13 00:00:05') every(1d) fill(null)") From 3f9fd91ad7f504891f263fc95bf1e8fad2ca1bbb Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 18 Nov 2022 17:03:14 +0800 Subject: [PATCH 27/59] update --- tests/system-test/2-query/tail.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/system-test/2-query/tail.py b/tests/system-test/2-query/tail.py index f925380c09..fe2f24ad25 100644 --- a/tests/system-test/2-query/tail.py +++ b/tests/system-test/2-query/tail.py @@ -436,16 +436,16 @@ class TDTestCase: ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+1s, -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) - tdSql.query(f"select tail(c2,2) from {dbname}.sub1_bound order by 1 desc") - tdSql.checkRows(2) - tdSql.checkData(0,0,9223372036854775803) + tdSql.query(f"select tail(c2,1) from {dbname}.sub1_bound order by 1 desc") + tdSql.checkRows(1) + tdSql.checkData(0,0,-9223372036854775803) def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() From 0cb539b181b28b434dbed7cbd53fdf6b273a555c Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 18 Nov 2022 17:09:43 +0800 Subject: [PATCH 28/59] update --- tests/system-test/2-query/avg.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/system-test/2-query/avg.py b/tests/system-test/2-query/avg.py index ec7ec34ed3..f68d0f5197 100644 --- a/tests/system-test/2-query/avg.py +++ b/tests/system-test/2-query/avg.py @@ -114,16 +114,10 @@ class TDTestCase: avg_result = tdSql.getResult(origin_query) origin_result = tdSql.getResult(check_query) - check_status = True + tdSql.query(origin_query) for row_index , row in enumerate(avg_result): for col_index , elem in enumerate(row): - if avg_result[row_index][col_index] != origin_result[row_index][col_index]: - check_status = False - if not check_status: - tdLog.notice("avg function value has not as expected , sql is \"%s\" "%origin_query ) - sys.exit(1) - else: - tdLog.info("avg value check pass , it work as expected ,sql is \"%s\" "%check_query ) + tdSql.checkData(row_index,col_index,origin_result[row_index][col_index]) def test_errors(self, dbname="db"): error_sql_lists = [ From 963ca6b14be17800ac8bb9b6cff33cbe6e0ce54a Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 18 Nov 2022 17:18:01 +0800 Subject: [PATCH 29/59] update --- tests/system-test/2-query/avg.py | 14 +++++++------- .../system-test/2-query/function_stateduration.py | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/system-test/2-query/avg.py b/tests/system-test/2-query/avg.py index f68d0f5197..910dd524cb 100644 --- a/tests/system-test/2-query/avg.py +++ b/tests/system-test/2-query/avg.py @@ -372,33 +372,33 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483645, 9223372036854775805, 32765, 125, 3.40E+37, 1.7e+307, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, 2147483645, 9223372036854775805, 32765, 125, 3.40E+37, 1.7e+307, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483644, 9223372036854775804, 32764, 124, 3.40E+37, 1.7e+307, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483644, 9223372036854775804, 32764, 124, 3.40E+37, 1.7e+307, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+15s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+20s, 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) #self.check_avg(f"select avg(c1), avg(c2), avg(c3) , avg(c4), avg(c5) ,avg(c6) from {dbname}.sub1_bound " , f" select sum(c1)/count(c1), sum(c2)/count(c2) ,sum(c3)/count(c3), sum(c4)/count(c4), sum(c5)/count(c5) ,sum(c6)/count(c6) from {dbname}.sub1_bound ") diff --git a/tests/system-test/2-query/function_stateduration.py b/tests/system-test/2-query/function_stateduration.py index ad9b8b0d79..728df0b91f 100644 --- a/tests/system-test/2-query/function_stateduration.py +++ b/tests/system-test/2-query/function_stateduration.py @@ -364,10 +364,10 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( @@ -375,15 +375,15 @@ class TDTestCase: ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+15s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.query(f"select stateduration(c1,'GT',1,1s) from {dbname}.sub1_bound") From d07dc94446c3356b4fb1062f7f27601f51d84b96 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 18 Nov 2022 18:14:52 +0800 Subject: [PATCH 30/59] update --- tests/system-test/2-query/abs.py | 8 ++-- tests/system-test/2-query/and_or_for_byte.py | 8 ++-- tests/system-test/2-query/arccos.py | 8 ++-- tests/system-test/2-query/arcsin.py | 8 ++-- tests/system-test/2-query/arctan.py | 8 ++-- tests/system-test/2-query/ceil.py | 10 ++--- tests/system-test/2-query/cos.py | 6 +-- tests/system-test/2-query/floor.py | 10 ++--- tests/system-test/2-query/last_row.py | 8 ++-- tests/system-test/2-query/log.py | 8 ++-- tests/system-test/2-query/pow.py | 8 ++-- tests/system-test/2-query/round.py | 10 ++--- tests/system-test/2-query/sin.py | 8 ++-- tests/system-test/2-query/sqrt.py | 8 ++-- tests/system-test/2-query/statecount.py | 10 ++--- tests/system-test/2-query/tail.py | 10 ++--- tests/system-test/2-query/tan.py | 8 ++-- tests/system-test/2-query/txt.txt | 40 ++++++++++++++++++++ tests/system-test/2-query/unique.py | 12 +++--- 19 files changed, 118 insertions(+), 78 deletions(-) create mode 100644 tests/system-test/2-query/txt.txt diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index 485abf6605..d64d550bc4 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -460,19 +460,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto(f"select c1, c2, c3 , c4, c5 ,c6 from {dbname}.sub1_bound ", f"select abs(c1), abs(c2) ,abs(c3), abs(c4), abs(c5) ,abs(c6) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/and_or_for_byte.py b/tests/system-test/2-query/and_or_for_byte.py index 479918f2f9..15e9110b3b 100644 --- a/tests/system-test/2-query/and_or_for_byte.py +++ b/tests/system-test/2-query/and_or_for_byte.py @@ -426,19 +426,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_function("&", False , f"{dbname}.sub1_bound" ,"c1","c2","c3","c4","c5","c6" ) self.check_function("&", False , f"{dbname}.sub1_bound","abs(c1)","abs(c2)","abs(c3)","abs(c4)","abs(c5)","abs(c6)" ) diff --git a/tests/system-test/2-query/arccos.py b/tests/system-test/2-query/arccos.py index df3061a0e0..f22d393ecd 100644 --- a/tests/system-test/2-query/arccos.py +++ b/tests/system-test/2-query/arccos.py @@ -405,19 +405,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto_acos( f"select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from {dbname}.sub1_bound ", f"select acos(abs(c1)), acos(abs(c2)) ,acos(abs(c3)), acos(abs(c4)), acos(abs(c5)) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/arcsin.py b/tests/system-test/2-query/arcsin.py index 73985005ba..1872518c5d 100644 --- a/tests/system-test/2-query/arcsin.py +++ b/tests/system-test/2-query/arcsin.py @@ -406,19 +406,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto_asin( f"select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from {dbname}.sub1_bound ", f"select asin(abs(c1)), asin(abs(c2)) ,asin(abs(c3)), asin(abs(c4)), asin(abs(c5)) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/arctan.py b/tests/system-test/2-query/arctan.py index 5964a2955a..9561637b76 100644 --- a/tests/system-test/2-query/arctan.py +++ b/tests/system-test/2-query/arctan.py @@ -402,19 +402,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto_atan( f"select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from {dbname}.sub1_bound ", f"select atan(abs(c1)), atan(abs(c2)) ,atan(abs(c3)), atan(abs(c4)), atan(abs(c5)) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/ceil.py b/tests/system-test/2-query/ceil.py index afc70d23a5..aabc716a74 100644 --- a/tests/system-test/2-query/ceil.py +++ b/tests/system-test/2-query/ceil.py @@ -372,10 +372,10 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( @@ -383,15 +383,15 @@ class TDTestCase: ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+15s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto( f"select c1, c2, c3 , c4, c5 ,c6 from {dbname}.sub1_bound ", f"select ceil(c1), ceil(c2) ,ceil(c3), ceil(c4), ceil(c5) ,ceil(c6) from {dbname}.sub1_bound") self.check_result_auto( f"select c1, c2, c3 , c3, c2 ,c1 from {dbname}.sub1_bound ", f"select ceil(c1), ceil(c2) ,ceil(c3), ceil(c3), ceil(c2) ,ceil(c1) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/cos.py b/tests/system-test/2-query/cos.py index 40b6257a5f..d2056805eb 100644 --- a/tests/system-test/2-query/cos.py +++ b/tests/system-test/2-query/cos.py @@ -397,16 +397,16 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) # self.check_result_auto_cos( f"select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from {dbname}.sub1_bound ", f"select cos(abs(c1)), cos(abs(c2)) ,cos(abs(c3)), cos(abs(c4)), cos(abs(c5)) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/floor.py b/tests/system-test/2-query/floor.py index 2b53868b1a..bf78aa8bfa 100644 --- a/tests/system-test/2-query/floor.py +++ b/tests/system-test/2-query/floor.py @@ -383,10 +383,10 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( @@ -394,15 +394,15 @@ class TDTestCase: ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+15s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto( f"select c1, c2, c3 , c4, c5 ,c6 from {dbname}.sub1_bound ", f"select floor(c1), floor(c2) ,floor(c3), floor(c4), floor(c5) ,floor(c6) from {dbname}.sub1_bound") self.check_result_auto( f"select c1, c2, c3 , c3, c2 ,c1 from {dbname}.sub1_bound ", f"select floor(c1), floor(c2) ,floor(c3), floor(c3), floor(c2) ,floor(c1) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index f8d6ce4c6c..01da658989 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -572,19 +572,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) # check basic elem for table per row diff --git a/tests/system-test/2-query/log.py b/tests/system-test/2-query/log.py index ba4fb8cc96..a05de53c42 100644 --- a/tests/system-test/2-query/log.py +++ b/tests/system-test/2-query/log.py @@ -518,19 +518,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto_log(None , f"select c1, c2, c3 , c4, c5 ,c6 from {dbname}.sub1_bound ", f"select log(c1), log(c2) ,log(c3), log(c4), log(c5) ,log(c6) from {dbname}.sub1_bound") self.check_result_auto_log( 2 , f"select c1, c2, c3 , c4, c5 ,c6 from {dbname}.sub1_bound ", f"select log(c1,2), log(c2,2) ,log(c3,2), log(c4,2), log(c5,2) ,log(c6,2) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/pow.py b/tests/system-test/2-query/pow.py index a067d66547..5647d81e27 100644 --- a/tests/system-test/2-query/pow.py +++ b/tests/system-test/2-query/pow.py @@ -507,19 +507,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto_pow(2, f"select c1, c3 , c4, c5 from {dbname}.sub1_bound ", f"select pow(c1,2), pow(c3,2), pow(c4,2), pow(c5,2) from {dbname}.sub1_bound") self.check_result_auto_pow(3, f"select c1, c3 , c4, c5 from {dbname}.sub1_bound ", f"select pow(c1,3), pow(c3,3), pow(c4,3), pow(c5,3) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/round.py b/tests/system-test/2-query/round.py index 1ad96b93d7..d647f516ae 100644 --- a/tests/system-test/2-query/round.py +++ b/tests/system-test/2-query/round.py @@ -384,10 +384,10 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( @@ -395,15 +395,15 @@ class TDTestCase: ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+15s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto( f"select c1, c2, c3 , c4, c5 ,c6 from {dbname}.sub1_bound ", f"select round(c1), round(c2) ,round(c3), round(c4), round(c5) ,round(c6) from {dbname}.sub1_bound") self.check_result_auto( f"select c1, c2, c3 , c3, c2 ,c1 from {dbname}.sub1_bound ", f"select round(c1), round(c2) ,round(c3), round(c3), round(c2) ,round(c1) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/sin.py b/tests/system-test/2-query/sin.py index 4fdec8fd73..c65e58852d 100644 --- a/tests/system-test/2-query/sin.py +++ b/tests/system-test/2-query/sin.py @@ -394,19 +394,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto_sin( f"select abs(c1), abs(c2), abs(c3) , abs(c4) from {dbname}.sub1_bound ", f"select sin(abs(c1)), sin(abs(c2)) ,sin(abs(c3)), sin(abs(c4)) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/sqrt.py b/tests/system-test/2-query/sqrt.py index 9229444f74..4db166808e 100644 --- a/tests/system-test/2-query/sqrt.py +++ b/tests/system-test/2-query/sqrt.py @@ -443,19 +443,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto_sqrt( f"select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from {dbname}.sub1_bound ", f"select sqrt(abs(c1)), sqrt(abs(c2)) ,sqrt(abs(c3)), sqrt(abs(c4)), sqrt(abs(c5)) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/statecount.py b/tests/system-test/2-query/statecount.py index 2aa9194d37..f76e153014 100644 --- a/tests/system-test/2-query/statecount.py +++ b/tests/system-test/2-query/statecount.py @@ -451,10 +451,10 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( @@ -462,15 +462,15 @@ class TDTestCase: ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+15s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.query(f"select statecount(c1,'GT',1) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/tail.py b/tests/system-test/2-query/tail.py index fe2f24ad25..43321810d0 100644 --- a/tests/system-test/2-query/tail.py +++ b/tests/system-test/2-query/tail.py @@ -421,10 +421,10 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( @@ -432,15 +432,15 @@ class TDTestCase: ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()+1s, -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+15s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.query(f"select tail(c2,1) from {dbname}.sub1_bound order by 1 desc") diff --git a/tests/system-test/2-query/tan.py b/tests/system-test/2-query/tan.py index 27e6efb475..e689eaba20 100644 --- a/tests/system-test/2-query/tan.py +++ b/tests/system-test/2-query/tan.py @@ -391,19 +391,19 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) self.check_result_auto_tan( f"select abs(c1), abs(c2), abs(c3) , abs(c4) from {dbname}.sub1_bound ", f"select tan(abs(c1)), tan(abs(c2)) ,tan(abs(c3)), tan(abs(c4)) from {dbname}.sub1_bound") diff --git a/tests/system-test/2-query/txt.txt b/tests/system-test/2-query/txt.txt new file mode 100644 index 0000000000..fd3b0b5e36 --- /dev/null +++ b/tests/system-test/2-query/txt.txt @@ -0,0 +1,40 @@ +abs.py: def check_boundary_values(self): +abs.py: self.check_boundary_values() +and_or_for_byte.py: def check_boundary_values(self, dbname="bound_test"): +and_or_for_byte.py: self.check_boundary_values() +arccos.py: def check_boundary_values(self, dbname="bound_test"): +arccos.py: self.check_boundary_values() +arcsin.py: def check_boundary_values(self, dbname="bound_test"): +arcsin.py: self.check_boundary_values() +arctan.py: def check_boundary_values(self, dbname="bound_test"): +arctan.py: self.check_boundary_values() +avg.py: def check_boundary_values(self, dbname="bound_test"): +avg.py: self.check_boundary_values() +ceil.py: def check_boundary_values(self, dbname="bound_test"): +ceil.py: self.check_boundary_values() +cos.py: def check_boundary_values(self, dbname="bound_test"): +cos.py: self.check_boundary_values() +floor.py: def check_boundary_values(self, dbname="bound_test"): +floor.py: self.check_boundary_values() +function_stateduration.py: def check_boundary_values(self, dbname="bound_test"): +function_stateduration.py: self.check_boundary_values() +last_row.py: def check_boundary_values(self, dbname="bound_test"): +last_row.py: self.check_boundary_values() +log.py: def check_boundary_values(self, dbname="bound_test"): +log.py: self.check_boundary_values() +pow.py: def check_boundary_values(self, dbname="bound_test"): +pow.py: self.check_boundary_values() +round.py: def check_boundary_values(self, dbname="bound_test"): +round.py: self.check_boundary_values() +sin.py: def check_boundary_values(self, dbname="testdb"): +sin.py: self.check_boundary_values() +sqrt.py: def check_boundary_values(self, dbname="bound_test"): +sqrt.py: self.check_boundary_values() +statecount.py: def check_boundary_values(self, dbname="bound_test"): +statecount.py: self.check_boundary_values() +tail.py: def check_boundary_values(self, dbname="bound_test"): +tail.py: self.check_boundary_values() +tan.py: def check_boundary_values(self, dbname="bound_test"): +tan.py: self.check_boundary_values() +unique.py: def check_boundary_values(self, dbname="bound_test"): +unique.py: self.check_boundary_values() diff --git a/tests/system-test/2-query/unique.py b/tests/system-test/2-query/unique.py index 6c51854b43..2b0336d2d7 100644 --- a/tests/system-test/2-query/unique.py +++ b/tests/system-test/2-query/unique.py @@ -451,26 +451,26 @@ class TDTestCase: ) tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-10s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-5s, 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()+1s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+5s, -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()+2s, 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+10s, 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into {dbname}.sub1_bound values ( now()+3s, -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+20s, -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()+30s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.query(f"select unique(c2) from {dbname}.sub1_bound order by 1 desc") From aa4b2aaa745f0eb87934f0475b32b8e6fdf4f9fa Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Sat, 19 Nov 2022 19:31:04 +0800 Subject: [PATCH 31/59] fix: compile taosws on loongarch Close [TD-20176](https://jira.taosdata.com:18080/browse/TD-20176) --- cmake/taosws_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index 7d48eb9d8a..e79c6b799f 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git - GIT_TAG 9843872 + GIT_TAG f406d51 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 40b2d3673b29081bdbffb036d1069058804edf28 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 20 Nov 2022 15:04:44 +0800 Subject: [PATCH 32/59] test: python asan --- tests/parallel_test/cases.task | 6 +++--- tests/pytest/util/log.py | 2 +- tests/system-test/test.sh | 7 ++++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index a43859ebfa..e1b7f9cb32 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -278,7 +278,7 @@ ,,y,script,./test.sh -f tsim/stable/values.sim ,,y,script,./test.sh -f tsim/stable/vnode3.sim ,,y,script,./test.sh -f tsim/stable/metrics_idx.sim -,,n,script,./test.sh -f tsim/sma/drop_sma.sim +,,y,script,./test.sh -f tsim/sma/drop_sma.sim ,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim @@ -438,11 +438,11 @@ ,,,system-test,python3 ./test.py -f 1-insert/database_pre_suf.py ,,,system-test,python3 ./test.py -f 1-insert/InsertFuturets.py ,,,system-test,python3 ./test.py -f 0-others/show.py -,,,system-test,python3 ./test.py -f 2-query/abs.py +,,,system-test,./test.sh python3 ./test.py -f 2-query/abs.py ,,,system-test,python3 ./test.py -f 2-query/abs.py -R ,,,system-test,python3 ./test.py -f 2-query/and_or_for_byte.py ,,,system-test,python3 ./test.py -f 2-query/and_or_for_byte.py -R -,,,system-test,python3 ./test.py -f 2-query/apercentile.py +,,,system-test,./test.sh python3 ./test.py -f 2-query/apercentile.py ,,,system-test,python3 ./test.py -f 2-query/apercentile.py -R ,,,system-test,python3 ./test.py -f 2-query/arccos.py ,,,system-test,python3 ./test.py -f 2-query/arccos.py -R diff --git a/tests/pytest/util/log.py b/tests/pytest/util/log.py index a132178308..000c907ea4 100644 --- a/tests/pytest/util/log.py +++ b/tests/pytest/util/log.py @@ -33,7 +33,7 @@ class TDLog: print("\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err)) def success(self, info): - print("\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info)) + printf("\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info)) def notice(self, err): print("\033[1;33m%s %s\033[0m" % (datetime.datetime.now(), err)) diff --git a/tests/system-test/test.sh b/tests/system-test/test.sh index 2a3187e641..bba6552fab 100755 --- a/tests/system-test/test.sh +++ b/tests/system-test/test.sh @@ -68,15 +68,16 @@ ulimit -c unlimited #sudo sysctl -w kernel.core_pattern=$TOP_DIR/core.%p.%e echo "ExcuteCmd:" $* -echo "AsanDir:" $ASAN_DIR/psim.asan +echo "AsanDir:" $ASAN_DIR/psim.info export LD_PRELOAD=libasan.so.5 -$* -a 2> $ASAN_DIR/psim.asan +$* -a 2> $ASAN_DIR/psim.info + result=$? echo "Execute result:" $result if [ $result -eq 0 ]; then - $CODE_DIR/sh/checkAsan.sh + $TOP_DIR/tests/script/sh/checkAsan.sh else exit 1 fi From ed7eb1cbf101f35b47eb139a6ae64fce44e0b078 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 20 Nov 2022 16:47:10 +0800 Subject: [PATCH 33/59] test: python asan --- tests/system-test/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/test.sh b/tests/system-test/test.sh index bba6552fab..c6238d37d0 100755 --- a/tests/system-test/test.sh +++ b/tests/system-test/test.sh @@ -45,7 +45,7 @@ declare -x SIM_DIR=$TOP_DIR/sim PROGRAM=$BUILD_DIR/build/bin/tsim PRG_DIR=$SIM_DIR/tsim ASAN_DIR=$SIM_DIR/asan -SYSTEM_TEST_DIR=$TOP_DIR/tests/system-test +SYSTEM_TEST_DIR=$CODE_DIR/tests/system-test chmod -R 777 $PRG_DIR echo "------------------------------------------------------------------------" @@ -77,7 +77,7 @@ result=$? echo "Execute result:" $result if [ $result -eq 0 ]; then - $TOP_DIR/tests/script/sh/checkAsan.sh + $CODE_DIR/tests/script/sh/checkAsan.sh else exit 1 fi From 4dc1b33414e918b90bc68f7b96228e129b088e30 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 20 Nov 2022 17:29:16 +0800 Subject: [PATCH 34/59] test: python asan --- tests/script/sh/checkAsan.sh | 1 + tests/script/sh/exec.sh | 1 + tests/script/sh/stop_dnodes.sh | 2 +- tests/system-test/test.py | 2 +- tests/system-test/test.sh | 28 +++++++++++++++++++--------- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 0ce74a989e..8e5738e911 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -3,6 +3,7 @@ set +e #set -x +export LD_PRELOAD= SCRIPT_DIR=`dirname $0` cd $SCRIPT_DIR/../ SCRIPT_DIR=`pwd` diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index 5ef4cca741..ad671631a9 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -11,6 +11,7 @@ set +e #set -x +export LD_PRELOAD= UNAME_BIN=`which uname` OS_TYPE=`$UNAME_BIN` diff --git a/tests/script/sh/stop_dnodes.sh b/tests/script/sh/stop_dnodes.sh index 38667d9b3f..34777ccb59 100755 --- a/tests/script/sh/stop_dnodes.sh +++ b/tests/script/sh/stop_dnodes.sh @@ -3,10 +3,10 @@ set +e #set -x +export LD_PRELOAD= UNAME_BIN=`which uname` OS_TYPE=`$UNAME_BIN` -export LD_PRELOAD= PID=`ps -ef|grep /usr/bin/taosd | grep -v grep | awk '{print $2}'` if [ -n "$PID" ]; then echo systemctl stop taosd diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 712b9689ba..cf9aba123c 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -555,5 +555,5 @@ if __name__ == "__main__": conn.close() if asan: tdDnodes.StopAllSigint() - tdLog.info("address sanitizer mode finished") + tdLog.info("Address sanitizer mode finished") sys.exit(0) diff --git a/tests/system-test/test.sh b/tests/system-test/test.sh index c6238d37d0..6fbc9961d6 100755 --- a/tests/system-test/test.sh +++ b/tests/system-test/test.sh @@ -45,13 +45,11 @@ declare -x SIM_DIR=$TOP_DIR/sim PROGRAM=$BUILD_DIR/build/bin/tsim PRG_DIR=$SIM_DIR/tsim ASAN_DIR=$SIM_DIR/asan -SYSTEM_TEST_DIR=$CODE_DIR/tests/system-test chmod -R 777 $PRG_DIR echo "------------------------------------------------------------------------" echo "Start TDengine Testing Case ..." echo "BUILD_DIR: $BUILD_DIR" -echo "SYSTEM_TEST_DIR : $SYSTEM_TEST_DIR" echo "SIM_DIR : $SIM_DIR" echo "CODE_DIR : $CODE_DIR" echo "ASAN_DIR : $ASAN_DIR" @@ -61,24 +59,36 @@ rm -rf $SIM_DIR/* mkdir -p $PRG_DIR mkdir -p $ASAN_DIR -cd $SYSTEM_TEST_DIR +cd $CODE_DIR ulimit -n 600000 ulimit -c unlimited #sudo sysctl -w kernel.core_pattern=$TOP_DIR/core.%p.%e echo "ExcuteCmd:" $* -echo "AsanDir:" $ASAN_DIR/psim.info +AsanFile=$ASAN_DIR/psim.asan +echo "AsanFile:" $AsanFile export LD_PRELOAD=libasan.so.5 -$* -a 2> $ASAN_DIR/psim.info +$* -a 2> $AsanFile -result=$? -echo "Execute result:" $result +export LD_PRELOAD= +AsanFileLen=`cat $AsanFile | wc -l` +while [ $AsanFileLen -lt 10 ] +do + sleep 1 + `cat $AsanFile | wc -l` +done +echo "AsanFileLen:" $AsanFileLen -if [ $result -eq 0 ]; then - $CODE_DIR/tests/script/sh/checkAsan.sh +AsanFileSuccessLen=`grep -w successfully $AsanFile | wc -l` +echo "AsanFileSuccessLen:" $AsanFileSuccessLen + +if [ $AsanFileSuccessLen -gt 0 ]; then + echo "Execute script successfully and check asan" + $CODE_DIR/../script/sh/checkAsan.sh else + echo "Execute script failure" exit 1 fi From 15c0bb944613f249ba2911218718ae04acd96187 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 20 Nov 2022 21:26:54 +0800 Subject: [PATCH 35/59] test: python asan --- tests/pytest/util/dnodes.py | 15 ++++++++++++--- tests/script/sh/checkAsan.sh | 2 +- tests/script/sh/exec.sh | 2 +- tests/script/sh/sigint_stop_dnodes.sh | 2 +- tests/script/sh/stop_dnodes.sh | 2 +- tests/system-test/{test.sh => pytest.sh} | 3 ++- 6 files changed, 18 insertions(+), 8 deletions(-) rename tests/system-test/{test.sh => pytest.sh} (98%) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index edb3d761cb..22e6127973 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -162,7 +162,11 @@ class TDDnode: def setAsan(self, value): self.asan = value if value: - self.execPath = os.path.abspath(self.path + "/tests/script/sh/exec.sh") + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + self.execPath = os.path.abspath(self.path + "/community/tests/script/sh/exec.sh") + else: + self.execPath = os.path.abspath(self.path + "/tests/script/sh/exec.sh") def getDataSize(self): totalSize = 0 @@ -670,8 +674,13 @@ class TDDnodes: def setAsan(self, value): self.asan = value if value: - self.stopDnodesPath = os.path.abspath(self.path + "/tests/script/sh/stop_dnodes.sh") - self.stopDnodesSigintPath = os.path.abspath(self.path + "/tests/script/sh/sigint_stop_dnodes.sh") + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + self.stopDnodesPath = os.path.abspath(self.path + "/community/tests/script/sh/stop_dnodes.sh") + self.stopDnodesSigintPath = os.path.abspath(self.path + "/community/tests/script/sh/sigint_stop_dnodes.sh") + else: + self.stopDnodesPath = os.path.abspath(self.path + "/tests/script/sh/stop_dnodes.sh") + self.stopDnodesSigintPath = os.path.abspath(self.path + "/tests/script/sh/sigint_stop_dnodes.sh") tdLog.info("run in address sanitizer mode") def setKillValgrind(self, value): diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 8e5738e911..5dfb19e0ae 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -3,7 +3,7 @@ set +e #set -x -export LD_PRELOAD= +unset LD_PRELOAD SCRIPT_DIR=`dirname $0` cd $SCRIPT_DIR/../ SCRIPT_DIR=`pwd` diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index ad671631a9..c8cb121b8a 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -11,7 +11,7 @@ set +e #set -x -export LD_PRELOAD= +unset LD_PRELOAD UNAME_BIN=`which uname` OS_TYPE=`$UNAME_BIN` diff --git a/tests/script/sh/sigint_stop_dnodes.sh b/tests/script/sh/sigint_stop_dnodes.sh index 12b58225de..a398e00772 100755 --- a/tests/script/sh/sigint_stop_dnodes.sh +++ b/tests/script/sh/sigint_stop_dnodes.sh @@ -3,7 +3,7 @@ set +e #set -x -export LD_PRELOAD= +unset LD_PRELOAD UNAME_BIN=`which uname` OS_TYPE=`$UNAME_BIN` diff --git a/tests/script/sh/stop_dnodes.sh b/tests/script/sh/stop_dnodes.sh index 34777ccb59..ce2d7144f9 100755 --- a/tests/script/sh/stop_dnodes.sh +++ b/tests/script/sh/stop_dnodes.sh @@ -3,7 +3,7 @@ set +e #set -x -export LD_PRELOAD= +unset LD_PRELOAD UNAME_BIN=`which uname` OS_TYPE=`$UNAME_BIN` diff --git a/tests/system-test/test.sh b/tests/system-test/pytest.sh similarity index 98% rename from tests/system-test/test.sh rename to tests/system-test/pytest.sh index 6fbc9961d6..f6d081da2a 100755 --- a/tests/system-test/test.sh +++ b/tests/system-test/pytest.sh @@ -69,10 +69,11 @@ echo "ExcuteCmd:" $* AsanFile=$ASAN_DIR/psim.asan echo "AsanFile:" $AsanFile +unset LD_PRELOAD export LD_PRELOAD=libasan.so.5 $* -a 2> $AsanFile -export LD_PRELOAD= +unset LD_PRELOAD AsanFileLen=`cat $AsanFile | wc -l` while [ $AsanFileLen -lt 10 ] do From c8f6235027c9651208c4537b6d62d2b56f5185d5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 20 Nov 2022 22:09:48 +0800 Subject: [PATCH 36/59] test: python asan --- tests/parallel_test/cases.task | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e1b7f9cb32..12e0a9c89d 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -438,11 +438,11 @@ ,,,system-test,python3 ./test.py -f 1-insert/database_pre_suf.py ,,,system-test,python3 ./test.py -f 1-insert/InsertFuturets.py ,,,system-test,python3 ./test.py -f 0-others/show.py -,,,system-test,./test.sh python3 ./test.py -f 2-query/abs.py +,,,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py ,,,system-test,python3 ./test.py -f 2-query/abs.py -R ,,,system-test,python3 ./test.py -f 2-query/and_or_for_byte.py ,,,system-test,python3 ./test.py -f 2-query/and_or_for_byte.py -R -,,,system-test,./test.sh python3 ./test.py -f 2-query/apercentile.py +,,,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py ,,,system-test,python3 ./test.py -f 2-query/apercentile.py -R ,,,system-test,python3 ./test.py -f 2-query/arccos.py ,,,system-test,python3 ./test.py -f 2-query/arccos.py -R From d17dd17be49a811cbafaf2d0fbb479a9111aad28 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 20 Nov 2022 23:14:33 +0800 Subject: [PATCH 37/59] test: preload libasan --- tests/system-test/pytest.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/system-test/pytest.sh b/tests/system-test/pytest.sh index f6d081da2a..d5ae59b4fe 100755 --- a/tests/system-test/pytest.sh +++ b/tests/system-test/pytest.sh @@ -70,7 +70,14 @@ AsanFile=$ASAN_DIR/psim.asan echo "AsanFile:" $AsanFile unset LD_PRELOAD -export LD_PRELOAD=libasan.so.5 +#export LD_PRELOAD=libasan.so.5 +#echo "export1:" $? +export LD_PRELOAD=`gcc -print-file-name=libasan.so` +echo "export2:" $? + +echo "export4:" "$BUILD_DIR/build/bin/taos" +ldd $BUILD_DIR/build/bin/taos + $* -a 2> $AsanFile unset LD_PRELOAD From ee9f5740cfb0ed0ba830564c3b496b7d4bf184fe Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Nov 2022 08:52:32 +0800 Subject: [PATCH 38/59] test: check asan errors --- tests/parallel_test/cases.task | 2 +- tests/script/sh/checkAsan.sh | 7 ++++++- tests/system-test/2-query/abs.py | 2 ++ tests/system-test/pytest.sh | 8 ++------ 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 12e0a9c89d..a9a02d8380 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -278,7 +278,7 @@ ,,y,script,./test.sh -f tsim/stable/values.sim ,,y,script,./test.sh -f tsim/stable/vnode3.sim ,,y,script,./test.sh -f tsim/stable/metrics_idx.sim -,,y,script,./test.sh -f tsim/sma/drop_sma.sim +,,n,script,./test.sh -f tsim/sma/drop_sma.sim ,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 5dfb19e0ae..c53fde1171 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -22,19 +22,24 @@ error_num=`cat ${LOG_DIR}/*.asan | grep "ERROR" | wc -l` memory_leak=`cat ${LOG_DIR}/*.asan | grep "Direct leak" | wc -l` indirect_leak=`cat ${LOG_DIR}/*.asan | grep "Indirect leak" | wc -l` runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | wc -l` +python_error=`cat ${LOG_DIR}/*.info | grep "stack" | wc -l` echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m" echo -e "\033[44;32;1m"asan memory_leak: $memory_leak"\033[0m" echo -e "\033[44;32;1m"asan indirect_leak: $indirect_leak"\033[0m" echo -e "\033[44;32;1m"asan runtime error: $runtime_error"\033[0m" +echo -e "\033[44;32;1m"asan python error: $python_error"\033[0m" -let "errors=$error_num+$memory_leak+$indirect_leak+$runtime_error" +let "errors=$error_num+$memory_leak+$indirect_leak+$runtime_error+$python_error" if [ $errors -eq 0 ]; then echo -e "\033[44;32;1m"no asan errors"\033[0m" exit 0 else echo -e "\033[44;31;1m"asan total errors: $errors"\033[0m" + if [ $python_error -ne 0 ]; then + cat ${LOG_DIR}/*.info + fi cat ${LOG_DIR}/*.asan exit 1 fi \ No newline at end of file diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index d7478a55a8..dcd9e6d620 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -487,6 +487,8 @@ class TDTestCase: self.check_result_auto( f"select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from {dbname}.sub1_bound;", f"select abs(c1) from {dbname}.sub1_bound") + tdSql.query(f"select t1 xxx") + # check basic elem for table per row tdSql.query( f"select abs(c1) ,abs(c2) , abs(c3) , abs(c4), abs(c5), abs(c6) from {dbname}.sub1_bound ") diff --git a/tests/system-test/pytest.sh b/tests/system-test/pytest.sh index d5ae59b4fe..3f03058342 100755 --- a/tests/system-test/pytest.sh +++ b/tests/system-test/pytest.sh @@ -66,17 +66,13 @@ ulimit -c unlimited #sudo sysctl -w kernel.core_pattern=$TOP_DIR/core.%p.%e echo "ExcuteCmd:" $* -AsanFile=$ASAN_DIR/psim.asan +AsanFile=$ASAN_DIR/psim.info echo "AsanFile:" $AsanFile unset LD_PRELOAD #export LD_PRELOAD=libasan.so.5 -#echo "export1:" $? export LD_PRELOAD=`gcc -print-file-name=libasan.so` -echo "export2:" $? - -echo "export4:" "$BUILD_DIR/build/bin/taos" -ldd $BUILD_DIR/build/bin/taos +echo "Preload AsanSo:" $? $* -a 2> $AsanFile From bb6ca0c0287630b26405402a2ae19fbad4ac8a14 Mon Sep 17 00:00:00 2001 From: wenzhouwww Date: Mon, 21 Nov 2022 09:32:51 +0800 Subject: [PATCH 39/59] Update crash_gen_main.py --- tests/pytest/crash_gen/crash_gen_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 49b68965d0..f8c5f970c5 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -2025,7 +2025,7 @@ class TdSuperTable: conf.set("enable.auto.commit", "true") def tmq_commit_cb_print(tmq, resp, offset, param=None): print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}") - # conf.set_auto_commit_cb(tmq_commit_cb_print, None) + conf.set_auto_commit_cb(tmq_commit_cb_print, None) consumer = conf.new_consumer() topic_list = TaosTmqList() for topic in current_topic_list: From c454f849b21f238af3fe3cb7dcb3fcb476d1d9cc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Nov 2022 09:45:38 +0800 Subject: [PATCH 40/59] test: adjust python asan script --- tests/script/sh/checkAsan.sh | 2 +- tests/system-test/2-query/abs.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index c53fde1171..8759db8722 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -22,7 +22,7 @@ error_num=`cat ${LOG_DIR}/*.asan | grep "ERROR" | wc -l` memory_leak=`cat ${LOG_DIR}/*.asan | grep "Direct leak" | wc -l` indirect_leak=`cat ${LOG_DIR}/*.asan | grep "Indirect leak" | wc -l` runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | wc -l` -python_error=`cat ${LOG_DIR}/*.info | grep "stack" | wc -l` +python_error=`cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l` echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m" echo -e "\033[44;32;1m"asan memory_leak: $memory_leak"\033[0m" diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index dcd9e6d620..d7478a55a8 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -487,8 +487,6 @@ class TDTestCase: self.check_result_auto( f"select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from {dbname}.sub1_bound;", f"select abs(c1) from {dbname}.sub1_bound") - tdSql.query(f"select t1 xxx") - # check basic elem for table per row tdSql.query( f"select abs(c1) ,abs(c2) , abs(c3) , abs(c4), abs(c5), abs(c6) from {dbname}.sub1_bound ") From 9aaf2bea428c4b62af10b128cfe55b59a58b7485 Mon Sep 17 00:00:00 2001 From: Pan YANG Date: Mon, 21 Nov 2022 10:32:22 +0800 Subject: [PATCH 41/59] docs: fix pathname of resources file --- docs/zh/05-get-started/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/05-get-started/index.md b/docs/zh/05-get-started/index.md index f179dea26d..832310aa7c 100644 --- a/docs/zh/05-get-started/index.md +++ b/docs/zh/05-get-started/index.md @@ -24,7 +24,7 @@ TDengine 知识地图中涵盖了 TDengine 的各种知识点，揭示了各概
- +
图 1. TDengine 知识地图
From e5470224cf0e9c4bc5ca9a1662aada971d9ad76b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Nov 2022 11:01:44 +0800 Subject: [PATCH 42/59] test: close all dnodes when tsim execution is complete --- tests/script/sh/sigint_stop_dnodes.sh | 2 +- tests/script/test.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/script/sh/sigint_stop_dnodes.sh b/tests/script/sh/sigint_stop_dnodes.sh index a398e00772..83a4f1c1d5 100755 --- a/tests/script/sh/sigint_stop_dnodes.sh +++ b/tests/script/sh/sigint_stop_dnodes.sh @@ -8,9 +8,9 @@ UNAME_BIN=`which uname` OS_TYPE=`$UNAME_BIN` PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` +echo "Killing taosd processes " $PID while [ -n "$PID" ]; do #echo "Killing taosd processes " $PID kill $PID - PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` done diff --git a/tests/script/test.sh b/tests/script/test.sh index f4979bfa5a..a7a5d34fbe 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -141,6 +141,7 @@ if [ -n "$FILE_NAME" ]; then echo "Execute result:" $result if [ $result -eq 0 ]; then + $CODE_DIR/sh/sigint_stop_dnodes.sh $CODE_DIR/sh/checkAsan.sh else echo "TSIM has asan errors" From 391389e989f7dc7a2f2175b5e75b7671568d9784 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Nov 2022 11:47:55 +0800 Subject: [PATCH 43/59] test: close all dnodes when tsim execution is complete --- tests/parallel_test/cases.task | 7 +- tests/script/tsim/sma/sma_leak.sim | 154 +++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+), 3 deletions(-) create mode 100644 tests/script/tsim/sma/sma_leak.sim diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index a9a02d8380..f9e8cd6271 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -216,7 +216,7 @@ ,,y,script,./test.sh -f tsim/stream/drop_stream.sim ,,y,script,./test.sh -f tsim/stream/fillHistoryBasic1.sim ,,y,script,./test.sh -f tsim/stream/fillHistoryBasic2.sim -,,y,script,./test.sh -f tsim/stream/fillHistoryBasic3.sim +,,n,script,./test.sh -f tsim/stream/fillHistoryBasic3.sim ,,y,script,./test.sh -f tsim/stream/distributeInterval0.sim ,,y,script,./test.sh -f tsim/stream/distributeIntervalRetrive0.sim ,,y,script,./test.sh -f tsim/stream/distributeSession0.sim @@ -227,11 +227,11 @@ ,,y,script,./test.sh -f tsim/stream/triggerSession0.sim ,,y,script,./test.sh -f tsim/stream/partitionby.sim ,,y,script,./test.sh -f tsim/stream/partitionby1.sim -,,y,script,./test.sh -f tsim/stream/schedSnode.sim +,,n,script,./test.sh -f tsim/stream/schedSnode.sim ,,y,script,./test.sh -f tsim/stream/windowClose.sim ,,y,script,./test.sh -f tsim/stream/ignoreExpiredData.sim ,,y,script,./test.sh -f tsim/stream/sliding.sim -,,y,script,./test.sh -f tsim/stream/partitionbyColumnInterval.sim +,,n,script,./test.sh -f tsim/stream/partitionbyColumnInterval.sim ,,y,script,./test.sh -f tsim/stream/partitionbyColumnSession.sim ,,y,script,./test.sh -f tsim/stream/partitionbyColumnState.sim ,,y,script,./test.sh -f tsim/stream/deleteInterval.sim @@ -279,6 +279,7 @@ ,,y,script,./test.sh -f tsim/stable/vnode3.sim ,,y,script,./test.sh -f tsim/stable/metrics_idx.sim ,,n,script,./test.sh -f tsim/sma/drop_sma.sim +,,n,script,./test.sh -f tsim/sma/sma_leak.sim ,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim diff --git a/tests/script/tsim/sma/sma_leak.sim b/tests/script/tsim/sma/sma_leak.sim new file mode 100644 index 0000000000..4f2d1ebeb0 --- /dev/null +++ b/tests/script/tsim/sma/sma_leak.sim @@ -0,0 +1,154 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/cfg.sh -n dnode2 -c supportVnodes -v 4 +system sh/cfg.sh -n dnode3 -c supportVnodes -v 4 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ========== step2 +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start + +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql select * from information_schema.ins_dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 3 then + return -1 +endi +if $data(1)[4] != ready then + goto step2 +endi +if $data(2)[4] != ready then + goto step2 +endi +if $data(3)[4] != ready then + goto step2 +endi + +print ========== step3 +sql create database d1 vgroups 1 +sql use d1; + +print --> create stb +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned); + +print --> create sma +sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) interval(6m,10s) sliding(6m); + +return + +print --> show sma +sql show indexes from stb from d1; +if $rows != 1 then + return -1 +endi +if $data[0][0] != sma_index_name1 then + return -1 +endi +if $data[0][1] != d1 then + return -1 +endi +if $data[0][2] != stb then + return -1 +endi + +print --> drop stb +sql drop table stb; + +print ========== step4 repeat + +print --> create stb +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned); + +print --> create sma +sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) interval(6m,10s) sliding(6m); + +print --> show sma +sql show indexes from stb from d1; +if $rows != 1 then + return -1 +endi +if $data[0][0] != sma_index_name1 then + return -1 +endi +if $data[0][1] != d1 then + return -1 +endi +if $data[0][2] != stb then + return -1 +endi + +print --> drop stb +sql drop table stb; + +print ========== step5 +sql drop database if exists db; +sql create database db duration 300; +sql use db; +sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint, c_float float, c_double double, c_bool bool, c_binary binary(16), c_nchar nchar(32), c_ts timestamp, c_tint_un tinyint unsigned, c_sint_un smallint unsigned, c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int); +sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s; + +print ========== step6 repeat +sql drop database if exists db; +sql create database db duration 300; +sql use db; +sql create table stb1(ts timestamp, c_int int, c_bint bigint ) tags (t_int int); +sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s; + +print ========== step7 +sql drop database if exists db; +sql create database db duration 300; +sql use db; +sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int); + +sql create table ct1 using stb1 tags ( 1 ); +sql create table ct2 using stb1 tags ( 2 ); +sql create table ct3 using stb1 tags ( 3 ); +sql create table ct4 using stb1 tags ( 4 ); + +sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s; +sql CREATE SMA INDEX sma_index_2 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) max_delay 6m; +sql CREATE SMA INDEX sma_index_3 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) watermark 5s max_delay 6m; + +sql DROP INDEX sma_index_1 ; +sql DROP INDEX sma_index_2 ; +sql DROP INDEX sma_index_3 ; + +print ========== step8 +sql drop database if exists db; +sql create database db duration 300; +sql use db; +sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int); + +sql create table ct1 using stb1 tags ( 1 ); +sql create table ct2 using stb1 tags ( 2 ); +sql create table ct3 using stb1 tags ( 3 ); +sql create table ct4 using stb1 tags ( 4 ); + +sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s; +sql CREATE SMA INDEX sma_index_2 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) max_delay 6m; +sql CREATE SMA INDEX sma_index_3 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) watermark 5s max_delay 6m; + +sql DROP INDEX sma_index_1 ; +sql DROP INDEX sma_index_2 ; +sql DROP INDEX sma_index_3 ; + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT From ada76477fc818986a18fe64a81501445b3d23ed3 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 21 Nov 2022 11:54:25 +0800 Subject: [PATCH 44/59] fix: free stream state cursor --- source/libs/stream/src/streamState.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 0374e22a4a..ea31347db5 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -411,6 +411,7 @@ int32_t streamStateGetFirst(SStreamState* pState, SWinKey* key) { streamStatePut(pState, &tmp, NULL, 0); SStreamStateCur* pCur = streamStateSeekKeyNext(pState, &tmp); int32_t code = streamStateGetKVByCur(pCur, key, NULL, 0); + streamStateFreeCur(pCur); streamStateDel(pState, &tmp); return code; } From afe7a19ba0f3affe572927b437dba3d1fff24b16 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Nov 2022 14:09:12 +0800 Subject: [PATCH 45/59] test: add asan case --- tests/parallel_test/cases.task | 24 ++++++++++++------------ tests/script/sh/checkAsan.sh | 9 ++++++++- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index f9e8cd6271..890412f8eb 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -439,31 +439,31 @@ ,,,system-test,python3 ./test.py -f 1-insert/database_pre_suf.py ,,,system-test,python3 ./test.py -f 1-insert/InsertFuturets.py ,,,system-test,python3 ./test.py -f 0-others/show.py -,,,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py ,,,system-test,python3 ./test.py -f 2-query/abs.py -R -,,,system-test,python3 ./test.py -f 2-query/and_or_for_byte.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/and_or_for_byte.py ,,,system-test,python3 ./test.py -f 2-query/and_or_for_byte.py -R -,,,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py ,,,system-test,python3 ./test.py -f 2-query/apercentile.py -R -,,,system-test,python3 ./test.py -f 2-query/arccos.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py ,,,system-test,python3 ./test.py -f 2-query/arccos.py -R -,,,system-test,python3 ./test.py -f 2-query/arcsin.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py ,,,system-test,python3 ./test.py -f 2-query/arcsin.py -R -,,,system-test,python3 ./test.py -f 2-query/arctan.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py ,,,system-test,python3 ./test.py -f 2-query/arctan.py -R ,,,system-test,python3 ./test.py -f 2-query/avg.py ,,,system-test,python3 ./test.py -f 2-query/avg.py -R -,,,system-test,python3 ./test.py -f 2-query/between.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py ,,,system-test,python3 ./test.py -f 2-query/between.py -R -,,,system-test,python3 ./test.py -f 2-query/bottom.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py ,,,system-test,python3 ./test.py -f 2-query/bottom.py -R -,,,system-test,python3 ./test.py -f 2-query/cast.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py ,,,system-test,python3 ./test.py -f 2-query/cast.py -R -,,,system-test,python3 ./test.py -f 2-query/ceil.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py ,,,system-test,python3 ./test.py -f 2-query/ceil.py -R -,,,system-test,python3 ./test.py -f 2-query/char_length.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py ,,,system-test,python3 ./test.py -f 2-query/char_length.py -R -,,,system-test,python3 ./test.py -f 2-query/check_tsdb.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py ,,,system-test,python3 ./test.py -f 2-query/check_tsdb.py -R ,,,system-test,python3 ./test.py -f 2-query/concat.py ,,,system-test,python3 ./test.py -f 2-query/concat.py -R diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 8759db8722..074956534f 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -21,7 +21,14 @@ LOG_DIR=$TAOS_DIR/sim/asan error_num=`cat ${LOG_DIR}/*.asan | grep "ERROR" | wc -l` memory_leak=`cat ${LOG_DIR}/*.asan | grep "Direct leak" | wc -l` indirect_leak=`cat ${LOG_DIR}/*.asan | grep "Indirect leak" | wc -l` -runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | wc -l` + +# ignore +# /root/TDengine/source/libs/scalar/src/sclfunc.c:735:11: runtime error: 4.75783e+11 is outside the range of representable values of type 'signed char' +# /root/TDengine/source/libs/scalar/src/sclfunc.c:790:11: runtime error: 3.4e+38 is outside the range of representable values of type 'long int' +# /root/TDengine/source/libs/scalar/src/sclfunc.c:772:11: runtime error: 3.52344e+09 is outside the range of representable values of type 'int' +# /root/TDengine/source/libs/scalar/src/sclfunc.c:753:11: runtime error: 4.75783e+11 is outside the range of representable values of type 'short int' +runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type" | wc -l` + python_error=`cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l` echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m" From 36a43fc016ae9db8b598d9be86e762e8a3e01003 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Mon, 21 Nov 2022 14:46:46 +0800 Subject: [PATCH 46/59] fix: set sleep time to 10s, set curl timeout --- packaging/docker/run.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packaging/docker/run.sh b/packaging/docker/run.sh index b5d6b011ea..b02b3cbd8a 100755 --- a/packaging/docker/run.sh +++ b/packaging/docker/run.sh @@ -4,7 +4,7 @@ TAOS_RUN_TAOSBENCHMARK_TEST_ONCE=0 #ADMIN_URL=${ADMIN_URL:-http://172.26.10.84:10001} TAOSD_STARTUP_TIMEOUT_SECOND=${TAOSD_STARTUP_TIMEOUT_SECOND:-160} TAOS_TIMEOUT_SECOND=${TAOS_TIMEOUT_SECOND:-5} -BACKUP_CORE_FOLDER=/data/corefile +BACKUP_CORE_FOLDER=/var/log/corefile ALERT_URL=app/system/alert/add echo "ADMIN_URL: ${ADMIN_URL}" @@ -37,7 +37,7 @@ function post_error_msg() { echo "service_state: ${service_state}" echo "`date` service_msg: ${service_msg}" echo "${taos_version}" - curl -X POST -H "Content-Type: application/json" \ + curl --connect-timeout 10 --max-time 20 -X POST -H "Content-Type: application/json" \ -d"{\"appName\":\"${app_name}\",\ \"alertLevel\":\"${service_state}\",\ \"taosVersion\":\"${taos_version}\",\ @@ -77,7 +77,7 @@ function post_disk_error_msg() { echo "disk_state: ${disk_state}" echo "`date` disk_msg: ${disk_msg}" echo "${taos_version}" - curl -X POST -H "Content-Type: application/json" \ + curl --connect-timeout 10 --max-time 20 -X POST -H "Content-Type: application/json" \ -d"{\"appName\":\"${app_name}\",\ \"alertLevel\":\"${disk_state}\",\ \"taosVersion\":\"${taos_version}\",\ @@ -200,5 +200,5 @@ do if [ $? -ne 0 ]; then taosadapter & fi - sleep 30 + sleep 10 done From cba374ae6b52b56c69f69ee41a9bfc348ae994de Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 21 Nov 2022 15:02:05 +0800 Subject: [PATCH 47/59] enh: add query response message processing --- source/libs/scheduler/src/schRemote.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 47a34ede7e..b0bc0df850 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -334,17 +334,17 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } - SQueryTableRsp *rsp = (SQueryTableRsp *)msg; - rsp->code = ntohl(rsp->code); - rsp->sversion = ntohl(rsp->sversion); - rsp->tversion = ntohl(rsp->tversion); - rsp->affectedRows = be64toh(rsp->affectedRows); + SQueryTableRsp rsp = {0}; + if (tDeserializeSQueryTableRsp(msg, msgSize, &rsp) < 0) { + SCH_TASK_ELOG("tDeserializeSQueryTableRsp failed, msgSize:%d", msgSize); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_MSG); + } + + SCH_ERR_JRET(rsp.code); - SCH_ERR_JRET(rsp->code); + SCH_ERR_JRET(schSaveJobExecRes(pJob, &rsp)); - SCH_ERR_JRET(schSaveJobExecRes(pJob, rsp)); - - atomic_add_fetch_32(&pJob->resNumOfRows, rsp->affectedRows); + atomic_add_fetch_32(&pJob->resNumOfRows, rsp.affectedRows); taosMemoryFreeClear(msg); From 1bd0cc4a5a0c71017560e8ddcc63b2c766297382 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 21 Nov 2022 15:09:18 +0800 Subject: [PATCH 48/59] refactor: do some internal refactor, and update the logs. --- source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/src/exchangeoperator.c | 20 +++++++++----------- source/libs/executor/src/sortoperator.c | 4 +++- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 9b9a1ef259..f179c7bd41 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -239,6 +239,7 @@ typedef struct SSourceDataInfo { int32_t index; SRetrieveTableRsp* pRsp; uint64_t totalRows; + int64_t startTime; int32_t code; EX_SOURCE_STATUS status; const char* taskId; diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index c57a1b38eb..a28066003a 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -44,7 +44,7 @@ typedef struct SFetchRspHandleWrapper { static void destroyExchangeOperatorInfo(void* param); static void freeBlock(void* pParam); static void freeSourceDataInfo(void* param); -static void* setAllSourcesCompleted(SOperatorInfo* pOperator, int64_t startTs); +static void* setAllSourcesCompleted(SOperatorInfo* pOperator); static int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code); static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex); @@ -59,7 +59,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn size_t totalSources = taosArrayGetSize(pExchangeInfo->pSourceDataInfo); int32_t completed = getCompletedSources(pExchangeInfo->pSourceDataInfo); if (completed == totalSources) { - setAllSourcesCompleted(pOperator, pExchangeInfo->openedTs); + setAllSourcesCompleted(pOperator); return; } @@ -113,7 +113,8 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn taosArrayPush(pExchangeInfo->pResultBlockList, &pb); } - updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, pExchangeInfo->openedTs, pOperator); + updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, pDataInfo->startTime, pOperator); + pDataInfo->totalRows += pRetrieveRsp->numOfRows; if (pRsp->completed == 1) { pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; @@ -388,6 +389,7 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, sourceIndex); SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); + pDataInfo->startTime = taosGetTimestampUs(); ASSERT(pDataInfo->status == EX_SOURCE_DATA_NOT_READY); @@ -493,18 +495,14 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pCo return TSDB_CODE_SUCCESS; } -void* setAllSourcesCompleted(SOperatorInfo* pOperator, int64_t startTs) { +void* setAllSourcesCompleted(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - int64_t el = taosGetTimestampUs() - startTs; SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; - - pLoadInfo->totalElapsed += el; - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - qDebug("%s all %" PRIzu " sources are exhausted, total rows: %" PRIu64 " bytes:%" PRIu64 ", elapsed:%.2f ms", - GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, + qDebug("%s all %" PRIzu " sources are exhausted, total rows: %" PRIu64 ", %.2f Kb, elapsed:%.2f ms", + GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, pLoadInfo->totalElapsed / 1000.0); setOperatorCompleted(pOperator); @@ -566,7 +564,7 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { while (1) { if (pExchangeInfo->current >= totalSources) { - setAllSourcesCompleted(pOperator, startTs); + setAllSourcesCompleted(pOperator); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index add580ce7c..14e3163455 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -690,6 +690,8 @@ SSDataBlock* doMultiwayMerge(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, code); } + qDebug("start to merge final sorted rows, %s", GET_TASKID(pTaskInfo)); + SSDataBlock* pBlock = getMultiwaySortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pInfo->matchInfo.pList, pOperator); if (pBlock != NULL) { pOperator->resultInfo.totalRows += pBlock->info.rows; @@ -754,7 +756,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0); SSDataBlock* pInputBlock = createResDataBlock(pChildNode->pOutputDataBlockDesc); - initResultSizeInfo(&pOperator->resultInfo, 1024); + initResultSizeInfo(&pOperator->resultInfo, 4096); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); pInfo->groupSort = pMergePhyNode->groupSort; From f7f7b97279d9c87e12d73ab4410174a186fd38c1 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Mon, 21 Nov 2022 15:19:19 +0800 Subject: [PATCH 49/59] fix: mv does not work cross devices --- packaging/docker/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packaging/docker/run.sh b/packaging/docker/run.sh index b02b3cbd8a..3654beadcb 100755 --- a/packaging/docker/run.sh +++ b/packaging/docker/run.sh @@ -59,7 +59,8 @@ function check_taosd_exit_type() { if [ ! -z "$core_files" ]; then # move core files to another folder mkdir -p ${BACKUP_CORE_FOLDER} - mv ${core_folder}/${core_prefix}* ${BACKUP_CORE_FOLDER}/ + cp ${core_folder}/${core_prefix}* ${BACKUP_CORE_FOLDER}/ + rm -f ${core_folder}/${core_prefix}* set_service_state "error" "taosd exit with core file" else set_service_state "error" "taosd exit without core file" From 5d471a0ceb701b4457a98d659d054097356d255f Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 21 Nov 2022 15:32:27 +0800 Subject: [PATCH 50/59] enh(taosx): split block when none column not match --- source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/tq/tqExec.c | 79 +++++++++++++++++++++--------- source/dnode/vnode/src/tq/tqRead.c | 4 +- source/dnode/vnode/src/tq/tqSink.c | 20 ++++---- 4 files changed, 68 insertions(+), 36 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 370103c222..27084700b0 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -239,6 +239,7 @@ int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t v bool tqNextDataBlock(STqReader *pReader); bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader); +int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas); int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 48c14bc758..97d3acba93 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -44,7 +44,7 @@ static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, SMqDataRsp* pRs return 0; } -static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp) { +static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, int32_t n) { SMetaReader mr = {0}; metaReaderInit(&mr, pTq->pVnode->pMeta, 0); // TODO add reference to gurantee success @@ -52,8 +52,10 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp) { metaReaderClear(&mr); return -1; } - char* tbName = strdup(mr.me.name); - taosArrayPush(pRsp->blockTbName, &tbName); + for (int32_t i = 0; i < n; i++) { + char* tbName = strdup(mr.me.name); + taosArrayPush(pRsp->blockTbName, &tbName); + } metaReaderClear(&mr); return 0; } @@ -111,7 +113,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs if (pRsp->withTbName) { if (pRsp->rspOffset.type == TMQ_OFFSET__LOG) { int64_t uid = pExec->pExecReader->msgIter.uid; - tqAddTbNameToRsp(pTq, uid, pRsp); + tqAddTbNameToRsp(pTq, uid, pRsp, 1); } else { pRsp->withTbName = false; } @@ -155,7 +157,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta int64_t uid = 0; if (pOffset->type == TMQ_OFFSET__LOG) { uid = pExec->pExecReader->msgIter.uid; - if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp) < 0) { + if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp, 1) < 0) { continue; } } else { @@ -225,18 +227,27 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp STqExecHandle* pExec = &pHandle->execHandle; ASSERT(pExec->subType != TOPIC_SUB_TYPE__COLUMN); + SArray* pBlocks = taosArrayInit(0, sizeof(SSDataBlock)); + SArray* pSchemas = taosArrayInit(0, sizeof(SSchemaWrapper)); + if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { STqReader* pReader = pExec->pExecReader; tqReaderSetDataMsg(pReader, pReq, 0); while (tqNextDataBlock(pReader)) { - SSDataBlock block = {0}; - if (tqRetrieveDataBlock(&block, pReader) < 0) { + /*SSDataBlock block = {0};*/ + /*if (tqRetrieveDataBlock(&block, pReader) < 0) {*/ + /*if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;*/ + /*}*/ + + taosArrayClear(pBlocks); + taosArrayClear(pSchemas); + if (tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; } if (pRsp->withTbName) { int64_t uid = pExec->pExecReader->msgIter.uid; - if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp) < 0) { - blockDataFreeRes(&block); + if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlocks)) < 0) { + taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); continue; } } @@ -255,24 +266,34 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp pRsp->createTableNum++; } } - tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock), - pTq->pVnode->config.tsdbCfg.precision); - blockDataFreeRes(&block); - tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp); - pRsp->blockNum++; + for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) { + SSDataBlock* pBlock = taosArrayGet(pBlocks, i); + tqAddBlockDataToRsp(pBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlock->pDataBlock), + pTq->pVnode->config.tsdbCfg.precision); + blockDataFreeRes(pBlock); + SSchemaWrapper* pSW = taosArrayGet(pSchemas, i); + taosArrayPush(pRsp->blockSchema, &pSW); + pRsp->blockNum++; + } } } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { STqReader* pReader = pExec->pExecReader; tqReaderSetDataMsg(pReader, pReq, 0); while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) { - SSDataBlock block = {0}; - if (tqRetrieveDataBlock(&block, pReader) < 0) { + /*SSDataBlock block = {0};*/ + /*if (tqRetrieveDataBlock(&block, pReader) < 0) {*/ + /*if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;*/ + /*}*/ + taosArrayClear(pBlocks); + taosArrayClear(pSchemas); + if (tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; } if (pRsp->withTbName) { int64_t uid = pExec->pExecReader->msgIter.uid; - if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp) < 0) { - blockDataFreeRes(&block); + if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlocks)) < 0) { + taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); + /*blockDataFreeRes(&block);*/ continue; } } @@ -291,14 +312,26 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp pRsp->createTableNum++; } } - tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock), - pTq->pVnode->config.tsdbCfg.precision); - blockDataFreeRes(&block); - tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp); - pRsp->blockNum++; + /*tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock),*/ + /*pTq->pVnode->config.tsdbCfg.precision);*/ + /*blockDataFreeRes(&block);*/ + /*tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp);*/ + /*pRsp->blockNum++;*/ + for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) { + SSDataBlock* pBlock = taosArrayGet(pBlocks, i); + tqAddBlockDataToRsp(pBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlock->pDataBlock), + pTq->pVnode->config.tsdbCfg.precision); + blockDataFreeRes(pBlock); + SSchemaWrapper* pSW = taosArrayGet(pSchemas, i); + taosArrayPush(pRsp->blockSchema, &pSW); + pRsp->blockNum++; + } } } + taosArrayDestroy(pBlocks); + taosArrayDestroy(pSchemas); + if (pRsp->blockNum == 0) { return -1; } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 5c5ba1205e..30897cbe48 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -556,7 +556,7 @@ FAIL: return -1; } -int32_t tqSplitRetrieveDataBlock(STqReader* pReader, SArray* blocks, SArray* schemas) { +int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas) { int32_t sversion = htonl(pReader->pBlock->sversion); if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || @@ -594,7 +594,7 @@ int32_t tqSplitRetrieveDataBlock(STqReader* pReader, SArray* blocks, SArray* sch int32_t curRow = 0; char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols); - if (assigned) return -1; + if (assigned == NULL) return -1; tInitSubmitBlkIter(&pReader->msgIter, pReader->pBlock, &pReader->blkIter); STSRowIter iter = {0}; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index b2624d1bc1..27bfea0534 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -349,7 +349,6 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d .contLen = len + sizeof(SMsgHead), }; if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { - rpcFreeCont(serializedDeleteReq); tqDebug("failed to put delete req into write-queue since %s", terrstr()); } } else { @@ -476,12 +475,12 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen; - SSubmitReq* ret = rpcMallocCont(cap); - ret->header.vgId = pVnode->config.vgId; - ret->length = sizeof(SSubmitReq); - ret->numOfBlocks = htonl(1); + SSubmitReq* pSubmit = rpcMallocCont(cap); + pSubmit->header.vgId = pVnode->config.vgId; + pSubmit->length = sizeof(SSubmitReq); + pSubmit->numOfBlocks = htonl(1); - SSubmitBlk* blkHead = POINTER_SHIFT(ret, sizeof(SSubmitReq)); + SSubmitBlk* blkHead = POINTER_SHIFT(pSubmit, sizeof(SSubmitReq)); blkHead->numOfRows = htonl(pDataBlock->info.rows); blkHead->sversion = htonl(pTSchema->version); @@ -531,17 +530,16 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d } blkHead->dataLen = htonl(dataLen); - ret->length += sizeof(SSubmitBlk) + schemaLen + dataLen; - ret->length = htonl(ret->length); + pSubmit->length += sizeof(SSubmitBlk) + schemaLen + dataLen; + pSubmit->length = htonl(pSubmit->length); SRpcMsg msg = { .msgType = TDMT_VND_SUBMIT, - .pCont = ret, - .contLen = ntohl(ret->length), + .pCont = pSubmit, + .contLen = ntohl(pSubmit->length), }; if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { - rpcFreeCont(ret); tqDebug("failed to put into write-queue since %s", terrstr()); } } From 2df8d3229a10ef28f9ea1758b464e9994fd53176 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Nov 2022 15:38:50 +0800 Subject: [PATCH 51/59] test: add asan case --- tests/parallel_test/cases.task | 56 +++++++++++++++++----------------- tests/system-test/pytest.sh | 11 ++++--- 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 890412f8eb..b1df6975d8 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -471,63 +471,63 @@ ,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -R ,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py ,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -R -,,,system-test,python3 ./test.py -f 2-query/cos.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py ,,,system-test,python3 ./test.py -f 2-query/cos.py -R -,,,system-test,python3 ./test.py -f 2-query/count_partition.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py ,,,system-test,python3 ./test.py -f 2-query/count_partition.py -R -,,,system-test,python3 ./test.py -f 2-query/count.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py ,,,system-test,python3 ./test.py -f 2-query/count.py -R -,,,system-test,python3 ./test.py -f 2-query/countAlwaysReturnValue.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py ,,,system-test,python3 ./test.py -f 2-query/countAlwaysReturnValue.py -R -,,,system-test,python3 ./test.py -f 2-query/db.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/db.py ,,,system-test,python3 ./test.py -f 2-query/db.py -R -,,,system-test,python3 ./test.py -f 2-query/diff.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py ,,,system-test,python3 ./test.py -f 2-query/diff.py -R -,,,system-test,python3 ./test.py -f 2-query/distinct.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py ,,,system-test,python3 ./test.py -f 2-query/distinct.py -R -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_apercentile.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py ,,,system-test,python3 ./test.py -f 2-query/distribute_agg_apercentile.py -R -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_avg.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py ,,,system-test,python3 ./test.py -f 2-query/distribute_agg_avg.py -R -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_count.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py ,,,system-test,python3 ./test.py -f 2-query/distribute_agg_count.py -R -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_max.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py ,,,system-test,python3 ./test.py -f 2-query/distribute_agg_max.py -R -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_min.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py ,,,system-test,python3 ./test.py -f 2-query/distribute_agg_min.py -R -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_spread.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py ,,,system-test,python3 ./test.py -f 2-query/distribute_agg_spread.py -R -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_stddev.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py ,,,system-test,python3 ./test.py -f 2-query/distribute_agg_stddev.py -R -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_sum.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py ,,,system-test,python3 ./test.py -f 2-query/distribute_agg_sum.py -R -,,,system-test,python3 ./test.py -f 2-query/explain.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py ,,,system-test,python3 ./test.py -f 2-query/explain.py -R -,,,system-test,python3 ./test.py -f 2-query/first.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py ,,,system-test,python3 ./test.py -f 2-query/first.py -R -,,,system-test,python3 ./test.py -f 2-query/floor.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py ,,,system-test,python3 ./test.py -f 2-query/floor.py -R -,,,system-test,python3 ./test.py -f 2-query/function_null.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py ,,,system-test,python3 ./test.py -f 2-query/function_null.py -R -,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_stateduration.py ,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -R -,,,system-test,python3 ./test.py -f 2-query/histogram.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/histogram.py ,,,system-test,python3 ./test.py -f 2-query/histogram.py -R -,,,system-test,python3 ./test.py -f 2-query/hyperloglog.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py ,,,system-test,python3 ./test.py -f 2-query/hyperloglog.py -R ,,,system-test,python3 ./test.py -f 2-query/interp.py ,,,system-test,python3 ./test.py -f 2-query/interp.py -R -,,,system-test,python3 ./test.py -f 2-query/irate.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py ,,,system-test,python3 ./test.py -f 2-query/irate.py -R -,,,system-test,python3 ./test.py -f 2-query/join.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py ,,,system-test,python3 ./test.py -f 2-query/join.py -R ,,,system-test,python3 ./test.py -f 2-query/last_row.py ,,,system-test,python3 ./test.py -f 2-query/last_row.py -R -,,,system-test,python3 ./test.py -f 2-query/last.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py ,,,system-test,python3 ./test.py -f 2-query/last.py -R -,,,system-test,python3 ./test.py -f 2-query/leastsquares.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py ,,,system-test,python3 ./test.py -f 2-query/leastsquares.py -R -,,,system-test,python3 ./test.py -f 2-query/length.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py ,,,system-test,python3 ./test.py -f 2-query/length.py -R ,,,system-test,python3 ./test.py -f 2-query/log.py ,,,system-test,python3 ./test.py -f 2-query/log.py -R @@ -535,7 +535,7 @@ ,,,system-test,python3 ./test.py -f 2-query/lower.py -R ,,,system-test,python3 ./test.py -f 2-query/ltrim.py ,,,system-test,python3 ./test.py -f 2-query/ltrim.py -R -,,,system-test,python3 ./test.py -f 2-query/mavg.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py ,,,system-test,python3 ./test.py -f 2-query/mavg.py -R ,,,system-test,python3 ./test.py -f 2-query/max_partition.py ,,,system-test,python3 ./test.py -f 2-query/max_partition.py -R diff --git a/tests/system-test/pytest.sh b/tests/system-test/pytest.sh index 3f03058342..148f80eca3 100755 --- a/tests/system-test/pytest.sh +++ b/tests/system-test/pytest.sh @@ -77,13 +77,14 @@ echo "Preload AsanSo:" $? $* -a 2> $AsanFile unset LD_PRELOAD -AsanFileLen=`cat $AsanFile | wc -l` -while [ $AsanFileLen -lt 10 ] +for ((i=1;i<=20;i++)) do - sleep 1 - `cat $AsanFile | wc -l` + AsanFileLen=`cat $AsanFile | wc -l` + echo "AsanFileLen:" $AsanFileLen + if [ $AsanFileLen -gt 10 ]; then + break + fi done -echo "AsanFileLen:" $AsanFileLen AsanFileSuccessLen=`grep -w successfully $AsanFile | wc -l` echo "AsanFileSuccessLen:" $AsanFileSuccessLen From 3e81e36522f24497fa4420e2518630a2af18df76 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 21 Nov 2022 15:52:46 +0800 Subject: [PATCH 52/59] feat: taosdump for windows (#18298) * feat: taos-tools 2333455 * feat: taosdump for windows --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 82a7052125..b996ffcd17 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 23e2b73 + GIT_TAG e00ebd9 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 8abdb21091295897881c8c14222026eab6fe38d6 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 21 Nov 2022 17:34:50 +0800 Subject: [PATCH 53/59] fix: fix memory leak issue --- source/libs/qworker/src/qwMsg.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 71f24f8acd..bb8a7cd140 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -385,6 +385,8 @@ int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg, bool chkGran code = qwPreprocessQuery(QW_FPARAMS(), &qwMsg); QW_SCH_TASK_DLOG("prerocessQuery end, handle:%p, code:%x", pMsg->info.handle, code); + tFreeSSubQueryMsg(&msg); + return TSDB_CODE_SUCCESS; } From 4f176e325d1623ba3437e47e8febd2a78f4fcaec Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 21 Nov 2022 18:01:48 +0800 Subject: [PATCH 54/59] fix: [ASAN] devive by zero --- source/libs/scalar/src/sclfunc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 336bc6d533..5496c5d1ab 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -24,6 +24,8 @@ static double tlog2(double v, double base) { return a; } else if (isnan(b) || isinf(b)) { return b; + } else if (b == 0) { + return INFINITY; } else { return a / b; } From 50b6d954c05708534ce03de606fe32547525cf08 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Nov 2022 18:03:56 +0800 Subject: [PATCH 55/59] test: add asan python case --- tests/parallel_test/cases.task | 82 +++++++++++++++++----------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index b1df6975d8..e849b7d443 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -418,27 +418,27 @@ ,,,system-test,python3 ./test.py -f 0-others/fsync.py ,,,system-test,python3 ./test.py -f 0-others/compatibility.py ,,,system-test,python3 ./test.py -f 1-insert/alter_database.py -,,,system-test,python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py -,,,system-test,python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py +,,y,system-test,./pytest,sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py +,,y,system-test,./pytest,sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py ,,,system-test,python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py ,,,system-test,python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py ,,,system-test,python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py -,,,system-test,python3 ./test.py -f 1-insert/alter_stable.py -,,,system-test,python3 ./test.py -f 1-insert/alter_table.py +,,y,system-test,./pytest,sh python3 ./test.py -f 1-insert/alter_stable.py +,,y,system-test,./pytest,sh python3 ./test.py -f 1-insert/alter_table.py ,,,system-test,python3 ./test.py -f 1-insert/boundary.py ,,,system-test,python3 ./test.py -f 1-insert/insertWithMoreVgroup.py ,,,system-test,python3 ./test.py -f 1-insert/table_comment.py ,,,system-test,python3 ./test.py -f 1-insert/time_range_wise.py ,,,system-test,python3 ./test.py -f 1-insert/block_wise.py ,,,system-test,python3 ./test.py -f 1-insert/create_retentions.py -,,,system-test,python3 ./test.py -f 1-insert/table_param_ttl.py +,,y,system-test,./pytest,sh python3 ./test.py -f 1-insert/table_param_ttl.py ,,,system-test,python3 ./test.py -f 1-insert/mutil_stage.py ,,,system-test,python3 ./test.py -f 1-insert/table_param_ttl.py -R -,,,system-test,python3 ./test.py -f 1-insert/update_data_muti_rows.py -,,,system-test,python3 ./test.py -f 1-insert/db_tb_name_check.py +,,y,system-test,./pytest,sh python3 ./test.py -f 1-insert/update_data_muti_rows.py +,,y,system-test,./pytest,sh python3 ./test.py -f 1-insert/db_tb_name_check.py ,,,system-test,python3 ./test.py -f 1-insert/database_pre_suf.py ,,,system-test,python3 ./test.py -f 1-insert/InsertFuturets.py -,,,system-test,python3 ./test.py -f 0-others/show.py +,,y,system-test,./pytest,sh python3 ./test.py -f 0-others/show.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py ,,,system-test,python3 ./test.py -f 2-query/abs.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/and_or_for_byte.py @@ -537,87 +537,87 @@ ,,,system-test,python3 ./test.py -f 2-query/ltrim.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py ,,,system-test,python3 ./test.py -f 2-query/mavg.py -R -,,,system-test,python3 ./test.py -f 2-query/max_partition.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/max_partition.py ,,,system-test,python3 ./test.py -f 2-query/max_partition.py -R -,,,system-test,python3 ./test.py -f 2-query/max.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/max.py ,,,system-test,python3 ./test.py -f 2-query/max.py -R -,,,system-test,python3 ./test.py -f 2-query/min.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/min.py ,,,system-test,python3 ./test.py -f 2-query/min.py -R ,,,system-test,python3 ./test.py -f 2-query/mode.py ,,,system-test,python3 ./test.py -f 2-query/mode.py -R -,,,system-test,python3 ./test.py -f 2-query/Now.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/Now.py ,,,system-test,python3 ./test.py -f 2-query/Now.py -R -,,,system-test,python3 ./test.py -f 2-query/percentile.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/percentile.py ,,,system-test,python3 ./test.py -f 2-query/percentile.py -R -,,,system-test,python3 ./test.py -f 2-query/pow.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/pow.py ,,,system-test,python3 ./test.py -f 2-query/pow.py -R -,,,system-test,python3 ./test.py -f 2-query/query_cols_tags_and_or.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py ,,,system-test,python3 ./test.py -f 2-query/query_cols_tags_and_or.py -R -,,,system-test,python3 ./test.py -f 2-query/round.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/round.py ,,,system-test,python3 ./test.py -f 2-query/round.py -R ,,,system-test,python3 ./test.py -f 2-query/rtrim.py ,,,system-test,python3 ./test.py -f 2-query/rtrim.py -R -,,,system-test,python3 ./test.py -f 2-query/sample.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/sample.py ,,,system-test,python3 ./test.py -f 2-query/sample.py -R ,,,system-test,python3 ./test.py -f 2-query/sin.py -,,,system-test,python3 ./test.py -f 2-query/sin.py -R +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/sin.py -R ,,,system-test,python3 ./test.py -f 2-query/smaTest.py ,,,system-test,python3 ./test.py -f 2-query/smaTest.py -R ,,,system-test,python3 ./test.py -f 2-query/sml.py ,,,system-test,python3 ./test.py -f 2-query/sml.py -R -,,,system-test,python3 ./test.py -f 2-query/spread.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/spread.py ,,,system-test,python3 ./test.py -f 2-query/spread.py -R -,,,system-test,python3 ./test.py -f 2-query/sqrt.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/sqrt.py ,,,system-test,python3 ./test.py -f 2-query/sqrt.py -R -,,,system-test,python3 ./test.py -f 2-query/statecount.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/statecount.py ,,,system-test,python3 ./test.py -f 2-query/statecount.py -R -,,,system-test,python3 ./test.py -f 2-query/stateduration.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/stateduration.py ,,,system-test,python3 ./test.py -f 2-query/stateduration.py -R ,,,system-test,python3 ./test.py -f 2-query/substr.py ,,,system-test,python3 ./test.py -f 2-query/substr.py -R -,,,system-test,python3 ./test.py -f 2-query/sum.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/sum.py ,,,system-test,python3 ./test.py -f 2-query/sum.py -R ,,,system-test,python3 ./test.py -f 2-query/tail.py ,,,system-test,python3 ./test.py -f 2-query/tail.py -R -,,,system-test,python3 ./test.py -f 2-query/tan.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/tan.py ,,,system-test,python3 ./test.py -f 2-query/tan.py -R -,,,system-test,python3 ./test.py -f 2-query/Timediff.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/Timediff.py ,,,system-test,python3 ./test.py -f 2-query/Timediff.py -R -,,,system-test,python3 ./test.py -f 2-query/timetruncate.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/timetruncate.py ,,,system-test,python3 ./test.py -f 2-query/timetruncate.py -R -,,,system-test,python3 ./test.py -f 2-query/timezone.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/timezone.py ,,,system-test,python3 ./test.py -f 2-query/timezone.py -R -,,,system-test,python3 ./test.py -f 2-query/To_iso8601.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/To_iso8601.py ,,,system-test,python3 ./test.py -f 2-query/To_iso8601.py -R -,,,system-test,python3 ./test.py -f 2-query/To_unixtimestamp.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/To_unixtimestamp.py ,,,system-test,python3 ./test.py -f 2-query/To_unixtimestamp.py -R -,,,system-test,python3 ./test.py -f 2-query/Today.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/Today.py ,,,system-test,python3 ./test.py -f 2-query/Today.py -R -,,,system-test,python3 ./test.py -f 2-query/top.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/top.py ,,,system-test,python3 ./test.py -f 2-query/top.py -R ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -R ,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py ,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py -R -,,,system-test,python3 ./test.py -f 2-query/twa.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/twa.py ,,,system-test,python3 ./test.py -f 2-query/twa.py -R ,,,system-test,python3 ./test.py -f 2-query/union.py ,,,system-test,python3 ./test.py -f 2-query/union.py -R ,,,system-test,python3 ./test.py -f 2-query/unique.py ,,,system-test,python3 ./test.py -f 2-query/unique.py -R -,,,system-test,python3 ./test.py -f 2-query/upper.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/upper.py ,,,system-test,python3 ./test.py -f 2-query/upper.py -R -,,,system-test,python3 ./test.py -f 2-query/varchar.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/varchar.py ,,,system-test,python3 ./test.py -f 2-query/varchar.py -R -,,,system-test,python3 ./test.py -f 2-query/case_when.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/case_when.py ,,,system-test,python3 ./test.py -f 2-query/case_when.py -R ,,,system-test,python3 ./test.py -f 1-insert/update_data.py ,,,system-test,python3 ./test.py -f 1-insert/tb_100w_data_order.py ,,,system-test,python3 ./test.py -f 1-insert/delete_stable.py ,,,system-test,python3 ./test.py -f 1-insert/delete_childtable.py ,,,system-test,python3 ./test.py -f 1-insert/delete_normaltable.py -,,,system-test,python3 ./test.py -f 1-insert/keep_expired.py -,,,system-test,python3 ./test.py -f 2-query/join2.py +,,y,system-test,./pytest,sh python3 ./test.py -f 1-insert/keep_expired.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/join2.py ,,,system-test,python3 ./test.py -f 2-query/union1.py ,,,system-test,python3 ./test.py -f 2-query/concat2.py ,,,system-test,python3 ./test.py -f 2-query/json_tag.py @@ -627,9 +627,9 @@ ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py ,,,system-test,python3 ./test.py -f 2-query/stablity.py ,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -,,,system-test,python3 ./test.py -f 2-query/elapsed.py -,,,system-test,python3 ./test.py -f 2-query/csum.py -,,,system-test,python3 ./test.py -f 2-query/function_diff.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/elapsed.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/csum.py +,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/function_diff.py ,,,system-test,python3 ./test.py -f 2-query/queryQnode.py ,,,system-test,python3 ./test.py -f 6-cluster/5dnode1mnode.py ,,,system-test,python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 @@ -666,7 +666,7 @@ ,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py -N 4 -M 1 ,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py -N 4 -M 1 ,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 -,,,system-test,python3 ./test.py -f 7-tmq/create_wrong_topic.py +,,y,system-test,./pytest,sh python3 ./test.py -f 7-tmq/create_wrong_topic.py ,,,system-test,python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 ,,,system-test,python3 ./test.py -f 7-tmq/basic5.py ,,,system-test,python3 ./test.py -f 7-tmq/subscribeDb.py From 7109f136d64a5edfde627cca42258d4b4a75908d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Nov 2022 19:15:22 +0800 Subject: [PATCH 56/59] test: add asan python case --- tests/parallel_test/cases.task | 82 +++++++++++++++++----------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e849b7d443..69950bfec9 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -418,27 +418,27 @@ ,,,system-test,python3 ./test.py -f 0-others/fsync.py ,,,system-test,python3 ./test.py -f 0-others/compatibility.py ,,,system-test,python3 ./test.py -f 1-insert/alter_database.py -,,y,system-test,./pytest,sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py -,,y,system-test,./pytest,sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py ,,,system-test,python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py ,,,system-test,python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py ,,,system-test,python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py -,,y,system-test,./pytest,sh python3 ./test.py -f 1-insert/alter_stable.py -,,y,system-test,./pytest,sh python3 ./test.py -f 1-insert/alter_table.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_stable.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_table.py ,,,system-test,python3 ./test.py -f 1-insert/boundary.py ,,,system-test,python3 ./test.py -f 1-insert/insertWithMoreVgroup.py ,,,system-test,python3 ./test.py -f 1-insert/table_comment.py ,,,system-test,python3 ./test.py -f 1-insert/time_range_wise.py ,,,system-test,python3 ./test.py -f 1-insert/block_wise.py ,,,system-test,python3 ./test.py -f 1-insert/create_retentions.py -,,y,system-test,./pytest,sh python3 ./test.py -f 1-insert/table_param_ttl.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/table_param_ttl.py ,,,system-test,python3 ./test.py -f 1-insert/mutil_stage.py ,,,system-test,python3 ./test.py -f 1-insert/table_param_ttl.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 1-insert/update_data_muti_rows.py -,,y,system-test,./pytest,sh python3 ./test.py -f 1-insert/db_tb_name_check.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/update_data_muti_rows.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/db_tb_name_check.py ,,,system-test,python3 ./test.py -f 1-insert/database_pre_suf.py ,,,system-test,python3 ./test.py -f 1-insert/InsertFuturets.py -,,y,system-test,./pytest,sh python3 ./test.py -f 0-others/show.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py ,,,system-test,python3 ./test.py -f 2-query/abs.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/and_or_for_byte.py @@ -537,87 +537,87 @@ ,,,system-test,python3 ./test.py -f 2-query/ltrim.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py ,,,system-test,python3 ./test.py -f 2-query/mavg.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/max_partition.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py ,,,system-test,python3 ./test.py -f 2-query/max_partition.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/max.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py ,,,system-test,python3 ./test.py -f 2-query/max.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/min.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py ,,,system-test,python3 ./test.py -f 2-query/min.py -R ,,,system-test,python3 ./test.py -f 2-query/mode.py ,,,system-test,python3 ./test.py -f 2-query/mode.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/Now.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py ,,,system-test,python3 ./test.py -f 2-query/Now.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/percentile.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py ,,,system-test,python3 ./test.py -f 2-query/percentile.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/pow.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py ,,,system-test,python3 ./test.py -f 2-query/pow.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py ,,,system-test,python3 ./test.py -f 2-query/query_cols_tags_and_or.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/round.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py ,,,system-test,python3 ./test.py -f 2-query/round.py -R ,,,system-test,python3 ./test.py -f 2-query/rtrim.py ,,,system-test,python3 ./test.py -f 2-query/rtrim.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/sample.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py ,,,system-test,python3 ./test.py -f 2-query/sample.py -R ,,,system-test,python3 ./test.py -f 2-query/sin.py -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/sin.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -R ,,,system-test,python3 ./test.py -f 2-query/smaTest.py ,,,system-test,python3 ./test.py -f 2-query/smaTest.py -R ,,,system-test,python3 ./test.py -f 2-query/sml.py ,,,system-test,python3 ./test.py -f 2-query/sml.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/spread.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py ,,,system-test,python3 ./test.py -f 2-query/spread.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/sqrt.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py ,,,system-test,python3 ./test.py -f 2-query/sqrt.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/statecount.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py ,,,system-test,python3 ./test.py -f 2-query/statecount.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/stateduration.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py ,,,system-test,python3 ./test.py -f 2-query/stateduration.py -R ,,,system-test,python3 ./test.py -f 2-query/substr.py ,,,system-test,python3 ./test.py -f 2-query/substr.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/sum.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sum.py ,,,system-test,python3 ./test.py -f 2-query/sum.py -R ,,,system-test,python3 ./test.py -f 2-query/tail.py ,,,system-test,python3 ./test.py -f 2-query/tail.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/tan.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py ,,,system-test,python3 ./test.py -f 2-query/tan.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/Timediff.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py ,,,system-test,python3 ./test.py -f 2-query/Timediff.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/timetruncate.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py ,,,system-test,python3 ./test.py -f 2-query/timetruncate.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/timezone.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py ,,,system-test,python3 ./test.py -f 2-query/timezone.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/To_iso8601.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py ,,,system-test,python3 ./test.py -f 2-query/To_iso8601.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/To_unixtimestamp.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py ,,,system-test,python3 ./test.py -f 2-query/To_unixtimestamp.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/Today.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py ,,,system-test,python3 ./test.py -f 2-query/Today.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/top.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py ,,,system-test,python3 ./test.py -f 2-query/top.py -R ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -R ,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py ,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/twa.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py ,,,system-test,python3 ./test.py -f 2-query/twa.py -R ,,,system-test,python3 ./test.py -f 2-query/union.py ,,,system-test,python3 ./test.py -f 2-query/union.py -R ,,,system-test,python3 ./test.py -f 2-query/unique.py ,,,system-test,python3 ./test.py -f 2-query/unique.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/upper.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py ,,,system-test,python3 ./test.py -f 2-query/upper.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/varchar.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py ,,,system-test,python3 ./test.py -f 2-query/varchar.py -R -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/case_when.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py ,,,system-test,python3 ./test.py -f 2-query/case_when.py -R ,,,system-test,python3 ./test.py -f 1-insert/update_data.py ,,,system-test,python3 ./test.py -f 1-insert/tb_100w_data_order.py ,,,system-test,python3 ./test.py -f 1-insert/delete_stable.py ,,,system-test,python3 ./test.py -f 1-insert/delete_childtable.py ,,,system-test,python3 ./test.py -f 1-insert/delete_normaltable.py -,,y,system-test,./pytest,sh python3 ./test.py -f 1-insert/keep_expired.py -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/join2.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/keep_expired.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py ,,,system-test,python3 ./test.py -f 2-query/union1.py ,,,system-test,python3 ./test.py -f 2-query/concat2.py ,,,system-test,python3 ./test.py -f 2-query/json_tag.py @@ -627,9 +627,9 @@ ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py ,,,system-test,python3 ./test.py -f 2-query/stablity.py ,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/elapsed.py -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/csum.py -,,y,system-test,./pytest,sh python3 ./test.py -f 2-query/function_diff.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py ,,,system-test,python3 ./test.py -f 2-query/queryQnode.py ,,,system-test,python3 ./test.py -f 6-cluster/5dnode1mnode.py ,,,system-test,python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 @@ -666,7 +666,7 @@ ,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py -N 4 -M 1 ,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py -N 4 -M 1 ,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 -,,y,system-test,./pytest,sh python3 ./test.py -f 7-tmq/create_wrong_topic.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/create_wrong_topic.py ,,,system-test,python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 ,,,system-test,python3 ./test.py -f 7-tmq/basic5.py ,,,system-test,python3 ./test.py -f 7-tmq/subscribeDb.py From 392d27523d7200c0cb0593a8431f4d89deac2d03 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Nov 2022 20:17:01 +0800 Subject: [PATCH 57/59] test: add asan python case --- tests/parallel_test/cases.task | 362 ++++++++++++++++----------------- 1 file changed, 181 insertions(+), 181 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 69950bfec9..8a54fd0c35 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -416,9 +416,9 @@ ,,,system-test,python3 ./test.py -f 0-others/sysinfo.py ,,,system-test,python3 ./test.py -f 0-others/user_control.py ,,,system-test,python3 ./test.py -f 0-others/fsync.py -,,,system-test,python3 ./test.py -f 0-others/compatibility.py +,,,system-test,python3 ./test.py -f 0-others/compatibility.py ,,,system-test,python3 ./test.py -f 1-insert/alter_database.py -,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py +,,,system-test,python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py ,,,system-test,python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py ,,,system-test,python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py @@ -724,18 +724,18 @@ ,,,system-test,python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py ,,,system-test,python3 ./test.py -f 99-TDcase/TD-19201.py ,,,system-test,python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 -,,,system-test,python3 ./test.py -f 2-query/between.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/distinct.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/varchar.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/ltrim.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/rtrim.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/length.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/char_length.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/upper.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/lower.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/join.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/join2.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/cast.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/substr.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/union.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 2 @@ -743,45 +743,45 @@ ,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/check_tsdb.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/spread.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/hyperloglog.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/explain.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/leastsquares.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/timezone.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/Now.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/Today.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/max.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/min.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/mode.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/count.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/last.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/first.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/To_iso8601.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/timetruncate.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/diff.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/Timediff.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/json_tag.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/top.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/bottom.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/percentile.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/apercentile.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/abs.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/ceil.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/floor.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/round.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/log.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/pow.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/sqrt.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/sin.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/cos.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/tan.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/arcsin.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/arccos.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/arctan.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 @@ -792,44 +792,44 @@ ,,,system-test,python3 ./test.py -f 2-query/avg.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/mavg.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/unique.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/stateduration.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/tail.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_count.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_max.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_min.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/twa.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/irate.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/function_null.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/count_partition.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/max_partition.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/sml.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/case_when.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/between.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/distinct.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/varchar.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/ltrim.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/rtrim.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/length.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/char_length.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/upper.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/lower.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/join.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/join2.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/cast.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/substr.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/union.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 3 @@ -837,45 +837,45 @@ ,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/check_tsdb.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/spread.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/hyperloglog.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/explain.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/leastsquares.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/timezone.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/Now.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/Today.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/max.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/min.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/mode.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/count.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/last.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/first.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/To_iso8601.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/timetruncate.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/diff.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/Timediff.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/json_tag.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/top.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/bottom.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/percentile.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/apercentile.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/abs.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/ceil.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/floor.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/round.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/log.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/pow.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/sqrt.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/sin.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/cos.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/tan.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/arcsin.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/arccos.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/arctan.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 @@ -885,43 +885,43 @@ ,,,system-test,python3 ./test.py -f 2-query/avg.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/mavg.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/unique.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/stateduration.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/tail.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_count.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_max.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_min.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/twa.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/irate.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/function_null.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/count_partition.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/max_partition.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/sml.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/case_when.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/between.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/distinct.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/varchar.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/ltrim.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/rtrim.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/length.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/char_length.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/upper.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/lower.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/join.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/join2.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/substr.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/union.py -Q 4 @@ -930,45 +930,45 @@ ,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/check_tsdb.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/spread.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/hyperloglog.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/explain.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/leastsquares.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/timezone.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/Now.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/Today.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/max.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/min.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/mode.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/count.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/last.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/first.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/To_iso8601.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/timetruncate.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/diff.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/Timediff.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/json_tag.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/top.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/bottom.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/percentile.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/apercentile.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/abs.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/ceil.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/floor.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/round.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/log.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/pow.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/sqrt.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/sin.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/cos.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/tan.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/arcsin.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/arccos.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/arctan.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 @@ -978,33 +978,33 @@ ,,,system-test,python3 ./test.py -f 2-query/avg.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/mavg.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/unique.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/stateduration.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/tail.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_count.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_max.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_min.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/twa.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/irate.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/function_null.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/count_partition.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/max_partition.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/sml.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/case_when.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 4 #develop test ,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py From 0b2bf6fc72dc372ef05518758d0c8bf5d501c4ed Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 21 Nov 2022 20:45:07 +0800 Subject: [PATCH 58/59] fix: fix query message encode issue --- source/common/src/tmsg.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 5dfcb62fad..35900638dd 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4666,7 +4666,7 @@ int32_t tSerializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq) { if (tEncodeU32(&encoder, pReq->sqlLen) < 0) return -1; if (tEncodeCStrWithLen(&encoder, pReq->sql, pReq->sqlLen) < 0) return -1; if (tEncodeU32(&encoder, pReq->msgLen) < 0) return -1; - if (tEncodeCStrWithLen(&encoder, pReq->msg, pReq->msgLen) < 0) return -1; + if (tEncodeBinary(&encoder, (uint8_t*)pReq->msg, pReq->msgLen) < 0) return -1; tEndEncode(&encoder); @@ -4706,7 +4706,7 @@ int32_t tDeserializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq) if (tDecodeU32(&decoder, &pReq->sqlLen) < 0) return -1; if (tDecodeCStrAlloc(&decoder, &pReq->sql) < 0) return -1; if (tDecodeU32(&decoder, &pReq->msgLen) < 0) return -1; - if (tDecodeCStrAlloc(&decoder, &pReq->msg) < 0) return -1; + if (tDecodeBinaryAlloc(&decoder, (void**)&pReq->msg, NULL) < 0) return -1; tEndDecode(&decoder); From e5d004e950fc1dc0869f9586ffa91f353cff3c06 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Nov 2022 22:41:26 +0800 Subject: [PATCH 59/59] test: add asan python case --- tests/parallel_test/cases.task | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 8a54fd0c35..75ff515078 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -529,8 +529,8 @@ ,,,system-test,python3 ./test.py -f 2-query/leastsquares.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py ,,,system-test,python3 ./test.py -f 2-query/length.py -R -,,,system-test,python3 ./test.py -f 2-query/log.py -,,,system-test,python3 ./test.py -f 2-query/log.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -R ,,,system-test,python3 ./test.py -f 2-query/lower.py ,,,system-test,python3 ./test.py -f 2-query/lower.py -R ,,,system-test,python3 ./test.py -f 2-query/ltrim.py @@ -558,8 +558,8 @@ ,,,system-test,python3 ./test.py -f 2-query/rtrim.py ,,,system-test,python3 ./test.py -f 2-query/rtrim.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -,,,system-test,python3 ./test.py -f 2-query/sample.py -R -,,,system-test,python3 ./test.py -f 2-query/sin.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -R ,,,system-test,python3 ./test.py -f 2-query/smaTest.py ,,,system-test,python3 ./test.py -f 2-query/smaTest.py -R @@ -772,7 +772,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/log.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -Q 2 @@ -866,7 +866,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/log.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -Q 3 @@ -959,7 +959,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/log.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -Q 4