diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 457937835d..dfb92861d6 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -415,6 +415,7 @@ typedef struct SSelectStmt { int32_t returnRows; // EFuncReturnRows ETimeLineMode timeLineCurMode; ETimeLineMode timeLineResMode; + bool hasProcessByRowFunc; bool timeLineFromOrderBy; bool isEmptyResult; bool isSubquery; diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 2a03489547..73134c55ef 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -728,7 +728,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc setPseudoOutputColInfo(pResult, pCtx, pPseudoList); pResult->info.dataLoad = 1; - SArray* diffFunctionCtx = NULL; + SArray* processByRowFunctionCtx = NULL; if (pSrcBlock == NULL) { for (int32_t k = 0; k < numOfOutput; ++k) { @@ -861,14 +861,14 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc } numOfRows = pResInfo->numOfRes; if (fmIsProcessByRowFunc(pfCtx->functionId)) { - if (NULL == diffFunctionCtx) { - diffFunctionCtx = taosArrayInit(1, sizeof(SqlFunctionCtx*)); - if (!diffFunctionCtx) { + if (NULL == processByRowFunctionCtx) { + processByRowFunctionCtx = taosArrayInit(1, sizeof(SqlFunctionCtx*)); + if (!processByRowFunctionCtx) { code = terrno; goto _exit; } } - taosArrayPush(diffFunctionCtx, &pfCtx); + taosArrayPush(processByRowFunctionCtx, &pfCtx); } } else if (fmIsAggFunc(pfCtx->functionId)) { // selective value output should be set during corresponding function execution @@ -918,9 +918,9 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc } } - if (diffFunctionCtx && taosArrayGetSize(diffFunctionCtx) > 0){ - SqlFunctionCtx** pfCtx = taosArrayGet(diffFunctionCtx, 0); - code = (*pfCtx)->fpSet.processFuncByRow(diffFunctionCtx); + if (processByRowFunctionCtx && taosArrayGetSize(processByRowFunctionCtx) > 0){ + SqlFunctionCtx** pfCtx = taosArrayGet(processByRowFunctionCtx, 0); + code = (*pfCtx)->fpSet.processFuncByRow(processByRowFunctionCtx); if (code != TSDB_CODE_SUCCESS) { goto _exit; } @@ -930,9 +930,9 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc pResult->info.rows += numOfRows; } _exit: - if(diffFunctionCtx) { - taosArrayDestroy(diffFunctionCtx); - diffFunctionCtx = NULL; + if(processByRowFunctionCtx) { + taosArrayDestroy(processByRowFunctionCtx); + processByRowFunctionCtx = NULL; } return code; } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 66b3697a24..5058ebb8c3 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1977,7 +1977,7 @@ static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { if (IS_SIGNED_NUMERIC_TYPE(colType) || IS_TIMESTAMP_TYPE(colType) || TSDB_DATA_TYPE_BOOL == colType) { resType = TSDB_DATA_TYPE_BIGINT; } else if (IS_UNSIGNED_NUMERIC_TYPE(colType)) { - resType = TSDB_DATA_TYPE_UBIGINT; + resType = TSDB_DATA_TYPE_BIGINT; } else { resType = TSDB_DATA_TYPE_DOUBLE; } @@ -1986,7 +1986,11 @@ static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { } static EFuncReturnRows diffEstReturnRows(SFunctionNode* pFunc) { - return FUNC_RETURN_ROWS_N_MINUS_1; + if (1 == LIST_LENGTH(pFunc->pParameterList)) { + return FUNC_RETURN_ROWS_N_MINUS_1; + } + return 1 < ((SValueNode*)nodesListGetNode(pFunc->pParameterList, 1))->datum.i ? FUNC_RETURN_ROWS_INDEFINITE + : FUNC_RETURN_ROWS_N_MINUS_1; } static int32_t translateLength(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 0feff5657c..be6c555e56 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3152,37 +3152,49 @@ static int32_t doSetPrevVal(SDiffInfo* pDiffInfo, int32_t type, const char* pv, return TSDB_CODE_SUCCESS; } -static int32_t diffIsNegtive(SDiffInfo* pDiffInfo, int32_t type, const char* pv) { +static int32_t diffIsNegtive(SDiffInfo* pDiffInfo, int32_t type, const char* pv) { switch (type) { - case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UINT: { + int64_t v = *(uint32_t*)pv; + return v < pDiffInfo->prev.i64; + } case TSDB_DATA_TYPE_INT: { - int32_t v = *(int32_t*)pv; + int64_t v = *(int32_t*)pv; + return v < pDiffInfo->prev.i64; + } + case TSDB_DATA_TYPE_BOOL: { + int64_t v = *(bool*)pv; + return v < pDiffInfo->prev.i64; + } + case TSDB_DATA_TYPE_UTINYINT: { + int64_t v = *(int8_t*)pv; return v < pDiffInfo->prev.i64; } - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_TINYINT: { - int8_t v = *(int8_t*)pv; + int64_t v = *(uint8_t*)pv; + return v < pDiffInfo->prev.i64; + } + case TSDB_DATA_TYPE_USMALLINT: { + int64_t v = *(uint16_t*)pv; return v < pDiffInfo->prev.i64; } - case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_SMALLINT: { - int16_t v = *(int16_t*)pv; + int64_t v = *(int16_t*)pv; return v < pDiffInfo->prev.i64; } case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_BIGINT: { int64_t v = *(int64_t*)pv; - return v < pDiffInfo->prev.i64; + return v - pDiffInfo->prev.i64 < 0; } case TSDB_DATA_TYPE_FLOAT: { - float v = *(float*)pv; - return v < pDiffInfo->prev.d64; + float v = *(float*)pv; + return v < pDiffInfo->prev.d64; } case TSDB_DATA_TYPE_DOUBLE: { double v = *(double*)pv; - return v < pDiffInfo->prev.d64; + return v < pDiffInfo->prev.d64; } default: return false; @@ -3191,6 +3203,17 @@ static int32_t diffIsNegtive(SDiffInfo* pDiffInfo, int32_t type, const char* pv) return false; } +static void tryToSetInt64(SDiffInfo* pDiffInfo, SColumnInfoData* pOutput, int64_t v, int32_t pos) { + int64_t delta = v - pDiffInfo->prev.i64; // direct previous may be null + if (delta < 0 && ignoreNegative(pDiffInfo->ignoreOption)) { + colDataSetNull_f_s(pOutput, pos); + pOutput->hasNull = true; + } else { + colDataSetInt64(pOutput, pos, &delta); + } + pDiffInfo->prev.i64 = v; +} + static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, SColumnInfoData* pOutput, int32_t pos, int64_t ts) { if (!pDiffInfo->hasPrev) { @@ -3199,43 +3222,38 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, } pDiffInfo->prevTs = ts; switch (type) { - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_INT: { - int32_t v = *(int32_t*)pv; - int64_t delta = v - pDiffInfo->prev.i64; // direct previous may be null - if (delta < 0 && ignoreNegative(pDiffInfo->ignoreOption)) { - colDataSetNull_f_s(pOutput, pos); - pOutput->hasNull = true; - } else { - colDataSetInt64(pOutput, pos, &delta); - } - pDiffInfo->prev.i64 = v; - + case TSDB_DATA_TYPE_UINT: { + int64_t v = *(uint32_t*)pv; + tryToSetInt64(pDiffInfo, pOutput, v, pos); + break; + } + case TSDB_DATA_TYPE_INT: { + int64_t v = *(int32_t*)pv; + tryToSetInt64(pDiffInfo, pOutput, v, pos); + break; + } + case TSDB_DATA_TYPE_BOOL: { + int64_t v = *(bool*)pv; + tryToSetInt64(pDiffInfo, pOutput, v, pos); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + int64_t v = *(int8_t*)pv; + tryToSetInt64(pDiffInfo, pOutput, v, pos); break; } - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_TINYINT: { - int8_t v = *(int8_t*)pv; - int64_t delta = v - pDiffInfo->prev.i64; // direct previous may be null - if (delta < 0 && ignoreNegative(pDiffInfo->ignoreOption)) { - colDataSetNull_f_s(pOutput, pos); - } else { - colDataSetInt64(pOutput, pos, &delta); - } - pDiffInfo->prev.i64 = v; + int64_t v = *(uint8_t*)pv; + tryToSetInt64(pDiffInfo, pOutput, v, pos); break; } case TSDB_DATA_TYPE_USMALLINT: + int64_t v = *(uint16_t*)pv; + tryToSetInt64(pDiffInfo, pOutput, v, pos); + break; case TSDB_DATA_TYPE_SMALLINT: { - int16_t v = *(int16_t*)pv; - int64_t delta = v - pDiffInfo->prev.i64; // direct previous may be null - if (delta < 0 && ignoreNegative(pDiffInfo->ignoreOption)) { - colDataSetNull_f_s(pOutput, pos); - } else { - colDataSetInt64(pOutput, pos, &delta); - } - pDiffInfo->prev.i64 = v; + int64_t v = *(int16_t*)pv; + tryToSetInt64(pDiffInfo, pOutput, v, pos); break; } case TSDB_DATA_TYPE_TIMESTAMP: @@ -3252,9 +3270,10 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, break; } case TSDB_DATA_TYPE_FLOAT: { - float v = *(float*)pv; + double v = *(float*)pv; double delta = v - pDiffInfo->prev.d64; // direct previous may be null - if ((delta < 0 && ignoreNegative(pDiffInfo->ignoreOption)) || isinf(delta) || isnan(delta)) { // check for overflow + if ((delta < 0 && ignoreNegative(pDiffInfo->ignoreOption)) || isinf(delta) || + isnan(delta)) { // check for overflow colDataSetNull_f_s(pOutput, pos); } else { colDataSetDouble(pOutput, pos, &delta); @@ -3265,7 +3284,8 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, case TSDB_DATA_TYPE_DOUBLE: { double v = *(double*)pv; double delta = v - pDiffInfo->prev.d64; // direct previous may be null - if ((delta < 0 && ignoreNegative(pDiffInfo->ignoreOption)) || isinf(delta) || isnan(delta)) { // check for overflow + if ((delta < 0 && ignoreNegative(pDiffInfo->ignoreOption)) || isinf(delta) || + isnan(delta)) { // check for overflow colDataSetNull_f_s(pOutput, pos); } else { colDataSetDouble(pOutput, pos, &delta); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b741db3ae6..35c42e2ffb 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2194,9 +2194,12 @@ static int32_t translateIndefiniteRowsFunc(STranslateContext* pCxt, SFunctionNod return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; - if (pSelect->hasAggFuncs || pSelect->hasMultiRowsFunc || - (pSelect->hasIndefiniteRowsFunc && - (FUNC_RETURN_ROWS_INDEFINITE == pSelect->returnRows || pSelect->returnRows != fmGetFuncReturnRows(pFunc)))) { + if (pSelect->hasAggFuncs || pSelect->hasMultiRowsFunc) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); + } + if (pSelect->hasIndefiniteRowsFunc && + (FUNC_RETURN_ROWS_INDEFINITE == pSelect->returnRows || pSelect->returnRows != fmGetFuncReturnRows(pFunc)) && + (!pSelect->hasProcessByRowFunc || !fmIsProcessByRowFunc(pFunc->funcId))) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } if (NULL != pSelect->pWindow || NULL != pSelect->pGroupByList) { @@ -2462,6 +2465,9 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) { } else if (fmIsInterpFunc(pFunc->funcId)) { pSelect->returnRows = fmGetFuncReturnRows(pFunc); } + if (fmIsProcessByRowFunc(pFunc->funcId)) { + pSelect->hasProcessByRowFunc = true; + } pSelect->hasMultiRowsFunc = pSelect->hasMultiRowsFunc ? true : fmIsMultiRowsFunc(pFunc->funcId); if (fmIsSelectFunc(pFunc->funcId)) { diff --git a/tests/system-test/2-query/diff.py b/tests/system-test/2-query/diff.py index ee71db6d34..4400e8c26d 100644 --- a/tests/system-test/2-query/diff.py +++ b/tests/system-test/2-query/diff.py @@ -154,6 +154,15 @@ class TDTestCase: tdSql.execute(f"insert into {dbname}.stb30749_2 values(%d, 4, 3)" % (ts1 + 0)) tdSql.execute(f"insert into {dbname}.stb30749_2 values(%d, null, null)" % (ts1 + 10)) + tdSql.query(f"select ts, diff(col1), diff(col2, 1) from {dbname}.stb30749") + tdSql.checkRows(8) + tdSql.checkData(2, 0, '2023-09-17 09:00:00.002') + tdSql.checkData(3, 0, '2023-09-17 09:00:00.003') + tdSql.checkData(2, 1, -1) + tdSql.checkData(2, 2, None) + tdSql.checkData(3, 1, 1) + tdSql.checkData(3, 2, 2) + tdSql.query(f"select ts, diff(col1), diff(col2) from {dbname}.stb30749") tdSql.checkRows(8) tdSql.checkData(2, 0, '2023-09-17 09:00:00.002') @@ -163,6 +172,42 @@ class TDTestCase: tdSql.checkData(3, 1, 1) tdSql.checkData(3, 2, 2) + tdSql.query(f"select ts, diff(col1), diff(col2, 3) from {dbname}.stb30749") + tdSql.checkRows(8) + tdSql.checkData(2, 0, '2023-09-17 09:00:00.002') + tdSql.checkData(3, 0, '2023-09-17 09:00:00.003') + tdSql.checkData(2, 1, -1) + tdSql.checkData(2, 2, None) + tdSql.checkData(3, 1, 1) + tdSql.checkData(3, 2, 2) + + tdSql.query(f"select ts, diff(col1, 1), diff(col2, 2) from {dbname}.stb30749") + tdSql.checkRows(8) + tdSql.checkData(2, 0, '2023-09-17 09:00:00.002') + tdSql.checkData(3, 0, '2023-09-17 09:00:00.003') + tdSql.checkData(2, 1, None) + tdSql.checkData(2, 2, None) + tdSql.checkData(3, 1, 1) + tdSql.checkData(3, 2, 2) + + tdSql.query(f"select ts, diff(col1, 1), diff(col2, 3) from {dbname}.stb30749") + tdSql.checkRows(8) + tdSql.checkData(2, 0, '2023-09-17 09:00:00.002') + tdSql.checkData(3, 0, '2023-09-17 09:00:00.003') + tdSql.checkData(2, 1, None) + tdSql.checkData(2, 2, None) + tdSql.checkData(3, 1, 1) + tdSql.checkData(3, 2, 2) + + tdSql.query(f"select ts, diff(col1, 2), diff(col2, 2) from {dbname}.stb30749") + tdSql.checkRows(6) + tdSql.checkData(2, 0, '2023-09-17 09:00:00.002') + tdSql.checkData(3, 0, '2023-09-17 09:00:00.003') + tdSql.checkData(2, 1, -1) + tdSql.checkData(2, 2, None) + tdSql.checkData(3, 1, 1) + tdSql.checkData(3, 2, 2) + tdSql.query(f"select ts, diff(col1, 3), diff(col2, 2) from {dbname}.stb30749") tdSql.checkRows(5) tdSql.checkData(2, 0, '2023-09-17 09:00:00.003') @@ -172,6 +217,24 @@ class TDTestCase: tdSql.checkData(3, 1, None) tdSql.checkData(3, 2, -2) + tdSql.query(f"select ts, diff(col1, 2), diff(col2, 3) from {dbname}.stb30749") + tdSql.checkRows(5) + tdSql.checkData(2, 0, '2023-09-17 09:00:00.003') + tdSql.checkData(3, 0, '2023-09-17 09:00:00.004') + tdSql.checkData(2, 1, 1) + tdSql.checkData(2, 2, 2) + tdSql.checkData(3, 1, -3) + tdSql.checkData(3, 2, None) + + tdSql.query(f"select ts, diff(col1, 3), diff(col2, 3) from {dbname}.stb30749") + tdSql.checkRows(3) + tdSql.checkData(1, 0, '2023-09-17 09:00:00.003') + tdSql.checkData(2, 0, '2023-09-17 09:00:00.005') + tdSql.checkData(1, 1, 1) + tdSql.checkData(1, 2, 2) + tdSql.checkData(2, 1, 1) + tdSql.checkData(2, 2, None) + tdSql.query(f"select ts, diff(col1), diff(col2) from {dbname}.stb30749 partition by tbname") tdSql.checkRows(7) tdSql.checkData(0, 0, '2023-09-17 09:00:00.002') @@ -210,6 +273,59 @@ class TDTestCase: tdSql.execute(f"insert into {dbname}.stb5_2 values(%d, 3, 3)" % (ts1 + 2)) tdSql.query(f"select ts, diff(col1, 3), diff(col2, 2) from {dbname}.stb5") tdSql.checkRows(2) + + + def typeOverflowTest(self): + dbname = "db" + + ts1 = 1694912400000 + tdSql.execute(f'''create table {dbname}.stb6(ts timestamp, c1 int, c2 smallint, c3 int unsigned, c4 BIGINT, c5 BIGINT unsigned) tags(loc nchar(20))''') + tdSql.execute(f"create table {dbname}.stb6_1 using {dbname}.stb6 tags('shanghai')") + + tdSql.execute(f"insert into {dbname}.stb6_1 values(%d, -2147483648, -32768, 0, 9223372036854775806, 9223372036854775806)" % (ts1 + 1)) + tdSql.execute(f"insert into {dbname}.stb6_1 values(%d, 2147483647, 32767, 4294967295, 0, 0)" % (ts1 + 2)) + tdSql.execute(f"insert into {dbname}.stb6_1 values(%d, -10, -10, 0, -9223372036854775806, 9223372036854775806)" % (ts1 + 3)) + + tdSql.query(f"select ts, diff(c1), diff(c2), diff(c3), diff(c4), diff(c5) from {dbname}.stb6_1") + tdSql.checkRows(2) + tdSql.checkData(0, 0, '2023-09-17 09:00:00.002') + tdSql.checkData(0, 1, 4294967295) + tdSql.checkData(0, 2, 65535) + tdSql.checkData(0, 3, 4294967295) + tdSql.checkData(0, 4, -9223372036854775806) + tdSql.checkData(0, 5, -9223372036854775806) + tdSql.checkData(1, 0, '2023-09-17 09:00:00.003') + tdSql.checkData(1, 1, -2147483657) + tdSql.checkData(1, 2, -32777) + tdSql.checkData(1, 3, -4294967295) + tdSql.checkData(1, 4, -9223372036854775806) + tdSql.checkData(1, 5, 9223372036854775806) + + tdSql.query(f"select ts, diff(c1, 1), diff(c2) from {dbname}.stb6_1") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 4294967295) + tdSql.checkData(0, 2, 65535) + tdSql.checkData(1, 1, None) + tdSql.checkData(1, 2, -32777) + + tdSql.query(f"select ts, diff(c1, 1), diff(c2, 1) from {dbname}.stb6_1") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 4294967295) + tdSql.checkData(0, 2, 65535) + tdSql.checkData(1, 1, None) + tdSql.checkData(1, 2, None) + + tdSql.query(f"select ts, diff(c1, 2), diff(c2, 3) from {dbname}.stb6_1") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 4294967295) + tdSql.checkData(0, 2, 65535) + tdSql.checkData(1, 1, -2147483657) + tdSql.checkData(1, 2, None) + + tdSql.query(f"select ts, diff(c1, 3), diff(c2, 3) from {dbname}.stb6_1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 4294967295) + tdSql.checkData(0, 2, 65535) def run(self): tdSql.prepare() @@ -220,6 +336,7 @@ class TDTestCase: self.ignoreTest() self.withPkTest() + self.typeOverflowTest() tdSql.execute( f"create table {dbname}.ntb(ts timestamp,c1 int,c2 double,c3 float)")