From b077af04a6e1c4006005a045fdc8a6423f6859d4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 16 Aug 2021 13:26:53 +0800 Subject: [PATCH 01/24] [6046] fix ts always in first output index using derivative function --- src/client/src/tscSQLParser.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 17b693faf2..781b1be76f 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2594,13 +2594,12 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // set the first column ts for diff query if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { - colIndex += 1; SColumnIndex indexTS = {.tableIndex = index.tableIndex, .columnIndex = 0}; SExprInfo* pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, getNewResColId(pCmd), TSDB_KEYSIZE, false); SColumnList ids = createColumnList(1, 0, 0); - insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].name, pExpr); + insertResultField(pQueryInfo, colIndex, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].name, pExpr); } SExprInfo* pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), intermediateResSize, false); From 059576a3a50baabbc68971bf6ba87cd98c00ea54 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 16 Aug 2021 13:28:07 +0800 Subject: [PATCH 02/24] [6046] fix ts always in first output index using derivative function --- src/query/src/qExecutor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 9000bcdf77..af7408880a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3622,7 +3622,7 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf // re-estabilish output buffer pointer. int32_t functionId = pBInfo->pCtx[i].functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { - pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[i-1].pOutput; + if(i > 0) pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[i-1].pOutput; } } } From 4b9c461c542e6e0e71beaf9d4b677e46ef89a822 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 16 Aug 2021 15:16:44 +0800 Subject: [PATCH 03/24] [6046] fix ts is null --- src/client/src/tscSQLParser.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 781b1be76f..bb2b8f3c52 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -7052,6 +7052,10 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, char* msg) { continue; } + if(functionId == TSDB_FUNC_DERIVATIVE){ // to avoid ts function id was modufied below + tagTsColExists = false; + } + if (functionId < 0) { SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { From 7c36f3de4a6014e00671758574948ba00f780d95 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 16 Aug 2021 17:15:56 +0800 Subject: [PATCH 04/24] [TD-6046] fix ts,derivative() output error for top/bottom/diff/derivative --- src/client/src/tscSQLParser.c | 5 +++-- src/query/inc/qAggMain.h | 1 + src/query/src/qAggMain.c | 14 ++++++++++++++ src/query/src/qExecutor.c | 10 ++++++++-- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index bb2b8f3c52..b9172f5c8c 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2872,7 +2872,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col const int32_t TS_COLUMN_INDEX = PRIMARYKEY_TIMESTAMP_COL_INDEX; SColumnList ids = createColumnList(1, index.tableIndex, TS_COLUMN_INDEX); - insertResultField(pQueryInfo, TS_COLUMN_INDEX, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, + insertResultField(pQueryInfo, colIndex, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS].name, pExpr); colIndex += 1; // the first column is ts @@ -7052,7 +7052,8 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, char* msg) { continue; } - if(functionId == TSDB_FUNC_DERIVATIVE){ // to avoid ts function id was modufied below + if(functionId == TSDB_FUNC_DERIVATIVE || functionId == TSDB_FUNC_DIFF || + functionId == TSDB_FUNC_TOP ||functionId == TSDB_FUNC_BOTTOM ){ // to avoid ts function id was modufied below tagTsColExists = false; } diff --git a/src/query/inc/qAggMain.h b/src/query/inc/qAggMain.h index d4116fbfb2..4b2de758ac 100644 --- a/src/query/inc/qAggMain.h +++ b/src/query/inc/qAggMain.h @@ -186,6 +186,7 @@ typedef struct SQLFunctionCtx { tVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param int64_t *ptsList; // corresponding timestamp array list void *ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ + void *ptsOriOutputBuf; SQLPreAggVal preAggVals; tVariant tag; diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index c19628eb37..89e91cb856 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -2789,6 +2789,7 @@ static void deriv_function(SQLFunctionCtx *pCtx) { int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; TSKEY *pTimestamp = pCtx->ptsOutputBuf; + TSKEY *pTimestampOri = pCtx->ptsOriOutputBuf; TSKEY *tsList = GET_TS_LIST(pCtx); double *pOutput = (double *)pCtx->pOutput; @@ -2808,6 +2809,7 @@ static void deriv_function(SQLFunctionCtx *pCtx) { if (pDerivInfo->ignoreNegative && *pOutput < 0) { } else { *pTimestamp = tsList[i]; + if (pTimestampOri) {*pTimestampOri = tsList[i]; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; notNullElems++; @@ -2835,6 +2837,7 @@ static void deriv_function(SQLFunctionCtx *pCtx) { if (pDerivInfo->ignoreNegative && *pOutput < 0) { } else { *pTimestamp = tsList[i]; + if (pTimestampOri) {*pTimestampOri = tsList[i]; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; notNullElems++; @@ -2861,6 +2864,7 @@ static void deriv_function(SQLFunctionCtx *pCtx) { if (pDerivInfo->ignoreNegative && *pOutput < 0) { } else { *pTimestamp = tsList[i]; + if (pTimestampOri) {*pTimestampOri = tsList[i]; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; notNullElems++; @@ -2888,6 +2892,7 @@ static void deriv_function(SQLFunctionCtx *pCtx) { if (pDerivInfo->ignoreNegative && *pOutput < 0) { } else { *pTimestamp = tsList[i]; + if (pTimestampOri) {*pTimestampOri = tsList[i]; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; notNullElems++; @@ -2914,6 +2919,7 @@ static void deriv_function(SQLFunctionCtx *pCtx) { if (pDerivInfo->ignoreNegative && *pOutput < 0) { } else { *pTimestamp = tsList[i]; + if (pTimestampOri) {*pTimestampOri = tsList[i]; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; notNullElems++; @@ -2940,6 +2946,7 @@ static void deriv_function(SQLFunctionCtx *pCtx) { if (pDerivInfo->ignoreNegative && *pOutput < 0) { } else { *pTimestamp = tsList[i]; + if (pTimestampOri) {*pTimestampOri = tsList[i]; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; @@ -2982,6 +2989,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; TSKEY* pTimestamp = pCtx->ptsOutputBuf; + TSKEY* pTimestampOri = pCtx->ptsOriOutputBuf; TSKEY* tsList = GET_TS_LIST(pCtx); switch (pCtx->inputType) { @@ -2997,6 +3005,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet *pOutput = (int32_t)(pData[i] - pCtx->param[1].i64); // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; + if (pTimestampOri) {*pTimestampOri = *pTimestamp; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; } @@ -3019,6 +3028,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet *pOutput = pData[i] - pCtx->param[1].i64; // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; + if (pTimestampOri) {*pTimestampOri = *pTimestamp; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; } @@ -3041,6 +3051,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet SET_DOUBLE_VAL(pOutput, pData[i] - pCtx->param[1].dKey); // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; + if (pTimestampOri) {*pTimestampOri = *pTimestamp; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; } @@ -3063,6 +3074,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet *pOutput = (float)(pData[i] - pCtx->param[1].dKey); // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; + if (pTimestampOri) {*pTimestampOri = *pTimestamp; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; } @@ -3085,6 +3097,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet *pOutput = (int16_t)(pData[i] - pCtx->param[1].i64); // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; + if (pTimestampOri) {*pTimestampOri = *pTimestamp; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; } @@ -3108,6 +3121,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet *pOutput = (int8_t)(pData[i] - pCtx->param[1].i64); // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; + if (pTimestampOri) {*pTimestampOri = *pTimestamp; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index af7408880a..4ce9a1d849 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3615,14 +3615,20 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf } } + char *tsbuf = NULL; for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); pBInfo->pCtx[i].pOutput = pColInfo->pData + pColInfo->info.bytes * pDataBlock->info.rows; // re-estabilish output buffer pointer. int32_t functionId = pBInfo->pCtx[i].functionId; - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { - if(i > 0) pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[i-1].pOutput; + if (functionId == TSDB_FUNC_PRJ && pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP){ + tsbuf = pBInfo->pCtx[i].pOutput; + } + else if ((i > 0) && + (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE)) { + pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[i-1].pOutput; + pBInfo->pCtx[i].ptsOriOutputBuf = tsbuf; } } } From fbd648b5f2a4b91527f1ee351b1817dc6a329fe1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 16 Aug 2021 17:57:08 +0800 Subject: [PATCH 05/24] [TD-6046] fix ts,derivative() output error for top/bottom/diff/derivative --- src/query/src/qExecutor.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 4ce9a1d849..caa6620b73 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3616,19 +3616,32 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf } char *tsbuf = NULL; + int16_t tsFuncIndex = -1; + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); + + // find the ts output data pointer + int32_t functionId = pBInfo->pCtx[i].functionId; + if (functionId == TSDB_FUNC_PRJ && pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) { + tsbuf = pColInfo->pData + pColInfo->info.bytes * pDataBlock->info.rows; + tsFuncIndex = i; + break; + } + } + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); pBInfo->pCtx[i].pOutput = pColInfo->pData + pColInfo->info.bytes * pDataBlock->info.rows; // re-estabilish output buffer pointer. int32_t functionId = pBInfo->pCtx[i].functionId; - if (functionId == TSDB_FUNC_PRJ && pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP){ - tsbuf = pBInfo->pCtx[i].pOutput; - } - else if ((i > 0) && + if ((i > 0) && (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE)) { pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[i-1].pOutput; pBInfo->pCtx[i].ptsOriOutputBuf = tsbuf; + if(tsFuncIndex != -1) { + pBInfo->pCtx[tsFuncIndex].functionId = TSDB_FUNC_TS_DUMMY; // to avoid query data + } } } } From 203f5c7078645e990b7f2d015dcf84543be3959f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 16 Aug 2021 18:14:21 +0800 Subject: [PATCH 06/24] [TD-6046] fix ts,derivative() output error for top/bottom/diff/derivative --- src/query/src/qExecutor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index caa6620b73..dfaf60d1d7 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2024,7 +2024,7 @@ static SQLFunctionCtx* createSQLFunctionCtx(SQueryRuntimeEnv* pRuntimeEnv, SExpr if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { int32_t f = pExpr[0].base.functionId; - assert(f == TSDB_FUNC_TS || f == TSDB_FUNC_TS_DUMMY); + assert(f == TSDB_FUNC_TS || f == TSDB_FUNC_TS_DUMMY || f == TSDB_FUNC_PRJ); pCtx->param[2].i64 = pQueryAttr->order.order; pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; From da702897976e2d7a0ee84fe369a97c5eac1b7cf2 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 16 Aug 2021 18:18:39 +0800 Subject: [PATCH 07/24] [TD-6046] fix ts,derivative() output error for top/bottom/diff/derivative --- src/client/src/tscSQLParser.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index b9172f5c8c..62f1843ee5 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -7052,8 +7052,7 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, char* msg) { continue; } - if(functionId == TSDB_FUNC_DERIVATIVE || functionId == TSDB_FUNC_DIFF || - functionId == TSDB_FUNC_TOP ||functionId == TSDB_FUNC_BOTTOM ){ // to avoid ts function id was modufied below + if(functionId == TSDB_FUNC_DERIVATIVE || functionId == TSDB_FUNC_DIFF){ // to avoid ts function id was modufied below tagTsColExists = false; } From bdc6409e7740e6a711688fb25d7530fd43509e08 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Aug 2021 00:47:42 +0800 Subject: [PATCH 08/24] [TD-6046] fix ts derivative error --- src/client/src/tscSQLParser.c | 6 +--- src/query/inc/qAggMain.h | 1 - src/query/inc/qExecutor.h | 1 + src/query/src/qAggMain.c | 14 ---------- src/query/src/qExecutor.c | 52 +++++++++++++++++++++-------------- 5 files changed, 33 insertions(+), 41 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 62f1843ee5..781b1be76f 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2872,7 +2872,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col const int32_t TS_COLUMN_INDEX = PRIMARYKEY_TIMESTAMP_COL_INDEX; SColumnList ids = createColumnList(1, index.tableIndex, TS_COLUMN_INDEX); - insertResultField(pQueryInfo, colIndex, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, + insertResultField(pQueryInfo, TS_COLUMN_INDEX, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS].name, pExpr); colIndex += 1; // the first column is ts @@ -7052,10 +7052,6 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, char* msg) { continue; } - if(functionId == TSDB_FUNC_DERIVATIVE || functionId == TSDB_FUNC_DIFF){ // to avoid ts function id was modufied below - tagTsColExists = false; - } - if (functionId < 0) { SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { diff --git a/src/query/inc/qAggMain.h b/src/query/inc/qAggMain.h index 4b2de758ac..d4116fbfb2 100644 --- a/src/query/inc/qAggMain.h +++ b/src/query/inc/qAggMain.h @@ -186,7 +186,6 @@ typedef struct SQLFunctionCtx { tVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param int64_t *ptsList; // corresponding timestamp array list void *ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ - void *ptsOriOutputBuf; SQLPreAggVal preAggVals; tVariant tag; diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 56fab57e26..5b810e217e 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -595,6 +595,7 @@ int32_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx* pCtx, int3 void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows); void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity); +void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput); void freeParam(SQueryParam *param); int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param); diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 89e91cb856..c19628eb37 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -2789,7 +2789,6 @@ static void deriv_function(SQLFunctionCtx *pCtx) { int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; TSKEY *pTimestamp = pCtx->ptsOutputBuf; - TSKEY *pTimestampOri = pCtx->ptsOriOutputBuf; TSKEY *tsList = GET_TS_LIST(pCtx); double *pOutput = (double *)pCtx->pOutput; @@ -2809,7 +2808,6 @@ static void deriv_function(SQLFunctionCtx *pCtx) { if (pDerivInfo->ignoreNegative && *pOutput < 0) { } else { *pTimestamp = tsList[i]; - if (pTimestampOri) {*pTimestampOri = tsList[i]; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; notNullElems++; @@ -2837,7 +2835,6 @@ static void deriv_function(SQLFunctionCtx *pCtx) { if (pDerivInfo->ignoreNegative && *pOutput < 0) { } else { *pTimestamp = tsList[i]; - if (pTimestampOri) {*pTimestampOri = tsList[i]; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; notNullElems++; @@ -2864,7 +2861,6 @@ static void deriv_function(SQLFunctionCtx *pCtx) { if (pDerivInfo->ignoreNegative && *pOutput < 0) { } else { *pTimestamp = tsList[i]; - if (pTimestampOri) {*pTimestampOri = tsList[i]; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; notNullElems++; @@ -2892,7 +2888,6 @@ static void deriv_function(SQLFunctionCtx *pCtx) { if (pDerivInfo->ignoreNegative && *pOutput < 0) { } else { *pTimestamp = tsList[i]; - if (pTimestampOri) {*pTimestampOri = tsList[i]; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; notNullElems++; @@ -2919,7 +2914,6 @@ static void deriv_function(SQLFunctionCtx *pCtx) { if (pDerivInfo->ignoreNegative && *pOutput < 0) { } else { *pTimestamp = tsList[i]; - if (pTimestampOri) {*pTimestampOri = tsList[i]; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; notNullElems++; @@ -2946,7 +2940,6 @@ static void deriv_function(SQLFunctionCtx *pCtx) { if (pDerivInfo->ignoreNegative && *pOutput < 0) { } else { *pTimestamp = tsList[i]; - if (pTimestampOri) {*pTimestampOri = tsList[i]; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; @@ -2989,7 +2982,6 @@ static void diff_function(SQLFunctionCtx *pCtx) { int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; TSKEY* pTimestamp = pCtx->ptsOutputBuf; - TSKEY* pTimestampOri = pCtx->ptsOriOutputBuf; TSKEY* tsList = GET_TS_LIST(pCtx); switch (pCtx->inputType) { @@ -3005,7 +2997,6 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet *pOutput = (int32_t)(pData[i] - pCtx->param[1].i64); // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; - if (pTimestampOri) {*pTimestampOri = *pTimestamp; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; } @@ -3028,7 +3019,6 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet *pOutput = pData[i] - pCtx->param[1].i64; // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; - if (pTimestampOri) {*pTimestampOri = *pTimestamp; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; } @@ -3051,7 +3041,6 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet SET_DOUBLE_VAL(pOutput, pData[i] - pCtx->param[1].dKey); // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; - if (pTimestampOri) {*pTimestampOri = *pTimestamp; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; } @@ -3074,7 +3063,6 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet *pOutput = (float)(pData[i] - pCtx->param[1].dKey); // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; - if (pTimestampOri) {*pTimestampOri = *pTimestamp; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; } @@ -3097,7 +3085,6 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet *pOutput = (int16_t)(pData[i] - pCtx->param[1].i64); // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; - if (pTimestampOri) {*pTimestampOri = *pTimestamp; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; } @@ -3121,7 +3108,6 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet *pOutput = (int8_t)(pData[i] - pCtx->param[1].i64); // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; - if (pTimestampOri) {*pTimestampOri = *pTimestamp; pTimestampOri += 1;} pOutput += 1; pTimestamp += 1; } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index dfaf60d1d7..3e1fac8fd3 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2024,7 +2024,7 @@ static SQLFunctionCtx* createSQLFunctionCtx(SQueryRuntimeEnv* pRuntimeEnv, SExpr if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { int32_t f = pExpr[0].base.functionId; - assert(f == TSDB_FUNC_TS || f == TSDB_FUNC_TS_DUMMY || f == TSDB_FUNC_PRJ); + assert(f == TSDB_FUNC_TS || f == TSDB_FUNC_TS_DUMMY); pCtx->param[2].i64 = pQueryAttr->order.order; pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; @@ -3615,19 +3615,6 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf } } - char *tsbuf = NULL; - int16_t tsFuncIndex = -1; - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); - - // find the ts output data pointer - int32_t functionId = pBInfo->pCtx[i].functionId; - if (functionId == TSDB_FUNC_PRJ && pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) { - tsbuf = pColInfo->pData + pColInfo->info.bytes * pDataBlock->info.rows; - tsFuncIndex = i; - break; - } - } for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); @@ -3635,13 +3622,8 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf // re-estabilish output buffer pointer. int32_t functionId = pBInfo->pCtx[i].functionId; - if ((i > 0) && - (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE)) { - pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[i-1].pOutput; - pBInfo->pCtx[i].ptsOriOutputBuf = tsbuf; - if(tsFuncIndex != -1) { - pBInfo->pCtx[tsFuncIndex].functionId = TSDB_FUNC_TS_DUMMY; // to avoid query data - } + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE){ + if (i > 0) pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[i-1].pOutput; } } } @@ -3659,7 +3641,34 @@ void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity) { } } +void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput) { + bool needCopyTs = false; + int32_t tsNum = 0; + for (int32_t i = 0; i < numOfOutput; i++) { + int32_t functionId = pCtx[i].functionId; + if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { + needCopyTs = true; + }else if(functionId == TSDB_FUNC_TS_COMP) { + tsNum++; + } + } + char *src = NULL; + for (int32_t col = 0; col < numOfOutput; ++col) { + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); + if (strlen(pColRes->pData) != 0) { + src = pColRes->pData; // find ts data + } + } + if (!needCopyTs) return; + if (tsNum < 2) return; + if (src == NULL) return; + + for (int32_t col = 0; col < numOfOutput; ++col) { + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); + memcpy(pColRes->pData, src, pColRes->info.bytes * pRes->info.rows); + } +} void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) { for (int32_t j = 0; j < size; ++j) { @@ -5635,6 +5644,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { if (pRes->info.rows >= 1000/*pRuntimeEnv->resultInfo.threshold*/) { break; } + copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); } clearNumOfRes(pInfo->pCtx, pOperator->numOfOutput); From 43a6843dba067997a2db892f60e4abbd1f9a36e8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Aug 2021 00:57:24 +0800 Subject: [PATCH 09/24] [TD-6046] fix ts derivative error --- src/query/src/qExecutor.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 3e1fac8fd3..6fd9f70fe6 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3644,22 +3644,20 @@ void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity) { void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput) { bool needCopyTs = false; int32_t tsNum = 0; + char *src = NULL; for (int32_t i = 0; i < numOfOutput; i++) { int32_t functionId = pCtx[i].functionId; if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { needCopyTs = true; - }else if(functionId == TSDB_FUNC_TS_COMP) { + }else if(functionId == TSDB_FUNC_TS_DUMMY) { + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i); + if (strlen(pColRes->pData) != 0) { + src = pColRes->pData; // find ts data + } tsNum++; } } - - char *src = NULL; - for (int32_t col = 0; col < numOfOutput; ++col) { - SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); - if (strlen(pColRes->pData) != 0) { - src = pColRes->pData; // find ts data - } - } + if (!needCopyTs) return; if (tsNum < 2) return; if (src == NULL) return; From dff67958b1ba5c7fa9d3aed1185a7ecbc2ed4131 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Aug 2021 01:02:53 +0800 Subject: [PATCH 10/24] [TD-6046] fix ts derivative error --- src/query/src/qExecutor.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6fd9f70fe6..7ccaeb7ac9 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3657,14 +3657,17 @@ void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput) tsNum++; } } - + if (!needCopyTs) return; if (tsNum < 2) return; if (src == NULL) return; - for (int32_t col = 0; col < numOfOutput; ++col) { - SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); - memcpy(pColRes->pData, src, pColRes->info.bytes * pRes->info.rows); + for (int32_t i = 0; i < numOfOutput; i++) { + int32_t functionId = pCtx[i].functionId; + if(functionId == TSDB_FUNC_TS_DUMMY) { + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i); + memcpy(pColRes->pData, src, pColRes->info.bytes * pRes->info.rows); + } } } From 2a45714d95d48c5001281427bdb8215f1dc2bacb Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Aug 2021 11:32:06 +0800 Subject: [PATCH 11/24] [TD-6046] fix ts derivative error --- src/query/src/qExecutor.c | 7 +++---- tests/pytest/functions/function_derivative.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 7ccaeb7ac9..0a092f1c2c 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5590,6 +5590,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { pRes->info.rows = getNumOfResult(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); if (pRes->info.rows >= pRuntimeEnv->resultInfo.threshold) { + copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); clearNumOfRes(pInfo->pCtx, pOperator->numOfOutput); return pRes; } @@ -5615,8 +5616,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { if (*newgroup) { if (pRes->info.rows > 0) { pProjectInfo->existDataBlock = pBlock; - clearNumOfRes(pInfo->pCtx, pOperator->numOfOutput); - return pInfo->pRes; + break; } else { // init output buffer for a new group data for (int32_t j = 0; j < pOperator->numOfOutput; ++j) { aAggs[pInfo->pCtx[j].functionId].xFinalize(&pInfo->pCtx[j]); @@ -5645,9 +5645,8 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { if (pRes->info.rows >= 1000/*pRuntimeEnv->resultInfo.threshold*/) { break; } - copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); } - + copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); clearNumOfRes(pInfo->pCtx, pOperator->numOfOutput); return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL; } diff --git a/tests/pytest/functions/function_derivative.py b/tests/pytest/functions/function_derivative.py index 9d60129672..f701379f5e 100644 --- a/tests/pytest/functions/function_derivative.py +++ b/tests/pytest/functions/function_derivative.py @@ -54,6 +54,19 @@ class TDTestCase: tdSql.query("select derivative(col, 10s, 0) from stb group by tbname") tdSql.checkRows(10) + tdSql.query("select ts,derivative(col, 10s, 1),ts from stb group by tbname") + tdSql.checkRows(4) + tdSql.checkData(0, 0, self.ts + 10000) + tdSql.checkData(0, 1, self.ts + 10000) + tdSql.checkData(0, 3, self.ts + 10000) + tdSql.checkData(3, 0, self.ts + 70000) + tdSql.checkData(3, 1, self.ts + 70000) + tdSql.checkData(3, 3, self.ts + 70000) + + tdSql.query("select ts from(select ts,derivative(col, 10s, 0) from stb group by tbname") + + tdSql.checkData(0, 1, 1) + tdSql.error("select derivative(col, 10s, 0) from tb1 group by tbname") tdSql.query("select derivative(col, 10s, 1) from tb1") From b37281d5d754dd10e4cc10c1b1cf7c409847945e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Aug 2021 12:08:58 +0800 Subject: [PATCH 12/24] [TD-6046] fix ts derivative error --- src/query/src/qExecutor.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 0a092f1c2c..7b320d9f58 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3645,15 +3645,16 @@ void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput) bool needCopyTs = false; int32_t tsNum = 0; char *src = NULL; + int32_t preFunctionId = TSDB_FUNC_TS_DUMMY; for (int32_t i = 0; i < numOfOutput; i++) { int32_t functionId = pCtx[i].functionId; if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { needCopyTs = true; - }else if(functionId == TSDB_FUNC_TS_DUMMY) { - SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i); - if (strlen(pColRes->pData) != 0) { - src = pColRes->pData; // find ts data + if (i > 0 && pCtx[i-1].functionId == TSDB_FUNC_TS_DUMMY){ + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i - 1); // find ts data + src = pColRes->pData; } + }else if(functionId == TSDB_FUNC_TS_DUMMY) { tsNum++; } } From 989299e59f8c3584cc78f69d23197eaede98ae5e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Aug 2021 12:11:31 +0800 Subject: [PATCH 13/24] [TD-6046] fix ts derivative error --- src/query/src/qExecutor.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 7b320d9f58..08146b6200 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3645,7 +3645,6 @@ void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput) bool needCopyTs = false; int32_t tsNum = 0; char *src = NULL; - int32_t preFunctionId = TSDB_FUNC_TS_DUMMY; for (int32_t i = 0; i < numOfOutput; i++) { int32_t functionId = pCtx[i].functionId; if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { From ee27f5758f08ff68a2d00f86aff936509a5c2cf0 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Aug 2021 12:24:34 +0800 Subject: [PATCH 14/24] [TD-6046] fix ts derivative error --- tests/pytest/functions/function_derivative.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/pytest/functions/function_derivative.py b/tests/pytest/functions/function_derivative.py index f701379f5e..0832977c3d 100644 --- a/tests/pytest/functions/function_derivative.py +++ b/tests/pytest/functions/function_derivative.py @@ -56,12 +56,12 @@ class TDTestCase: tdSql.query("select ts,derivative(col, 10s, 1),ts from stb group by tbname") tdSql.checkRows(4) - tdSql.checkData(0, 0, self.ts + 10000) - tdSql.checkData(0, 1, self.ts + 10000) - tdSql.checkData(0, 3, self.ts + 10000) - tdSql.checkData(3, 0, self.ts + 70000) - tdSql.checkData(3, 1, self.ts + 70000) - tdSql.checkData(3, 3, self.ts + 70000) + tdSql.checkData(0, 0, "2018-09-17 09:00:10.000") + tdSql.checkData(0, 1, "2018-09-17 09:00:10.000") + tdSql.checkData(0, 3, "2018-09-17 09:00:10.000") + tdSql.checkData(3, 0, "2018-09-17 09:01:20.000") + tdSql.checkData(3, 1, "2018-09-17 09:01:20.000") + tdSql.checkData(3, 3, "2018-09-17 09:01:20.000") tdSql.query("select ts from(select ts,derivative(col, 10s, 0) from stb group by tbname") From ea981c627de14f9b2fb76d1505327d3336b289ca Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Aug 2021 12:28:52 +0800 Subject: [PATCH 15/24] [TD-6046] fix ts derivative error --- tests/pytest/functions/function_derivative.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/functions/function_derivative.py b/tests/pytest/functions/function_derivative.py index 0832977c3d..7d34b3ce60 100644 --- a/tests/pytest/functions/function_derivative.py +++ b/tests/pytest/functions/function_derivative.py @@ -63,9 +63,9 @@ class TDTestCase: tdSql.checkData(3, 1, "2018-09-17 09:01:20.000") tdSql.checkData(3, 3, "2018-09-17 09:01:20.000") - tdSql.query("select ts from(select ts,derivative(col, 10s, 0) from stb group by tbname") + tdSql.query("select ts from(select ts,derivative(col, 10s, 0) from stb group by tbname)") - tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 0, "2018-09-17 09:00:10.000") tdSql.error("select derivative(col, 10s, 0) from tb1 group by tbname") From c2cf58938bdd59c51a64e0957ab629039b574394 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Aug 2021 14:48:17 +0800 Subject: [PATCH 16/24] [TD-6046] fix ts top/bottom error --- src/client/src/tscSQLParser.c | 2 +- src/query/src/qExecutor.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 781b1be76f..857d1a4419 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2872,7 +2872,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col const int32_t TS_COLUMN_INDEX = PRIMARYKEY_TIMESTAMP_COL_INDEX; SColumnList ids = createColumnList(1, index.tableIndex, TS_COLUMN_INDEX); - insertResultField(pQueryInfo, TS_COLUMN_INDEX, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, + insertResultField(pQueryInfo, colIndex, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS].name, pExpr); colIndex += 1; // the first column is ts diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 08146b6200..086b3d073c 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3851,7 +3851,7 @@ void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pRe } if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - pCtx[i].ptsOutputBuf = pCtx[0].pOutput; + if(i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; } if (!pResInfo->initialized) { From 1bd580402a508c69cd9e841054452952ad14dea3 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Aug 2021 18:11:47 +0800 Subject: [PATCH 17/24] [TD-6046] fix ts top/bottom error --- src/client/src/tscGlobalmerge.c | 2 +- src/query/src/qExecutor.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscGlobalmerge.c b/src/client/src/tscGlobalmerge.c index e696d54abd..ced81ff2f0 100644 --- a/src/client/src/tscGlobalmerge.c +++ b/src/client/src/tscGlobalmerge.c @@ -643,7 +643,7 @@ static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSD for(int32_t j = 0; j < numOfExpr; ++j) { pCtx[j].pOutput += (pCtx[j].outputBytes * numOfRows); if (pCtx[j].functionId == TSDB_FUNC_TOP || pCtx[j].functionId == TSDB_FUNC_BOTTOM) { - pCtx[j].ptsOutputBuf = pCtx[0].pOutput; + if(j > 0)pCtx[j].ptsOutputBuf = pCtx[j - 1].pOutput; } } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 086b3d073c..74c7f70437 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3587,7 +3587,7 @@ void setDefaultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, i // set the timestamp output buffer for top/bottom/diff query int32_t fid = pCtx[i].functionId; if (fid == TSDB_FUNC_TOP || fid == TSDB_FUNC_BOTTOM || fid == TSDB_FUNC_DIFF || fid == TSDB_FUNC_DERIVATIVE) { - pCtx[i].ptsOutputBuf = pCtx[0].pOutput; + if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; } } @@ -3912,7 +3912,7 @@ void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLF int32_t functionId = pCtx[i].functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { - pCtx[i].ptsOutputBuf = pCtx[0].pOutput; + if(i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; } /* From 2798480b00151a6b92a53f97d0e5a265412d163c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Aug 2021 18:34:34 +0800 Subject: [PATCH 18/24] [TD-6046] fix ts top/bottom error --- tests/pytest/functions/function_bottom.py | 15 +++++++++++++++ tests/pytest/functions/function_top.py | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/tests/pytest/functions/function_bottom.py b/tests/pytest/functions/function_bottom.py index abb9ac48e7..e9e5003f6f 100644 --- a/tests/pytest/functions/function_bottom.py +++ b/tests/pytest/functions/function_bottom.py @@ -104,6 +104,21 @@ class TDTestCase: tdSql.checkRows(2) tdSql.checkData(0, 1, 1) tdSql.checkData(1, 1, 2) + + tdSql.query("select ts,bottom(col1, 2),ts from test1") + tdSql.checkRows(2) + tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") + tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") + tdSql.checkData(1, 0, "2018-09-17 09:00:00.001") + tdSql.checkData(1, 3, "2018-09-17 09:00:00.001") + + + tdSql.query("select ts,bottom(col1, 2),ts from test group by tbname") + tdSql.checkRows(2) + tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") + tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") + tdSql.checkData(1, 0, "2018-09-17 09:00:00.001") + tdSql.checkData(1, 3, "2018-09-17 09:00:00.001") #TD-2457 bottom + interval + order by tdSql.error('select top(col2,1) from test interval(1y) order by col2;') diff --git a/tests/pytest/functions/function_top.py b/tests/pytest/functions/function_top.py index f8318402b5..9824afc19f 100644 --- a/tests/pytest/functions/function_top.py +++ b/tests/pytest/functions/function_top.py @@ -117,6 +117,21 @@ class TDTestCase: tdSql.checkRows(2) tdSql.checkData(0, 1, 8.1) tdSql.checkData(1, 1, 9.1) + + tdSql.query("select ts,top(col1, 2),ts from test1") + tdSql.checkRows(2) + tdSql.checkData(0, 0, "2018-09-17 09:00:00.008") + tdSql.checkData(0, 1, "2018-09-17 09:00:00.008") + tdSql.checkData(1, 0, "2018-09-17 09:00:00.009") + tdSql.checkData(1, 3, "2018-09-17 09:00:00.009") + + + tdSql.query("select ts,top(col1, 2),ts from test group by tbname") + tdSql.checkRows(2) + tdSql.checkData(0, 0, "2018-09-17 09:00:00.008") + tdSql.checkData(0, 1, "2018-09-17 09:00:00.008") + tdSql.checkData(1, 0, "2018-09-17 09:00:00.009") + tdSql.checkData(1, 3, "2018-09-17 09:00:00.009") #TD-2563 top + super_table + interval tdSql.execute("create table meters(ts timestamp, c int) tags (d int)") From 045f8736369a7d53f092039b5f3ed8dd628a784b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Aug 2021 18:45:56 +0800 Subject: [PATCH 19/24] [TD-6046] fix ts top/bottom error --- tests/pytest/functions/function_derivative.py | 10 ++++++++++ tests/pytest/functions/function_diff.py | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/tests/pytest/functions/function_derivative.py b/tests/pytest/functions/function_derivative.py index 7d34b3ce60..8b77159ffa 100644 --- a/tests/pytest/functions/function_derivative.py +++ b/tests/pytest/functions/function_derivative.py @@ -63,6 +63,16 @@ class TDTestCase: tdSql.checkData(3, 1, "2018-09-17 09:01:20.000") tdSql.checkData(3, 3, "2018-09-17 09:01:20.000") + tdSql.query("select ts,derivative(col1, 10, 1),ts from tb1") + tdSql.checkRows(2) + tdSql.checkData(0, 0, "2018-09-17 09:00:10.000") + tdSql.checkData(0, 1, "2018-09-17 09:00:10.000") + tdSql.checkData(0, 3, "2018-09-17 09:00:10.000") + tdSql.checkData(1, 0, "2018-09-17 09:00:20.009") + tdSql.checkData(1, 1, "2018-09-17 09:00:20.009") + tdSql.checkData(1, 3, "2018-09-17 09:00:20.009") + + tdSql.query("select ts from(select ts,derivative(col, 10s, 0) from stb group by tbname)") tdSql.checkData(0, 0, "2018-09-17 09:00:10.000") diff --git a/tests/pytest/functions/function_diff.py b/tests/pytest/functions/function_diff.py index fba3b4c0d4..4ef8ef7a98 100644 --- a/tests/pytest/functions/function_diff.py +++ b/tests/pytest/functions/function_diff.py @@ -95,6 +95,24 @@ class TDTestCase: tdSql.error("select diff(col14) from test") + tdSql.query("select ts,diff(col1),ts from test1") + tdSql.checkRows(10) + tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") + tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") + tdSql.checkData(0, 3, "2018-09-17 09:00:00.000") + tdSql.checkData(9, 0, "2018-09-17 09:00:00.009") + tdSql.checkData(9, 1, "2018-09-17 09:00:00.009") + tdSql.checkData(9, 3, "2018-09-17 09:00:00.009") + + tdSql.query("select ts,diff(col1),ts from test group by tbname") + tdSql.checkRows(10) + tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") + tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") + tdSql.checkData(0, 3, "2018-09-17 09:00:00.000") + tdSql.checkData(9, 0, "2018-09-17 09:00:00.009") + tdSql.checkData(9, 1, "2018-09-17 09:00:00.009") + tdSql.checkData(9, 3, "2018-09-17 09:00:00.009") + tdSql.query("select diff(col1) from test1") tdSql.checkRows(10) From e98863b5fd6feec4d9aaf92264e40550146ec6e9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Aug 2021 18:48:01 +0800 Subject: [PATCH 20/24] [TD-6046] fix ts top/bottom error --- tests/pytest/functions/function_derivative.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/functions/function_derivative.py b/tests/pytest/functions/function_derivative.py index 8b77159ffa..61a3ed72fe 100644 --- a/tests/pytest/functions/function_derivative.py +++ b/tests/pytest/functions/function_derivative.py @@ -63,7 +63,7 @@ class TDTestCase: tdSql.checkData(3, 1, "2018-09-17 09:01:20.000") tdSql.checkData(3, 3, "2018-09-17 09:01:20.000") - tdSql.query("select ts,derivative(col1, 10, 1),ts from tb1") + tdSql.query("select ts,derivative(col, 10, 1),ts from tb1") tdSql.checkRows(2) tdSql.checkData(0, 0, "2018-09-17 09:00:10.000") tdSql.checkData(0, 1, "2018-09-17 09:00:10.000") From 0e4b683b6d55a4e9c5df07b135df571da7533de3 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Aug 2021 18:48:50 +0800 Subject: [PATCH 21/24] [TD-6046] fix ts top/bottom error --- tests/pytest/functions/function_derivative.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/functions/function_derivative.py b/tests/pytest/functions/function_derivative.py index 61a3ed72fe..a97a041d0b 100644 --- a/tests/pytest/functions/function_derivative.py +++ b/tests/pytest/functions/function_derivative.py @@ -63,7 +63,7 @@ class TDTestCase: tdSql.checkData(3, 1, "2018-09-17 09:01:20.000") tdSql.checkData(3, 3, "2018-09-17 09:01:20.000") - tdSql.query("select ts,derivative(col, 10, 1),ts from tb1") + tdSql.query("select ts,derivative(col, 10s, 1),ts from tb1") tdSql.checkRows(2) tdSql.checkData(0, 0, "2018-09-17 09:00:10.000") tdSql.checkData(0, 1, "2018-09-17 09:00:10.000") From dcb822e00c365e20a3a9d1ed508cb0d34847d932 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 19 Aug 2021 17:28:27 +0800 Subject: [PATCH 22/24] [TD-6046] fix ts top/bottom index error when using order by --- src/client/inc/tscUtil.h | 1 + src/client/src/tscSQLParser.c | 12 ++++++++---- src/client/src/tscUtil.c | 11 +++++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index f2d25c1e84..f14cd8f9e2 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -213,6 +213,7 @@ SExprInfo* tscExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t function int16_t size); size_t tscNumOfExprs(SQueryInfo* pQueryInfo); +size_t tscExprTopBottomIndex(SQueryInfo* pQueryInfo); SExprInfo *tscExprGet(SQueryInfo* pQueryInfo, int32_t index); int32_t tscExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy); int32_t tscExprCopyAll(SArray* dst, const SArray* src, bool deepcopy); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 857d1a4419..63ce5ebf2c 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5864,10 +5864,12 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; } else if (isTopBottomQuery(pQueryInfo)) { /* order of top/bottom query in interval is not valid */ - SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); + size_t pos = tscExprTopBottomIndex(pQueryInfo); + assert(pos > 0); + SExprInfo* pExpr = tscExprGet(pQueryInfo, pos - 1); assert(pExpr->base.functionId == TSDB_FUNC_TS); - pExpr = tscExprGet(pQueryInfo, 1); + pExpr = tscExprGet(pQueryInfo, pos); if (pExpr->base.colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } @@ -5949,10 +5951,12 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq } } else { /* order of top/bottom query in interval is not valid */ - SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); + size_t pos = tscExprTopBottomIndex(pQueryInfo); + assert(pos > 0); + SExprInfo* pExpr = tscExprGet(pQueryInfo, pos - 1); assert(pExpr->base.functionId == TSDB_FUNC_TS); - pExpr = tscExprGet(pQueryInfo, 1); + pExpr = tscExprGet(pQueryInfo, pos); if (pExpr->base.colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 19a816faeb..72791568ba 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2427,6 +2427,17 @@ size_t tscNumOfExprs(SQueryInfo* pQueryInfo) { return taosArrayGetSize(pQueryInfo->exprList); } +size_t tscExprTopBottomIndex(SQueryInfo* pQueryInfo){ + size_t numOfExprs = tscNumOfExprs(pQueryInfo); + for(int32_t i = 0; i < numOfExprs; ++i) { + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); + if (pExpr->base.functionId == TSDB_FUNC_TOP || pExpr->base.functionId == TSDB_FUNC_BOTTOM) { + return i; + } + } + return -1; +} + // todo REFACTOR void tscExprAddParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes) { assert (pExpr != NULL || argument != NULL || bytes != 0); From 938275238990e5c79f08d5c9ad14356bfd624260 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 20 Aug 2021 10:34:53 +0800 Subject: [PATCH 23/24] [TD-6046] fix ts top/bottom index error when using order by --- src/client/inc/tscUtil.h | 2 +- src/client/src/tscSQLParser.c | 4 ++-- src/client/src/tscUtil.c | 6 ++++-- tests/pytest/functions/function_top.py | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index f14cd8f9e2..690ec2c277 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -213,7 +213,7 @@ SExprInfo* tscExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t function int16_t size); size_t tscNumOfExprs(SQueryInfo* pQueryInfo); -size_t tscExprTopBottomIndex(SQueryInfo* pQueryInfo); +int32_t tscExprTopBottomIndex(SQueryInfo* pQueryInfo); SExprInfo *tscExprGet(SQueryInfo* pQueryInfo, int32_t index); int32_t tscExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy); int32_t tscExprCopyAll(SArray* dst, const SArray* src, bool deepcopy); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 63ce5ebf2c..bb871488fc 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5864,7 +5864,7 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; } else if (isTopBottomQuery(pQueryInfo)) { /* order of top/bottom query in interval is not valid */ - size_t pos = tscExprTopBottomIndex(pQueryInfo); + int32_t pos = tscExprTopBottomIndex(pQueryInfo); assert(pos > 0); SExprInfo* pExpr = tscExprGet(pQueryInfo, pos - 1); assert(pExpr->base.functionId == TSDB_FUNC_TS); @@ -5951,7 +5951,7 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq } } else { /* order of top/bottom query in interval is not valid */ - size_t pos = tscExprTopBottomIndex(pQueryInfo); + int32_t pos = tscExprTopBottomIndex(pQueryInfo); assert(pos > 0); SExprInfo* pExpr = tscExprGet(pQueryInfo, pos - 1); assert(pExpr->base.functionId == TSDB_FUNC_TS); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 72791568ba..c5aedf1b71 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2427,10 +2427,12 @@ size_t tscNumOfExprs(SQueryInfo* pQueryInfo) { return taosArrayGetSize(pQueryInfo->exprList); } -size_t tscExprTopBottomIndex(SQueryInfo* pQueryInfo){ +int32_t tscExprTopBottomIndex(SQueryInfo* pQueryInfo){ size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for(int32_t i = 0; i < numOfExprs; ++i) { + for(size_t i = 0; i < numOfExprs; ++i) { SExprInfo* pExpr = tscExprGet(pQueryInfo, i); + if (pExpr == NULL) + continue; if (pExpr->base.functionId == TSDB_FUNC_TOP || pExpr->base.functionId == TSDB_FUNC_BOTTOM) { return i; } diff --git a/tests/pytest/functions/function_top.py b/tests/pytest/functions/function_top.py index 9824afc19f..03a00d918a 100644 --- a/tests/pytest/functions/function_top.py +++ b/tests/pytest/functions/function_top.py @@ -132,7 +132,7 @@ class TDTestCase: tdSql.checkData(0, 1, "2018-09-17 09:00:00.008") tdSql.checkData(1, 0, "2018-09-17 09:00:00.009") tdSql.checkData(1, 3, "2018-09-17 09:00:00.009") - + #TD-2563 top + super_table + interval tdSql.execute("create table meters(ts timestamp, c int) tags (d int)") tdSql.execute("create table t1 using meters tags (1)") From aa3102cd72270145c3a2becbd2b4cdabf38690b4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 21 Aug 2021 11:10:59 +0800 Subject: [PATCH 24/24] [TD-6046] fix ts top/bottom error --- src/client/src/tscUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index c5aedf1b71..c2df10b223 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2429,7 +2429,7 @@ size_t tscNumOfExprs(SQueryInfo* pQueryInfo) { int32_t tscExprTopBottomIndex(SQueryInfo* pQueryInfo){ size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for(size_t i = 0; i < numOfExprs; ++i) { + for(int32_t i = 0; i < numOfExprs; ++i) { SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr == NULL) continue;