From 0d7e1eb4e03a02d1e9995285a28a9b4106ecd412 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 13 Sep 2022 16:45:40 +0800 Subject: [PATCH 1/6] fix(query): twa function output NULL if input has overlap timestamps TD-18224 --- source/libs/function/src/builtinsimpl.c | 54 ++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 0d7fd1a6da..35e3e07839 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -5398,8 +5398,8 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { int32_t i = pInput->startRowIndex; if (pCtx->start.key != INT64_MIN) { - ASSERT((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) || - (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC)); + //ASSERT((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) || + // (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC)); ASSERT(last->key == INT64_MIN); for (; i < pInput->numOfRows + pInput->startRowIndex; ++i) { @@ -5446,6 +5446,11 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } numOfElems++; + if (pInfo->p.key == st.key) { + numOfElems = 0; + goto _twa_over; + } + INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; @@ -5461,6 +5466,11 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } numOfElems++; + if (pInfo->p.key == st.key) { + numOfElems = 0; + goto _twa_over; + } + INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; @@ -5475,6 +5485,11 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } numOfElems++; + if (pInfo->p.key == st.key) { + numOfElems = 0; + goto _twa_over; + } + INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; @@ -5489,6 +5504,11 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } numOfElems++; + if (pInfo->p.key == st.key) { + numOfElems = 0; + goto _twa_over; + } + INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; @@ -5503,6 +5523,11 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } numOfElems++; + if (pInfo->p.key == st.key) { + numOfElems = 0; + goto _twa_over; + } + INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; @@ -5517,6 +5542,11 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } numOfElems++; + if (pInfo->p.key == st.key) { + numOfElems = 0; + goto _twa_over; + } + INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; @@ -5531,6 +5561,11 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } numOfElems++; + if (pInfo->p.key == st.key) { + numOfElems = 0; + goto _twa_over; + } + INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; @@ -5545,6 +5580,11 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } numOfElems++; + if (pInfo->p.key == st.key) { + numOfElems = 0; + goto _twa_over; + } + INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; @@ -5559,6 +5599,11 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } numOfElems++; + if (pInfo->p.key == st.key) { + numOfElems = 0; + goto _twa_over; + } + INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; @@ -5573,6 +5618,11 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } numOfElems++; + if (pInfo->p.key == st.key) { + numOfElems = 0; + goto _twa_over; + } + INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; From 6b2bf42f6824df1599303768160cfc403c04b7c1 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 13 Sep 2022 16:45:40 +0800 Subject: [PATCH 2/6] fix(query): twa function output NULL if input has overlap timestamps TD-18224 --- include/util/taoserror.h | 1 + source/libs/function/src/builtinsimpl.c | 30 +++++++++---------------- source/util/src/terror.c | 1 + 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index d16a599811..0058b03b30 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -577,6 +577,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FUNC_FUNTION_PARA_TYPE TAOS_DEF_ERROR_CODE(0, 0x2802) #define TSDB_CODE_FUNC_FUNTION_PARA_VALUE TAOS_DEF_ERROR_CODE(0, 0x2803) #define TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION TAOS_DEF_ERROR_CODE(0, 0x2804) +#define TSDB_CODE_FUNC_DUP_TIMESTAMP TAOS_DEF_ERROR_CODE(0, 0x012B) //udf #define TSDB_CODE_UDF_STOPPING TAOS_DEF_ERROR_CODE(0, 0x2901) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 35e3e07839..7d8c231c15 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -5447,8 +5447,7 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { numOfElems++; if (pInfo->p.key == st.key) { - numOfElems = 0; - goto _twa_over; + return TSDB_CODE_FUNC_DUP_TIMESTAMP; } INIT_INTP_POINT(st, tsList[i], val[i]); @@ -5467,8 +5466,7 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { numOfElems++; if (pInfo->p.key == st.key) { - numOfElems = 0; - goto _twa_over; + return TSDB_CODE_FUNC_DUP_TIMESTAMP; } INIT_INTP_POINT(st, tsList[i], val[i]); @@ -5486,8 +5484,7 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { numOfElems++; if (pInfo->p.key == st.key) { - numOfElems = 0; - goto _twa_over; + return TSDB_CODE_FUNC_DUP_TIMESTAMP; } INIT_INTP_POINT(st, tsList[i], val[i]); @@ -5505,8 +5502,7 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { numOfElems++; if (pInfo->p.key == st.key) { - numOfElems = 0; - goto _twa_over; + return TSDB_CODE_FUNC_DUP_TIMESTAMP; } INIT_INTP_POINT(st, tsList[i], val[i]); @@ -5524,8 +5520,7 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { numOfElems++; if (pInfo->p.key == st.key) { - numOfElems = 0; - goto _twa_over; + return TSDB_CODE_FUNC_DUP_TIMESTAMP; } INIT_INTP_POINT(st, tsList[i], val[i]); @@ -5543,8 +5538,7 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { numOfElems++; if (pInfo->p.key == st.key) { - numOfElems = 0; - goto _twa_over; + return TSDB_CODE_FUNC_DUP_TIMESTAMP; } INIT_INTP_POINT(st, tsList[i], val[i]); @@ -5562,8 +5556,7 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { numOfElems++; if (pInfo->p.key == st.key) { - numOfElems = 0; - goto _twa_over; + return TSDB_CODE_FUNC_DUP_TIMESTAMP; } INIT_INTP_POINT(st, tsList[i], val[i]); @@ -5581,8 +5574,7 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { numOfElems++; if (pInfo->p.key == st.key) { - numOfElems = 0; - goto _twa_over; + return TSDB_CODE_FUNC_DUP_TIMESTAMP; } INIT_INTP_POINT(st, tsList[i], val[i]); @@ -5600,8 +5592,7 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { numOfElems++; if (pInfo->p.key == st.key) { - numOfElems = 0; - goto _twa_over; + return TSDB_CODE_FUNC_DUP_TIMESTAMP; } INIT_INTP_POINT(st, tsList[i], val[i]); @@ -5619,8 +5610,7 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { numOfElems++; if (pInfo->p.key == st.key) { - numOfElems = 0; - goto _twa_over; + return TSDB_CODE_FUNC_DUP_TIMESTAMP; } INIT_INTP_POINT(st, tsList[i], val[i]); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 044cdc86b4..ab0771d218 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -579,6 +579,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_FUNTION_PARA_NUM, "Invalid function par TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_FUNTION_PARA_TYPE, "Invalid function para type") TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_FUNTION_PARA_VALUE, "Invalid function para value") TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION, "Not buildin function") +TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_DUP_TIMESTAMP, "Duplicate timestamps not allowed in function") //udf TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING, "udf is stopping") From 5463159a3d5a8f56a4f3f08f9864a79c24013c0d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 13 Sep 2022 18:24:29 +0800 Subject: [PATCH 3/6] fix twa function input dup timestamps --- source/libs/function/src/builtinsimpl.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 7d8c231c15..5862a06aac 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -5446,11 +5446,11 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } numOfElems++; + INIT_INTP_POINT(st, tsList[i], val[i]); if (pInfo->p.key == st.key) { return TSDB_CODE_FUNC_DUP_TIMESTAMP; } - INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -5465,11 +5465,11 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } numOfElems++; + INIT_INTP_POINT(st, tsList[i], val[i]); if (pInfo->p.key == st.key) { return TSDB_CODE_FUNC_DUP_TIMESTAMP; } - INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -5483,11 +5483,11 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } numOfElems++; + INIT_INTP_POINT(st, tsList[i], val[i]); if (pInfo->p.key == st.key) { return TSDB_CODE_FUNC_DUP_TIMESTAMP; } - INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -5501,11 +5501,11 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } numOfElems++; + INIT_INTP_POINT(st, tsList[i], val[i]); if (pInfo->p.key == st.key) { return TSDB_CODE_FUNC_DUP_TIMESTAMP; } - INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -5519,11 +5519,11 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } numOfElems++; + INIT_INTP_POINT(st, tsList[i], val[i]); if (pInfo->p.key == st.key) { return TSDB_CODE_FUNC_DUP_TIMESTAMP; } - INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -5537,11 +5537,11 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } numOfElems++; + INIT_INTP_POINT(st, tsList[i], val[i]); if (pInfo->p.key == st.key) { return TSDB_CODE_FUNC_DUP_TIMESTAMP; } - INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -5555,11 +5555,11 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } numOfElems++; + INIT_INTP_POINT(st, tsList[i], val[i]); if (pInfo->p.key == st.key) { return TSDB_CODE_FUNC_DUP_TIMESTAMP; } - INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -5573,11 +5573,11 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } numOfElems++; + INIT_INTP_POINT(st, tsList[i], val[i]); if (pInfo->p.key == st.key) { return TSDB_CODE_FUNC_DUP_TIMESTAMP; } - INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -5591,11 +5591,11 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } numOfElems++; + INIT_INTP_POINT(st, tsList[i], val[i]); if (pInfo->p.key == st.key) { return TSDB_CODE_FUNC_DUP_TIMESTAMP; } - INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -5609,11 +5609,11 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } numOfElems++; + INIT_INTP_POINT(st, tsList[i], val[i]); if (pInfo->p.key == st.key) { return TSDB_CODE_FUNC_DUP_TIMESTAMP; } - INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } From 9ee003c971aa2c064c1da800aa1e366ffd262eab Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 13 Sep 2022 18:32:17 +0800 Subject: [PATCH 4/6] fix test case --- tests/system-test/2-query/twa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/twa.py b/tests/system-test/2-query/twa.py index 62940477cf..4c163da485 100644 --- a/tests/system-test/2-query/twa.py +++ b/tests/system-test/2-query/twa.py @@ -124,7 +124,7 @@ class TDTestCase: tdSql.checkData(0,1,4.500000000) # mixup with other functions - tdSql.query(f"select twa(c1),twa(c2),max(c1),elapsed(ts) from {dbname}.stb1 ") + tdSql.query(f"select twa(c1),twa(c2),max(c1),elapsed(ts) from {dbname}.ct1 ") tdSql.checkData(0,0,1.000000000) tdSql.checkData(0,1,11111.000000000) tdSql.checkData(0,2,1) From e076b8c7c24208b3c9d2ca3653c6930454380f85 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 14 Sep 2022 11:05:24 +0800 Subject: [PATCH 5/6] fix test case --- tests/system-test/2-query/max_partition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/max_partition.py b/tests/system-test/2-query/max_partition.py index 01c2677242..08bb7675ad 100644 --- a/tests/system-test/2-query/max_partition.py +++ b/tests/system-test/2-query/max_partition.py @@ -181,7 +181,7 @@ class TDTestCase: # bug need fix tdSql.checkData(0,1,None) - tdSql.query(f"select c1 , twa(c1) from {dbname}.stb partition by c1 order by c1") + tdSql.query(f"select c1 , twa(c1) from {dbname}.sub_stb_1 partition by c1 order by c1") tdSql.checkRows(11) tdSql.checkData(0,1,None) From 64226e5e55131c39ae987f9a4553b03c8f2329d8 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 14 Sep 2022 15:01:14 +0800 Subject: [PATCH 6/6] fix error code number --- include/util/taoserror.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 0058b03b30..086de593d0 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -577,7 +577,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FUNC_FUNTION_PARA_TYPE TAOS_DEF_ERROR_CODE(0, 0x2802) #define TSDB_CODE_FUNC_FUNTION_PARA_VALUE TAOS_DEF_ERROR_CODE(0, 0x2803) #define TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION TAOS_DEF_ERROR_CODE(0, 0x2804) -#define TSDB_CODE_FUNC_DUP_TIMESTAMP TAOS_DEF_ERROR_CODE(0, 0x012B) +#define TSDB_CODE_FUNC_DUP_TIMESTAMP TAOS_DEF_ERROR_CODE(0, 0x2805) //udf #define TSDB_CODE_UDF_STOPPING TAOS_DEF_ERROR_CODE(0, 0x2901)