From 5b2d908cb27586af7c2620d1319dcc04392d87f1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 19 Jun 2021 09:16:35 +0800 Subject: [PATCH 1/6] [TD-4734] session_window and state window support main query --- src/client/src/tscSQLParser.c | 12 +++++++++++- src/query/src/qExecutor.c | 10 ++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 9532d1e202..93d94c8711 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -7774,8 +7774,15 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf return code; } } + + int32_t timeWindowQuery = + (TPARSER_HAS_TOKEN(pSqlNode->interval.interval) || TPARSER_HAS_TOKEN(pSqlNode->sessionVal.gap)); - if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, false, false, false) != TSDB_CODE_SUCCESS) { + if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, false, false, timeWindowQuery) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + // parse the window_state + if (validateStateWindowNode(pCmd, pQueryInfo, pSqlNode, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } @@ -7815,6 +7822,9 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf if (validateIntervalNode(pSql, pQueryInfo, pSqlNode) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } else { + if (validateSessionNode(pCmd, pQueryInfo, pSqlNode) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } if (isTimeWindowQuery(pQueryInfo) || pQueryInfo->sessionWindow.gap > 0) { // check if the first column of the nest query result is timestamp column SColumn* pCol = taosArrayGetP(pQueryInfo->colList, 0); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 7b106c178d..f4c0a394d6 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1751,7 +1751,10 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf case OP_SessionWindow: { pRuntimeEnv->proot = createSWindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; + if (opType != OP_DummyInput) { + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + } break; } case OP_MultiTableAggregate: { @@ -1787,7 +1790,10 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf } case OP_StateWindow: { pRuntimeEnv->proot = createStatewindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; + if (opType != OP_DummyInput) { + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + } break; } From 998c5f18421a2cb5fd31c3b7e1187c438864fb56 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sat, 19 Jun 2021 16:09:41 +0800 Subject: [PATCH 2/6] [TD-4735]:support select last_row from subquery --- src/client/src/tscSQLParser.c | 11 +++++++++-- tests/script/general/parser/lastrow.sim | 15 ++++++++++++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 9532d1e202..d7fbdd3ff8 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2125,7 +2125,10 @@ void setResultColName(char* name, tSqlExprItem* pItem, int32_t functionId, SStrT } static void updateLastScanOrderIfNeeded(SQueryInfo* pQueryInfo) { - if (pQueryInfo->sessionWindow.gap > 0 || tscGroupbyColumn(pQueryInfo)) { + if (pQueryInfo->sessionWindow.gap > 0 || + pQueryInfo->stateWindow || + taosArrayGetSize(pQueryInfo->pUpstream) > 0 || + tscGroupbyColumn(pQueryInfo)) { size_t numOfExpr = tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < numOfExpr; ++i) { SExprInfo* pExpr = tscExprGet(pQueryInfo, i); @@ -2385,7 +2388,9 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // NOTE: has time range condition or normal column filter condition, the last_row query will be transferred to last query SConvertFunc cvtFunc = {.originFuncId = functionId, .execFuncId = functionId}; - if (functionId == TSDB_FUNC_LAST_ROW && ((!TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER)) || (hasNormalColumnFilter(pQueryInfo)))) { + if (functionId == TSDB_FUNC_LAST_ROW && ((!TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER)) || + (hasNormalColumnFilter(pQueryInfo)) || + taosArrayGetSize(pQueryInfo->pUpstream)>0)) { cvtFunc.execFuncId = TSDB_FUNC_LAST; } @@ -7836,6 +7841,8 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf if ((code = doFunctionsCompatibleCheck(pCmd, pQueryInfo, tscGetErrorMsgPayload(pCmd))) != TSDB_CODE_SUCCESS) { return code; } + + updateLastScanOrderIfNeeded(pQueryInfo); } else { pQueryInfo->command = TSDB_SQL_SELECT; diff --git a/tests/script/general/parser/lastrow.sim b/tests/script/general/parser/lastrow.sim index 2b8f294d5d..fea322ec16 100644 --- a/tests/script/general/parser/lastrow.sim +++ b/tests/script/general/parser/lastrow.sim @@ -70,4 +70,17 @@ sleep 100 run general/parser/lastrow_query.sim -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +print =================== last_row + nested query +sql use $db +sql create table lr_nested(ts timestamp, f int) +sql insert into lr_nested values(now, 1) +sql insert into lr_nested values(now+1s, null) +sql select last_row(*) from (select * from lr_nested) +if $rows != 1 then + return -1 +endi +if $data01 != NULL then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 65eeca1964fda463446eef892cbe1efd81206255 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 20 Jun 2021 21:51:15 +0800 Subject: [PATCH 3/6] [TD-4734] session_window and state window support main query --- src/client/src/tscSQLParser.c | 11 +++++-- tests/pytest/query/querySession.py | 49 ++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 93d94c8711..3a8e2407c0 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -7792,6 +7792,12 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf int32_t f = pExpr->base.functionId; if (f == TSDB_FUNC_STDDEV || f == TSDB_FUNC_PERCT) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); + } + + if ((timeWindowQuery || pQueryInfo->stateWindow) && f == TSDB_FUNC_LAST) { + pExpr->base.numOfParams = 1; + pExpr->base.param[0].i64 = TSDB_ORDER_ASC; + pExpr->base.param[0].nType = TSDB_DATA_TYPE_INT; } } @@ -7825,7 +7831,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf if (validateSessionNode(pCmd, pQueryInfo, pSqlNode) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } - if (isTimeWindowQuery(pQueryInfo) || pQueryInfo->sessionWindow.gap > 0) { + if (isTimeWindowQuery(pQueryInfo)) { // check if the first column of the nest query result is timestamp column SColumn* pCol = taosArrayGetP(pQueryInfo->colList, 0); if (pCol->info.type != TSDB_DATA_TYPE_TIMESTAMP) { @@ -7934,8 +7940,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf return TSDB_CODE_TSC_INVALID_OPERATION; } - if ((isTimeWindowQuery(pQueryInfo) || pQueryInfo->sessionWindow.gap > 0) && - (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) { + if (isTimeWindowQuery(pQueryInfo) && (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) { return TSDB_CODE_TSC_INVALID_OPERATION; } diff --git a/tests/pytest/query/querySession.py b/tests/pytest/query/querySession.py index 620f755bcb..216ff68b71 100644 --- a/tests/pytest/query/querySession.py +++ b/tests/pytest/query/querySession.py @@ -51,38 +51,73 @@ class TDTestCase: tdSql.checkRows(15) tdSql.checkData(0, 1, 2) + # session(ts,5a) main query + tdSql.query("select count(*) from (select * from dev_001) session(ts,5a)") + tdSql.checkRows(15) + tdSql.checkData(0, 1, 2) + # session(ts,1s) tdSql.query("select count(*) from dev_001 session(ts,1s)") tdSql.checkRows(12) tdSql.checkData(0, 1, 5) + # session(ts,1s) main query + tdSql.query("select count(*) from (select * from dev_001) session(ts,1s)") + tdSql.checkRows(12) + tdSql.checkData(0, 1, 5) + tdSql.query("select count(*) from dev_001 session(ts,1000a)") tdSql.checkRows(12) tdSql.checkData(0, 1, 5) + tdSql.query("select count(*) from (select * from dev_001) session(ts,1000a)") + tdSql.checkRows(12) + tdSql.checkData(0, 1, 5) + # session(ts,1m) tdSql.query("select count(*) from dev_001 session(ts,1m)") tdSql.checkRows(9) tdSql.checkData(0, 1, 8) + # session(ts,1m) + tdSql.query("select count(*) from (select * from dev_001) session(ts,1m)") + tdSql.checkRows(9) + tdSql.checkData(0, 1, 8) + # session(ts,1h) tdSql.query("select count(*) from dev_001 session(ts,1h)") tdSql.checkRows(6) tdSql.checkData(0, 1, 11) + # session(ts,1h) + tdSql.query("select count(*) from (select * from dev_001) session(ts,1h)") + tdSql.checkRows(6) + tdSql.checkData(0, 1, 11) + # session(ts,1d) tdSql.query("select count(*) from dev_001 session(ts,1d)") tdSql.checkRows(4) tdSql.checkData(0, 1, 13) + # session(ts,1d) + tdSql.query("select count(*) from (select * from dev_001) session(ts,1d)") + tdSql.checkRows(4) + tdSql.checkData(0, 1, 13) + # session(ts,1w) tdSql.query("select count(*) from dev_001 session(ts,1w)") tdSql.checkRows(2) tdSql.checkData(0, 1, 15) + # session(ts,1w) + tdSql.query("select count(*) from (select * from dev_001) session(ts,1w)") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 15) + # session with where tdSql.query("select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1),spread(tagtype),stddev(tagtype),percentile(tagtype,0) from dev_001 where ts <'2020-05-20 0:0:0' session(ts,1d)") + tdSql.checkRows(2) tdSql.checkData(0, 1, 13) tdSql.checkData(0, 2, 1) @@ -97,6 +132,20 @@ class TDTestCase: tdSql.checkData(0, 11, 1) tdSql.checkData(1, 11, 14) + # session with where main + + tdSql.query("select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1) from (select * from dev_001 where ts <'2020-05-20 0:0:0') session(ts,1d)") + + tdSql.checkRows(2) + tdSql.checkData(0, 1, 13) + tdSql.checkData(0, 2, 1) + tdSql.checkData(0, 3, 13) + tdSql.checkData(0, 4, 7) + tdSql.checkData(0, 5, 91) + tdSql.checkData(0, 6, 1) + tdSql.checkData(0, 7, 13) + tdSql.checkData(0, 8, '{slop:1.000000, intercept:0.000000}') + # tdsql err tdSql.error("select * from dev_001 session(ts,1w)") tdSql.error("select count(*) from st session(ts,1w)") From c1cd4079db810ccb700eddcaeb2ed31d27b1b826 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 21 Jun 2021 00:24:46 +0800 Subject: [PATCH 4/6] [TD-4734] session_window and state window support main query --- src/client/src/tscSQLParser.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 3a8e2407c0..b4115e5842 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -7785,7 +7785,6 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf if (validateStateWindowNode(pCmd, pQueryInfo, pSqlNode, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } - // todo NOT support yet for(int32_t i = 0; i < tscNumOfExprs(pQueryInfo); ++i) { SExprInfo* pExpr = tscExprGet(pQueryInfo, i); From 207481330e0255b2d2a3ae40e0a8197320aed5f5 Mon Sep 17 00:00:00 2001 From: zyyang-taosdata Date: Mon, 21 Jun 2021 17:55:32 +0800 Subject: [PATCH 5/6] change version number --- cmake/version.inc | 2 +- snap/snapcraft.yaml | 2 +- src/connector/go | 2 +- src/connector/hivemq-tdengine-extension | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/version.inc b/cmake/version.inc index ed8e7a156c..134f09f179 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.1.1.0") + SET(TD_VER_NUMBER "2.1.3.0") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index a63225ab32..c7c7846062 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.1.1.0' +version: '2.1.3.0' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | diff --git a/src/connector/go b/src/connector/go index 7a26c432f8..b8f76da4a7 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit 7a26c432f8b4203e42344ff3290b9b9b01b983d5 +Subproject commit b8f76da4a708d158ec3cc4b844571dc4414e36b4 diff --git a/src/connector/hivemq-tdengine-extension b/src/connector/hivemq-tdengine-extension index b62a26ecc1..ce52010141 160000 --- a/src/connector/hivemq-tdengine-extension +++ b/src/connector/hivemq-tdengine-extension @@ -1 +1 @@ -Subproject commit b62a26ecc164a310104df57691691b237e091c89 +Subproject commit ce5201014136503d34fecbd56494b67b4961056c From 1b6c658a2596671872cf87796613f8797d2c9f7d Mon Sep 17 00:00:00 2001 From: zyyang-taosdata Date: Mon, 21 Jun 2021 18:07:55 +0800 Subject: [PATCH 6/6] change snap version --- snap/snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index c7c7846062..65a8e6b684 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.1.1.0 + - usr/lib/libtaos.so.2.1.3.0 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so