From f18fc19d769d44ab2fb4b9154592e13feb60215e Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 22 May 2023 16:13:50 +0800 Subject: [PATCH 1/5] fix: tsdb read and tsma _wstart append --- source/dnode/vnode/src/tsdb/tsdbRead.c | 6 +++--- source/libs/planner/src/planOptimizer.c | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index cde2672541..b66fc42f74 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -810,7 +810,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd if (pReader->suppInfo.colId[0] != PRIMARYKEY_TIMESTAMP_COL_ID) { tsdbError("the first column isn't primary timestamp, %d, %s", pReader->suppInfo.colId[0], pReader->idStr); - terrno = TSDB_CODE_INVALID_PARA; + code = TSDB_CODE_INVALID_PARA; goto _end; } @@ -819,7 +819,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd if (type != TSDB_DATA_TYPE_TIMESTAMP) { tsdbError("the first column isn't primary timestamp in result block, actual: %s, %s", tDataTypes[type].name, pReader->idStr); - terrno = TSDB_CODE_INVALID_PARA; + code = TSDB_CODE_INVALID_PARA; goto _end; } @@ -4506,7 +4506,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL _err: tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr); - tsdbReaderClose(pReader); + tsdbReaderClose(*ppReader); *ppReader = NULL; // reset the pointer value. return code; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 72e5081ab9..874caa0313 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1299,6 +1299,7 @@ static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNo int32_t code = TSDB_CODE_SUCCESS; int32_t index = 0; int32_t smaFuncIndex = -1; + bool hasWStart = false; FOREACH(pFunc, pFuncs) { smaFuncIndex = smaIndexOptFindSmaFunc(pFunc, pSmaFuncs); if (smaFuncIndex < 0) { @@ -1308,11 +1309,28 @@ static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNo if (TSDB_CODE_SUCCESS != code) { break; } + if (!hasWStart) { + SColumnNode* pTail = (SColumnNode*)pCols->pTail->pNode; + if (pTail->colId == PRIMARYKEY_TIMESTAMP_COL_ID && pTail->colType == TSDB_DATA_TYPE_TIMESTAMP) { + hasWStart = true; + } + } } ++index; } if (TSDB_CODE_SUCCESS == code && smaFuncIndex >= 0) { + if (!hasWStart) { + SExprNode exprNode; + exprNode.resType = ((SExprNode*)pSmaFuncs->pHead->pNode)->resType; + sprintf(exprNode.aliasName, "#expr_%d", index + 1); + code = nodesListMakeStrictAppend( + &pCols, smaIndexOptCreateSmaCol((SNode*)&exprNode, tableId, PRIMARYKEY_TIMESTAMP_COL_ID)); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyList(pCols); + return code; + } + } *pOutput = pCols; } else { nodesDestroyList(pCols); From c75aa54a3d279e398d0a8f79a2de3adbefcaa142 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 22 May 2023 20:33:29 +0800 Subject: [PATCH 2/5] fix: tsdb read and append _wstart for tsma --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- source/libs/planner/src/planOptimizer.c | 14 +++- .../script/tsim/sma/tsmaCreateInsertQuery.sim | 77 ++++++++++++++++++- 3 files changed, 85 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b66fc42f74..fffebcd83b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -765,7 +765,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd } if (VND_IS_TSMA(pVnode)) { - tsdbDebug("vgId:%d, tsma is selected to query, %s", TD_VID(pVnode), idstr); + tsdbInfo("vgId:%d, tsma is selected to query, %s", TD_VID(pVnode), idstr); } initReaderStatus(&pReader->status); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 874caa0313..7f3435a84d 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1311,8 +1311,13 @@ static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNo } if (!hasWStart) { SColumnNode* pTail = (SColumnNode*)pCols->pTail->pNode; - if (pTail->colId == PRIMARYKEY_TIMESTAMP_COL_ID && pTail->colType == TSDB_DATA_TYPE_TIMESTAMP) { - hasWStart = true; + if (pTail->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + if (pTail->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { + hasWStart = true; + } else { + nodesDestroyList(pCols); + return TSDB_CODE_APP_ERROR; + } } } } @@ -1324,9 +1329,10 @@ static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNo SExprNode exprNode; exprNode.resType = ((SExprNode*)pSmaFuncs->pHead->pNode)->resType; sprintf(exprNode.aliasName, "#expr_%d", index + 1); - code = nodesListMakeStrictAppend( - &pCols, smaIndexOptCreateSmaCol((SNode*)&exprNode, tableId, PRIMARYKEY_TIMESTAMP_COL_ID)); + SNode* pkNode = smaIndexOptCreateSmaCol((SNode*)&exprNode, tableId, PRIMARYKEY_TIMESTAMP_COL_ID); + code = nodesListPushFront(pCols, pkNode); if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pkNode); nodesDestroyList(pCols); return code; } diff --git a/tests/script/tsim/sma/tsmaCreateInsertQuery.sim b/tests/script/tsim/sma/tsmaCreateInsertQuery.sim index 9b6fc96bc0..242231e408 100644 --- a/tests/script/tsim/sma/tsmaCreateInsertQuery.sim +++ b/tests/script/tsim/sma/tsmaCreateInsertQuery.sim @@ -78,7 +78,7 @@ if $rows != 5 then return -1 endi -print =============== select * from stb from memory in designated vgroup +print =============== select with _wstart from stb from memory in designated vgroup sql select _wstart, _wend, min(c1),max(c2),max(c1) from stb interval(5m,10s) sliding(5m); print $data00 $data01 $data02 $data03 $data04 print $data10 $data11 $data12 $data13 $data14 @@ -102,6 +102,30 @@ if $data04 != 20 then return -1 endi +print =============== select without _wstart from stb from memory in designated vgroup +sql select min(c1),max(c2),max(c1) from stb interval(5m,10s) sliding(5m); +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows != 1 then + print rows $rows != 1 + return -1 +endi + +if $data00 != -13 then + print data00 $data00 != -13 + return -1 +endi + +if $data01 != 20.00000 then + print data01 $data01 != 20.00000 + return -1 +endi + +if $data02 != 20 then + print data02 $data02 != 20 + return -1 +endi + print =============== select * from stb from memory in common vgroups sql select _wstart, _wend, min(c1),max(c2),max(c1),max(c3) from stb interval(5m,10s) sliding(5m); print $data00 $data01 $data02 $data03 $data04 $data05 @@ -148,7 +172,7 @@ if $rows != 5 then return -1 endi -print =============== select * from stb from memory in designated vgroup after reboot +print =============== select with _wstart from stb from memory in designated vgroup after reboot sql select _wstart, _wend, min(c1),max(c2),max(c1) from stb interval(5m,10s) sliding(5m); print $data00 $data01 $data02 $data03 $data04 print $data10 $data11 $data12 $data13 $data14 @@ -172,6 +196,30 @@ if $data04 != 20 then return -1 endi +print =============== select without _wstart from stb from memory in designated vgroup after reboot +sql select min(c1),max(c2),max(c1) from stb interval(5m,10s) sliding(5m); +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows != 1 then + print rows $rows != 1 + return -1 +endi + +if $data00 != -13 then + print data00 $data00 != -13 + return -1 +endi + +if $data01 != 20.00000 then + print data01 $data01 != 20.00000 + return -1 +endi + +if $data02 != 20 then + print data02 $data02 != 20 + return -1 +endi + print =============== select * from stb from memory in common vgroups after reboot sql select _wstart, _wend, min(c1),max(c2),max(c1),max(c3) from stb interval(5m,10s) sliding(5m); print $data00 $data01 $data02 $data03 $data04 $data05 @@ -218,7 +266,7 @@ if $rows != 5 then return -1 endi -print =============== select * from stb from file in designated vgroup +print =============== select with _wstart from stb from file in designated vgroup sql select _wstart, _wend, min(c1),max(c2),max(c1) from stb interval(5m,10s) sliding(5m); print $data00 $data01 $data02 $data03 $data04 if $rows != 1 then @@ -241,6 +289,29 @@ if $data04 != 20 then return -1 endi +print =============== select without _wstart from stb from file in designated vgroup +sql select _wend, min(c1),max(c2),max(c1) from stb interval(5m,10s) sliding(5m); +print $data00 $data01 $data02 $data03 +if $rows != 1 then + print rows $rows != 1 + return -1 +endi + +if $data01 != -13 then + print data01 $data01 != -13 + return -1 +endi + +if $data02 != 20.00000 then + print data02 $data02 != 20.00000 + return -1 +endi + +if $data03 != 20 then + print data03 $data03 != 20 + return -1 +endi + print =============== select * from stb from file in common vgroups sql select _wstart, _wend, min(c1),max(c2),max(c1),max(c3) from stb interval(5m,10s) sliding(5m); print $data00 $data01 $data02 $data03 $data04 $data05 From a3bf49d22f31e763362776b63da3ceeee268d909 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 22 May 2023 20:35:52 +0800 Subject: [PATCH 3/5] chore: revert --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index fffebcd83b..b66fc42f74 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -765,7 +765,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd } if (VND_IS_TSMA(pVnode)) { - tsdbInfo("vgId:%d, tsma is selected to query, %s", TD_VID(pVnode), idstr); + tsdbDebug("vgId:%d, tsma is selected to query, %s", TD_VID(pVnode), idstr); } initReaderStatus(&pReader->status); From 329797ce233666b31ca1375245e8e6d319a5d478 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 23 May 2023 10:57:34 +0800 Subject: [PATCH 4/5] fix: add _wstart if not exists --- source/libs/planner/src/planOptimizer.c | 27 ++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index c3d46ed5e7..c9ea890574 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1292,6 +1292,16 @@ static int32_t smaIndexOptFindSmaFunc(SNode* pQueryFunc, SNodeList* pSmaFuncs) { return -1; } +static SNode* smaIndexOptFindWStartFunc(SNodeList* pSmaFuncs) { + SNode* pSmaFunc = NULL; + FOREACH(pSmaFunc, pSmaFuncs) { + if (QUERY_NODE_FUNCTION == nodeType(pSmaFunc) && FUNCTION_TYPE_WSTART == ((SFunctionNode*)pSmaFunc)->funcType) { + return pSmaFunc; + } + } + return NULL; +} + static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, SNodeList** pOutput) { SNodeList* pCols = NULL; @@ -1310,14 +1320,8 @@ static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNo break; } if (!hasWStart) { - SColumnNode* pTail = (SColumnNode*)pCols->pTail->pNode; - if (pTail->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - if (pTail->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { - hasWStart = true; - } else { - nodesDestroyList(pCols); - return TSDB_CODE_APP_ERROR; - } + if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pCols->pTail->pNode)->colId) { + hasWStart = true; } } } @@ -1326,8 +1330,13 @@ static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNo if (TSDB_CODE_SUCCESS == code && smaFuncIndex >= 0) { if (!hasWStart) { + SNode* pWsNode = smaIndexOptFindWStartFunc(pSmaFuncs); + if (!pWsNode) { + nodesDestroyList(pCols); + return TSDB_CODE_APP_ERROR; + } SExprNode exprNode; - exprNode.resType = ((SExprNode*)pSmaFuncs->pHead->pNode)->resType; + exprNode.resType = ((SExprNode*)pWsNode)->resType; sprintf(exprNode.aliasName, "#expr_%d", index + 1); SNode* pkNode = smaIndexOptCreateSmaCol((SNode*)&exprNode, tableId, PRIMARYKEY_TIMESTAMP_COL_ID); code = nodesListPushFront(pCols, pkNode); From a37d2c40fc72355d811f511a1bc5175cfe9393e2 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 23 May 2023 11:06:44 +0800 Subject: [PATCH 5/5] chore: error logs of create sma cols --- source/libs/planner/src/planOptimizer.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index c9ea890574..b6c7c3b359 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1333,7 +1333,9 @@ static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNo SNode* pWsNode = smaIndexOptFindWStartFunc(pSmaFuncs); if (!pWsNode) { nodesDestroyList(pCols); - return TSDB_CODE_APP_ERROR; + code = TSDB_CODE_APP_ERROR; + qError("create sma cols failed since %s(_wstart not exist)", tstrerror(code)); + return code; } SExprNode exprNode; exprNode.resType = ((SExprNode*)pWsNode)->resType;