diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 2e2c7d1700..6653fdd9cd 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -236,7 +236,7 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* int8_t needCompress); const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData); -void blockDebugShowData(const SArray* dataBlocks, const char* flag); +void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag); // for debug char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index e23b84c1f8..0bd917a9c6 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -24,6 +24,8 @@ extern "C" { #include "querynodes.h" #include "tname.h" +#define SLOT_NAME_LEN TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN + typedef struct SLogicNode { ENodeType type; SNodeList* pTargets; // SColumnNode @@ -74,8 +76,8 @@ typedef struct SScanLogicNode { int16_t tsColId; double filesFactor; SArray* pSmaIndexes; - SNodeList* pPartTags; - bool partSort; + SNodeList* pGroupTags; + bool groupSort; } SScanLogicNode; typedef struct SJoinLogicNode { @@ -100,6 +102,7 @@ typedef struct SProjectLogicNode { typedef struct SIndefRowsFuncLogicNode { SLogicNode node; SNodeList* pFuncs; + bool isTailFunc; } SIndefRowsFuncLogicNode; typedef struct SInterpFuncLogicNode { @@ -138,6 +141,7 @@ typedef struct SMergeLogicNode { SNodeList* pInputs; int32_t numOfChannels; int32_t srcGroupId; + bool groupSort; } SMergeLogicNode; typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType; @@ -184,6 +188,7 @@ typedef struct SFillLogicNode { typedef struct SSortLogicNode { SLogicNode node; SNodeList* pSortKeys; + bool groupSort; } SSortLogicNode; typedef struct SPartitionLogicNode { @@ -230,6 +235,7 @@ typedef struct SSlotDescNode { bool reserve; bool output; bool tag; + char name[SLOT_NAME_LEN]; } SSlotDescNode; typedef struct SDataBlockDescNode { @@ -279,7 +285,8 @@ typedef struct STableScanPhysiNode { double ratio; int32_t dataRequired; SNodeList* pDynamicScanFuncs; - SNodeList* pPartitionTags; + SNodeList* pGroupTags; + bool groupSort; int64_t interval; int64_t offset; int64_t sliding; @@ -353,6 +360,7 @@ typedef struct SMergePhysiNode { SNodeList* pTargets; int32_t numOfChannels; int32_t srcGroupId; + bool groupSort; } SMergePhysiNode; typedef struct SWinodwPhysiNode { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index a6e466e73e..39c569acef 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -259,6 +259,7 @@ typedef struct SSelectStmt { bool hasTailFunc; bool hasInterpFunc; bool hasLastRowFunc; + bool groupSort; } SSelectStmt; typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType; diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 9f89d72172..1ec298ee15 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1605,7 +1605,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { return buf; } -void blockDebugShowData(const SArray* dataBlocks, const char* flag) { +void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) { char pBuf[128] = {0}; int32_t sz = taosArrayGetSize(dataBlocks); for (int32_t i = 0; i < sz; i++) { @@ -1613,7 +1613,7 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) { size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); int32_t rows = pDataBlock->info.rows; - printf("%s |block type %d |child id %d|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId); + printf("%s |block type %d |child id %d|group id %zX\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId); for (int32_t j = 0; j < rows; j++) { printf("%s |", flag); for (int32_t k = 0; k < numOfCols; k++) { diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index bc2d5c82b1..1497bba11c 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -39,8 +39,10 @@ typedef struct { int32_t id; int32_t errCode; int32_t acceptableCode; - ETrnStage stage; + int32_t retryCode; ETrnAct actionType; + ETrnStage stage; + int8_t reserved; int8_t rawWritten; int8_t msgSent; int8_t msgReceived; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 029a1e6f8c..164bcc7d60 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -234,7 +234,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { if (retrieveReq.user[0] != 0) { memcpy(pReq->info.conn.user, retrieveReq.user, TSDB_USER_LEN); } else { - memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, TSDB_USER_LEN); + memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, strlen(TSDB_DEFAULT_USER) + 1); } if (mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db) != 0) { return -1; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index a9de1a05a7..16970e5d78 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -15,15 +15,15 @@ #define _DEFAULT_SOURCE #include "mndTrans.h" -#include "mndPrivilege.h" #include "mndConsumer.h" #include "mndDb.h" +#include "mndPrivilege.h" #include "mndShow.h" #include "mndSync.h" #include "mndUser.h" -#define TRANS_VER_NUMBER 1 -#define TRANS_ARRAY_SIZE 8 +#define TRANS_VER_NUMBER 1 +#define TRANS_ARRAY_SIZE 8 #define TRANS_RESERVE_SIZE 64 static SSdbRaw *mndTransActionEncode(STrans *pTrans); @@ -55,7 +55,7 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans); static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsMaster(pMnode); } static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans); -static int32_t mndProcessTransReq(SRpcMsg *pReq); +static int32_t mndProcessTransTimer(SRpcMsg *pReq); static int32_t mndProcessTtl(SRpcMsg *pReq); static int32_t mndProcessKillTransReq(SRpcMsg *pReq); @@ -73,7 +73,7 @@ int32_t mndInitTrans(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndTransActionDelete, }; - mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransReq); + mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransTimer); mndSetMsgHandle(pMnode, TDMT_MND_KILL_TRANS, mndProcessKillTransReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TRANS, mndRetrieveTrans); @@ -139,8 +139,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) + SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER) if (pAction->actionType == TRANS_ACTION_RAW) { int32_t len = sdbGetRawTotalSize(pAction->pRaw); SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER) @@ -163,8 +165,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) + SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER) if (pAction->actionType == TRANS_ACTION_RAW) { int32_t len = sdbGetRawTotalSize(pAction->pRaw); SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER) @@ -187,8 +191,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) + SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER) if (pAction->actionType == TRANS_ACTION_RAW) { int32_t len = sdbGetRawTotalSize(pAction->pRaw); SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER) @@ -291,10 +297,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER) SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) action.actionType = actionType; SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) action.stage = stage; + SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER) if (action.actionType == TRANS_ACTION_RAW) { SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER) SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) @@ -324,10 +332,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER) SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) action.actionType = actionType; SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) action.stage = stage; + SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER) if (action.actionType == TRANS_ACTION_RAW) { SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER) SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) @@ -357,10 +367,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER) SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) action.actionType = actionType; SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) action.stage = stage; + SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER) if (action.actionType) { SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER) SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) @@ -463,15 +475,25 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { if (fp) { (*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen); } + pTrans->startFunc = 0; } return 0; } static void mndTransDropData(STrans *pTrans) { - mndTransDropActions(pTrans->redoActions); - mndTransDropActions(pTrans->undoActions); - mndTransDropActions(pTrans->commitActions); + if (pTrans->redoActions != NULL) { + mndTransDropActions(pTrans->redoActions); + pTrans->redoActions = NULL; + } + if (pTrans->undoActions != NULL) { + mndTransDropActions(pTrans->undoActions); + pTrans->undoActions = NULL; + } + if (pTrans->commitActions != NULL) { + mndTransDropActions(pTrans->commitActions); + pTrans->commitActions = NULL; + } if (pTrans->rpcRsp != NULL) { taosMemoryFree(pTrans->rpcRsp); pTrans->rpcRsp = NULL; @@ -492,6 +514,7 @@ static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) { if (fp) { (*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen); } + pTrans->stopFunc = 0; } mndTransDropData(pTrans); @@ -805,7 +828,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { sendRsp = true; } } else { - if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 3) { + if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 6) { if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR; sendRsp = true; } @@ -875,8 +898,8 @@ int32_t mndTransProcessRsp(SRpcMsg *pRsp) { pAction->errCode = pRsp->code; } - mDebug("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x", transId, mndTransStr(pAction->stage), action, - pRsp->code, pAction->acceptableCode); + mDebug("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId, + mndTransStr(pAction->stage), action, pRsp->code, pAction->acceptableCode, pAction->retryCode); mndTransExecute(pMnode, pTrans); _OVER: @@ -884,6 +907,21 @@ _OVER: return 0; } +static void mndTransResetAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { + pAction->rawWritten = 0; + pAction->msgSent = 0; + pAction->msgReceived = 0; + if (pAction->errCode == TSDB_CODE_RPC_REDIRECT || pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR || + pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) { + pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps; + mDebug("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage), + pAction->id, pAction->epSet.inUse); + } else { + mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), pAction->id); + } + pAction->errCode = 0; +} + static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) { int32_t numOfActions = taosArrayGetSize(pArray); @@ -894,18 +932,7 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) continue; if (pAction->rawWritten && (pAction->errCode == 0 || pAction->errCode == pAction->acceptableCode)) continue; - pAction->rawWritten = 0; - pAction->msgSent = 0; - pAction->msgReceived = 0; - if (pAction->errCode == TSDB_CODE_RPC_REDIRECT || pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR || - pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) { - pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps; - mDebug("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage), - action, pAction->epSet.inUse); - } else { - mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), action); - } - pAction->errCode = 0; + mndTransResetAction(pMnode, pTrans, pAction); } } @@ -1112,9 +1139,9 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) if (pAction->msgReceived) { if (pAction->errCode != 0 && pAction->errCode != pAction->acceptableCode) { code = pAction->errCode; - pAction->msgSent = 0; - pAction->msgReceived = 0; - mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), action); + mndTransResetAction(pMnode, pTrans, pAction); + } else { + mDebug("trans:%d, %s:%d execute successfully", pTrans->id, mndTransStr(pAction->stage), action); } } else { code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -1123,6 +1150,8 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) if (pAction->rawWritten) { if (pAction->errCode != 0 && pAction->errCode != pAction->acceptableCode) { code = pAction->errCode; + } else { + mDebug("trans:%d, %s:%d write successfully", pTrans->id, mndTransStr(pAction->stage), action); } } } @@ -1156,9 +1185,15 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) { mDebug("trans:%d, %s:%d is in progress and wait it finish", pTrans->id, mndTransStr(pAction->stage), pAction->id); break; + } else if (code == pAction->retryCode) { + mDebug("trans:%d, %s:%d receive code:0x%x and retry", pTrans->id, mndTransStr(pAction->stage), pAction->id, code); + taosMsleep(300); + continue; } else { terrno = code; pTrans->code = code; + mDebug("trans:%d, %s:%d receive code:0x%x and wait another schedule, failedTimes:%d", pTrans->id, + mndTransStr(pAction->stage), pAction->id, code, pTrans->failedTimes); break; } } @@ -1343,7 +1378,7 @@ void mndTransExecute(SMnode *pMnode, STrans *pTrans) { mndTransSendRpcRsp(pMnode, pTrans); } -static int32_t mndProcessTransReq(SRpcMsg *pReq) { +static int32_t mndProcessTransTimer(SRpcMsg *pReq) { mndTransPullup(pReq->info.node); return 0; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 0e931e0a9c..0e3271a1bb 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -15,10 +15,10 @@ #define _DEFAULT_SOURCE #include "mndVgroup.h" -#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" +#include "mndPrivilege.h" #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" @@ -896,6 +896,8 @@ int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD action.pCont = pHead; action.contLen = contLen; action.msgType = TDMT_VND_ALTER_CONFIRM; + // incorrect redirect result will cause this erro + action.retryCode = TSDB_CODE_VND_INVALID_VGROUP_ID; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pHead); @@ -942,6 +944,8 @@ static int32_t mndAddSetVnodeStandByAction(SMnode *pMnode, STrans *pTrans, SDbOb action.contLen = contLen; action.msgType = TDMT_SYNC_SET_VNODE_STANDBY; action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + // Keep retrying until the target vnode is not the leader + action.retryCode = TSDB_CODE_SYN_IS_LEADER; if (isRedo) { if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -1229,7 +1233,8 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { } if (req.dnodeId1 == pVgroup->vnodeGid[0].dnodeId) { - terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; + // terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; + code = 0; goto _OVER; } @@ -1351,7 +1356,8 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { } if (pNew1 == NULL && pOld1 == NULL && pNew2 == NULL && pOld2 == NULL && pNew3 == NULL && pOld3 == NULL) { - terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; + // terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; + code = 0; goto _OVER; } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 26b9a34fa3..4ffd52f189 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -487,7 +487,7 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) #if 1 char flag[10] = {0}; snprintf(flag, 10, "level %" PRIi8, pItem->level); - blockDebugShowData(pResult, flag); + blockDebugShowDataBlocks(pResult, flag); #endif STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2); SSubmitReq *pReq = NULL; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 88bdea3ae7..72c766e0ae 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -299,7 +299,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, int64_t version, SRpcMsg *pMsg, SRp void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { // TODO - blockDebugShowData(data, __func__); + blockDebugShowDataBlocks(data, __func__); tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data); } diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h index 35bdd4ee55..972f4469e4 100644 --- a/source/libs/executor/inc/tsort.h +++ b/source/libs/executor/inc/tsort.h @@ -136,6 +136,13 @@ bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colId); */ void* tsortGetValue(STupleHandle* pVHandle, int32_t colId); +/** + * + * @param pVHandle + * @return + */ +uint64_t tsortGetGroupId(STupleHandle* pVHandle); + /** * * @param pSortHandle diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index d89e6f8874..c51ef44154 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2051,7 +2051,7 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle qDebug("no table qualified for query, TID:0x%" PRIx64 ", QID:0x%" PRIx64, taskId, queryId); return TSDB_CODE_SUCCESS; } - code = generateGroupIdMap(pTableListInfo, pHandle, pTableScanNode->pPartitionTags); + code = generateGroupIdMap(pTableListInfo, pHandle, pTableScanNode->pGroupTags); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2455,7 +2455,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN if (pInfo == NULL || pOperator == NULL) { goto _error; } - if (pTableScanNode->pPartitionTags) { + if (pTableScanNode->pGroupTags) { taosArraySort(pTableListInfo->pTableList, compareTableKeyInfoByGid); } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 09cc4e47be..96549ab6ae 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -142,7 +142,8 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i SSDataBlock* loadNextDataBlock(void* param) { SOperatorInfo* pOperator = (SOperatorInfo*)param; - return pOperator->fpSet.getNextFn(pOperator); + SSDataBlock *pBlock = pOperator->fpSet.getNextFn(pOperator); + return pBlock; } // todo refactor: merged with fetch fp @@ -505,7 +506,9 @@ typedef struct SMultiwaySortMergeOperatorInfo { SSDataBlock* pInputBlock; int64_t startTs; // sort start time - uint64_t groupId; + bool hasGroupId; + uint64_t groupId; + STupleHandle *prefetchedTuple; } SMultiwaySortMergeOperatorInfo; int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { @@ -560,12 +563,30 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData blockDataEnsureCapacity(p, capacity); while (1) { - STupleHandle* pTupleHandle = tsortNextTuple(pHandle); + STupleHandle* pTupleHandle = NULL; + if (pInfo->prefetchedTuple == NULL) { + pTupleHandle = tsortNextTuple(pHandle); + } else { + pTupleHandle = pInfo->prefetchedTuple; + pInfo->groupId = tsortGetGroupId(pTupleHandle); + pInfo->prefetchedTuple = NULL; + } + if (pTupleHandle == NULL) { break; } - appendOneRowToDataBlock(p, pTupleHandle); + uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle); + if (!pInfo->hasGroupId) { + pInfo->groupId = tupleGroupId; + pInfo->hasGroupId = true; + appendOneRowToDataBlock(p, pTupleHandle); + } else if (pInfo->groupId == tupleGroupId) { + appendOneRowToDataBlock(p, pTupleHandle); + } else { + pInfo->prefetchedTuple = pTupleHandle; + break; + } if (p->info.rows >= capacity) { break; } @@ -608,7 +629,6 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) { SSDataBlock* pBlock = getMultiwaySortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo, pOperator); - if (pBlock != NULL) { pOperator->resultInfo.totalRows += pBlock->info.rows; } else { diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 50474ff62e..7d44d41b55 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -752,6 +752,10 @@ void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) { } } +uint64_t tsortGetGroupId(STupleHandle* pVHandle) { + return pVHandle->pBlock->info.groupId; +} + SSortExecInfo tsortGetSortExecInfo(SSortHandle* pHandle) { SSortExecInfo info = {0}; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index ac197912cf..9d8baf472b 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -351,7 +351,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_SCALAR_FIELD(watermark); COPY_SCALAR_FIELD(tsColId); COPY_SCALAR_FIELD(filesFactor); - CLONE_NODE_LIST_FIELD(pPartTags); + CLONE_NODE_LIST_FIELD(pGroupTags); return TSDB_CODE_SUCCESS; } @@ -401,6 +401,7 @@ static int32_t logicMergeCopy(const SMergeLogicNode* pSrc, SMergeLogicNode* pDst CLONE_NODE_LIST_FIELD(pInputs); COPY_SCALAR_FIELD(numOfChannels); COPY_SCALAR_FIELD(srcGroupId); + COPY_SCALAR_FIELD(groupSort); return TSDB_CODE_SUCCESS; } @@ -436,6 +437,7 @@ static int32_t logicFillCopy(const SFillLogicNode* pSrc, SFillLogicNode* pDst) { static int32_t logicSortCopy(const SSortLogicNode* pSrc, SSortLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pSortKeys); + COPY_SCALAR_FIELD(groupSort); return TSDB_CODE_SUCCESS; } @@ -500,7 +502,7 @@ static int32_t physiTableScanCopy(const STableScanPhysiNode* pSrc, STableScanPhy COPY_SCALAR_FIELD(ratio); COPY_SCALAR_FIELD(dataRequired); CLONE_NODE_LIST_FIELD(pDynamicScanFuncs); - CLONE_NODE_LIST_FIELD(pPartitionTags); + CLONE_NODE_LIST_FIELD(pGroupTags); COPY_SCALAR_FIELD(interval); COPY_SCALAR_FIELD(offset); COPY_SCALAR_FIELD(sliding); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 514d65d52e..df7429bd88 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -234,6 +234,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiMerge"; case QUERY_NODE_PHYSICAL_PLAN_SORT: return "PhysiSort"; + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: + return "PhysiGroupSort"; case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: return "PhysiHashInterval"; case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: @@ -539,7 +541,7 @@ static const char* jkScanLogicPlanScanPseudoCols = "ScanPseudoCols"; static const char* jkScanLogicPlanTableId = "TableId"; static const char* jkScanLogicPlanTableType = "TableType"; static const char* jkScanLogicPlanTagCond = "TagCond"; -static const char* jkScanLogicPlanPartTags = "PartTags"; +static const char* jkScanLogicPlanGroupTags = "GroupTags"; static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { const SScanLogicNode* pNode = (const SScanLogicNode*)pObj; @@ -561,7 +563,7 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddObject(pJson, jkScanLogicPlanTagCond, nodeToJson, pNode->pTagCond); } if (TSDB_CODE_SUCCESS == code) { - code = nodeListToJson(pJson, jkScanLogicPlanPartTags, pNode->pPartTags); + code = nodeListToJson(pJson, jkScanLogicPlanGroupTags, pNode->pGroupTags); } return code; @@ -588,7 +590,7 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { code = jsonToNodeObject(pJson, jkScanLogicPlanTagCond, &pNode->pTagCond); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeList(pJson, jkScanLogicPlanPartTags, &pNode->pPartTags); + code = jsonToNodeList(pJson, jkScanLogicPlanGroupTags, &pNode->pGroupTags); } return code; @@ -1430,7 +1432,8 @@ static const char* jkTableScanPhysiPlanTriggerType = "triggerType"; static const char* jkTableScanPhysiPlanWatermark = "watermark"; static const char* jkTableScanPhysiPlanTsColId = "tsColId"; static const char* jkTableScanPhysiPlanFilesFactor = "FilesFactor"; -static const char* jkTableScanPhysiPlanPartitionTags = "PartitionTags"; +static const char* jkTableScanPhysiPlanGroupTags = "GroupTags"; +static const char* jkTableScanPhysiPlanGroupSort = "GroupSort"; static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj; @@ -1485,7 +1488,10 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddDoubleToObject(pJson, jkTableScanPhysiPlanFilesFactor, pNode->filesFactor); } if (TSDB_CODE_SUCCESS == code) { - code = nodeListToJson(pJson, jkTableScanPhysiPlanPartitionTags, pNode->pPartitionTags); + code = nodeListToJson(pJson, jkTableScanPhysiPlanGroupTags, pNode->pGroupTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanGroupSort, pNode->groupSort); } return code; @@ -1544,7 +1550,10 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { code = tjsonGetDoubleValue(pJson, jkTableScanPhysiPlanFilesFactor, &pNode->filesFactor); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeList(pJson, jkTableScanPhysiPlanPartitionTags, &pNode->pPartitionTags); + code = jsonToNodeList(pJson, jkTableScanPhysiPlanGroupTags, &pNode->pGroupTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanGroupSort, &pNode->groupSort); } return code; @@ -1725,6 +1734,7 @@ static const char* jkMergePhysiPlanMergeKeys = "MergeKeys"; static const char* jkMergePhysiPlanTargets = "Targets"; static const char* jkMergePhysiPlanNumOfChannels = "NumOfChannels"; static const char* jkMergePhysiPlanSrcGroupId = "SrcGroupId"; +static const char* jkMergePhysiPlanGroupSort = "GroupSort"; static int32_t physiMergeNodeToJson(const void* pObj, SJson* pJson) { const SMergePhysiNode* pNode = (const SMergePhysiNode*)pObj; @@ -1742,6 +1752,9 @@ static int32_t physiMergeNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkMergePhysiPlanSrcGroupId, pNode->srcGroupId); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkMergePhysiPlanGroupSort, pNode->groupSort); + } return code; } @@ -1762,6 +1775,9 @@ static int32_t jsonToPhysiMergeNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkMergePhysiPlanSrcGroupId, &pNode->srcGroupId); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkMergePhysiPlanGroupSort, &pNode->groupSort); + } return code; } @@ -3369,6 +3385,7 @@ static const char* jkSlotDescSlotId = "SlotId"; static const char* jkSlotDescDataType = "DataType"; static const char* jkSlotDescReserve = "Reserve"; static const char* jkSlotDescOutput = "Output"; +static const char* jkSlotDescName = "Name"; static int32_t slotDescNodeToJson(const void* pObj, SJson* pJson) { const SSlotDescNode* pNode = (const SSlotDescNode*)pObj; @@ -3383,6 +3400,9 @@ static int32_t slotDescNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkSlotDescOutput, pNode->output); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkSlotDescName, pNode->name); + } return code; } @@ -3400,6 +3420,9 @@ static int32_t jsonToSlotDescNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkSlotDescOutput, &pNode->output); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkSlotDescName, pNode->name); + } return code; } @@ -4137,6 +4160,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN_MERGE: return physiMergeNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_SORT: + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: return physiSortNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: @@ -4280,6 +4304,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { case QUERY_NODE_PHYSICAL_PLAN_MERGE: return jsonToPhysiMergeNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_SORT: + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: return jsonToPhysiSortNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index 1fb2db9f23..3747dde9ed 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -500,7 +500,8 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk } break; } - case QUERY_NODE_PHYSICAL_PLAN_SORT: { + case QUERY_NODE_PHYSICAL_PLAN_SORT: + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: { SSortPhysiNode* pSort = (SSortPhysiNode*)pNode; res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext); if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 734f287cfb..be6faa92cb 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -288,6 +288,8 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SMergePhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_SORT: return makeNode(type, sizeof(SSortPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: + return makeNode(type, sizeof(SGroupSortPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: return makeNode(type, sizeof(SIntervalPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: @@ -709,7 +711,7 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pLogicNode->pTagCond); nodesDestroyNode(pLogicNode->pTagIndexCond); taosArrayDestroy(pLogicNode->pSmaIndexes); - nodesDestroyList(pLogicNode->pPartTags); + nodesDestroyList(pLogicNode->pGroupTags); break; } case QUERY_NODE_LOGIC_PLAN_JOIN: { @@ -813,7 +815,7 @@ void nodesDestroyNode(SNode* pNode) { STableScanPhysiNode* pPhyNode = (STableScanPhysiNode*)pNode; destroyScanPhysiNode((SScanPhysiNode*)pNode); nodesDestroyList(pPhyNode->pDynamicScanFuncs); - nodesDestroyList(pPhyNode->pPartitionTags); + nodesDestroyList(pPhyNode->pGroupTags); break; } case QUERY_NODE_PHYSICAL_PLAN_PROJECT: { @@ -850,7 +852,8 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyList(pPhyNode->pTargets); break; } - case QUERY_NODE_PHYSICAL_PLAN_SORT: { + case QUERY_NODE_PHYSICAL_PLAN_SORT: + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: { SSortPhysiNode* pPhyNode = (SSortPhysiNode*)pNode; destroyPhysiNode((SPhysiNode*)pPhyNode); nodesDestroyList(pPhyNode->pExprs); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b68a4cc568..45240536e3 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2456,90 +2456,6 @@ static int32_t rewriteUniqueStmt(STranslateContext* pCxt, SSelectStmt* pSelect) return cxt.pTranslateCxt->errCode; } -typedef struct SRwriteTailCxt { - STranslateContext* pTranslateCxt; - int64_t limit; - int64_t offset; -} SRwriteTailCxt; - -static EDealRes rewriteTailFunc(SNode** pNode, void* pContext) { - SRwriteTailCxt* pCxt = pContext; - if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { - SFunctionNode* pFunc = (SFunctionNode*)*pNode; - if (FUNCTION_TYPE_TAIL == pFunc->funcType) { - pCxt->limit = ((SValueNode*)nodesListGetNode(pFunc->pParameterList, 1))->datum.i; - if (3 == LIST_LENGTH(pFunc->pParameterList)) { - pCxt->offset = ((SValueNode*)nodesListGetNode(pFunc->pParameterList, 2))->datum.i; - } - SNode* pExpr = nodesListGetNode(pFunc->pParameterList, 0); - strcpy(((SExprNode*)pExpr)->aliasName, ((SExprNode*)*pNode)->aliasName); - NODES_CLEAR_LIST(pFunc->pParameterList); - nodesDestroyNode(*pNode); - *pNode = pExpr; - return DEAL_RES_IGNORE_CHILD; - } - } - return DEAL_RES_CONTINUE; -} - -static int32_t createLimieNode(SRwriteTailCxt* pCxt, SLimitNode** pOutput) { - *pOutput = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT); - if (NULL == *pOutput) { - return TSDB_CODE_OUT_OF_MEMORY; - } - (*pOutput)->limit = pCxt->limit; - (*pOutput)->offset = pCxt->offset; - return TSDB_CODE_SUCCESS; -} - -static SNode* createOrderByExpr(STranslateContext* pCxt) { - SOrderByExprNode* pOrder = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); - if (NULL == pOrder) { - return NULL; - } - pCxt->errCode = createPrimaryKeyCol(pCxt, &pOrder->pExpr); - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - nodesDestroyNode((SNode*)pOrder); - return NULL; - } - pOrder->order = ORDER_DESC; - pOrder->nullOrder = NULL_ORDER_FIRST; - return (SNode*)pOrder; -} - -/* case 1: - * in: select tail(expr, k, f) from t where_clause - * out: select expr from t where_clause order by _rowts desc limit k offset f - * - * case 2: - * in: select tail(expr, k, f) from t where_clause partition_by_clause - * out: select expr from t where_clause partition_by_clause sort by _rowts desc limit k offset f - * - * case 3: - * in: select tail(expr, k, f) from t where_clause order_by_clause limit_clause - * out: select expr from ( - * select expr from t where_clause order by _rowts desc limit k offset f - * ) order_by_clause limit_clause - * - * case 4: - * in: select tail(expr, k, f) from t where_clause partition_by_clause limit_clause - * out: - */ -static int32_t rewriteTailStmt(STranslateContext* pCxt, SSelectStmt* pSelect) { - if (!pSelect->hasTailFunc) { - return TSDB_CODE_SUCCESS; - } - - SRwriteTailCxt cxt = {.pTranslateCxt = pCxt, .limit = -1, .offset = -1}; - nodesRewriteExprs(pSelect->pProjectionList, rewriteTailFunc, &cxt); - int32_t code = nodesListMakeStrictAppend(&pSelect->pOrderByList, createOrderByExpr(pCxt)); - if (TSDB_CODE_SUCCESS == code) { - code = createLimieNode(&cxt, &pSelect->pLimit); - } - pSelect->hasIndefiniteRowsFunc = false; - return code; -} - typedef struct SReplaceOrderByAliasCxt { STranslateContext* pTranslateCxt; SNodeList* pProjectionList; @@ -2616,9 +2532,6 @@ static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect if (TSDB_CODE_SUCCESS == code) { code = rewriteUniqueStmt(pCxt, pSelect); } - // if (TSDB_CODE_SUCCESS == code) { - // code = rewriteTailStmt(pCxt, pSelect); - // } if (TSDB_CODE_SUCCESS == code) { code = rewriteTimelineFunc(pCxt, pSelect); } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index a4cdcd35d3..396a7b6193 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -507,6 +507,8 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt return TSDB_CODE_OUT_OF_MEMORY; } + pIdfRowsFunc->isTailFunc = pSelect->hasTailFunc; + // indefinite rows functions and _select_values functions int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsVectorFunc, &pIdfRowsFunc->pFuncs); if (TSDB_CODE_SUCCESS == code) { @@ -733,6 +735,8 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect return TSDB_CODE_OUT_OF_MEMORY; } + pSort->groupSort = pSelect->groupSort; + int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, COLLECT_COL_TYPE_ALL, &pSort->node.pTargets); if (TSDB_CODE_SUCCESS == code && NULL == pSort->node.pTargets) { code = nodesListMakeStrictAppend(&pSort->node.pTargets, diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 8420f40ec9..9481c26d7a 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -20,9 +20,8 @@ #define OPTIMIZE_FLAG_MASK(n) (1 << n) -#define OPTIMIZE_FLAG_OSD OPTIMIZE_FLAG_MASK(0) -#define OPTIMIZE_FLAG_CPD OPTIMIZE_FLAG_MASK(1) -#define OPTIMIZE_FLAG_OPK OPTIMIZE_FLAG_MASK(2) +#define OPTIMIZE_FLAG_SCAN_PATH OPTIMIZE_FLAG_MASK(0) +#define OPTIMIZE_FLAG_PUSH_DOWN_CONDE OPTIMIZE_FLAG_MASK(1) #define OPTIMIZE_FLAG_SET_MASK(val, mask) (val) |= (mask) #define OPTIMIZE_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0) @@ -76,7 +75,7 @@ static SLogicNode* optFindPossibleNode(SLogicNode* pNode, FMayBeOptimized func) return NULL; } -EDealRes osdHaveNormalColImpl(SNode* pNode, void* pContext) { +EDealRes scanPathOptHaveNormalColImpl(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { // *((bool*)pContext) = (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType); *((bool*)pContext) = true; @@ -85,14 +84,14 @@ EDealRes osdHaveNormalColImpl(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static bool osdHaveNormalCol(SNodeList* pList) { +static bool scanPathOptHaveNormalCol(SNodeList* pList) { bool res = false; - nodesWalkExprsPostOrder(pList, osdHaveNormalColImpl, &res); + nodesWalkExprsPostOrder(pList, scanPathOptHaveNormalColImpl, &res); return res; } -static bool osdMayBeOptimized(SLogicNode* pNode) { - if (OPTIMIZE_FLAG_TEST_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_OSD)) { +static bool scanPathOptMayBeOptimized(SLogicNode* pNode) { + if (OPTIMIZE_FLAG_TEST_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_SCAN_PATH)) { return false; } if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode)) { @@ -108,10 +107,10 @@ static bool osdMayBeOptimized(SLogicNode* pNode) { QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent->pParent))) { return true; } - return !osdHaveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys); + return !scanPathOptHaveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys); } -static SNodeList* osdGetAllFuncs(SLogicNode* pNode) { +static SNodeList* scanPathOptGetAllFuncs(SLogicNode* pNode) { switch (nodeType(pNode)) { case QUERY_NODE_LOGIC_PLAN_WINDOW: return ((SWindowLogicNode*)pNode)->pFuncs; @@ -123,7 +122,7 @@ static SNodeList* osdGetAllFuncs(SLogicNode* pNode) { return NULL; } -static bool needOptimizeDataRequire(const SFunctionNode* pFunc) { +static bool scanPathOptNeedOptimizeDataRequire(const SFunctionNode* pFunc) { if (!fmIsSpecialDataRequiredFunc(pFunc->funcId)) { return false; } @@ -136,7 +135,7 @@ static bool needOptimizeDataRequire(const SFunctionNode* pFunc) { return true; } -static bool needOptimizeDynamicScan(const SFunctionNode* pFunc) { +static bool scanPathOptNeedDynOptimize(const SFunctionNode* pFunc) { if (!fmIsDynamicScanOptimizedFunc(pFunc->funcId)) { return false; } @@ -149,17 +148,17 @@ static bool needOptimizeDynamicScan(const SFunctionNode* pFunc) { return true; } -static int32_t osdGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs, SNodeList** pDsoFuncs) { - SNodeList* pAllFuncs = osdGetAllFuncs(pScan->node.pParent); +static int32_t scanPathOptGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs, SNodeList** pDsoFuncs) { + SNodeList* pAllFuncs = scanPathOptGetAllFuncs(pScan->node.pParent); SNodeList* pTmpSdrFuncs = NULL; SNodeList* pTmpDsoFuncs = NULL; SNode* pFunc = NULL; bool otherFunc = false; FOREACH(pFunc, pAllFuncs) { int32_t code = TSDB_CODE_SUCCESS; - if (needOptimizeDataRequire((SFunctionNode*)pFunc)) { + if (scanPathOptNeedOptimizeDataRequire((SFunctionNode*)pFunc)) { code = nodesListMakeStrictAppend(&pTmpSdrFuncs, nodesCloneNode(pFunc)); - } else if (needOptimizeDynamicScan((SFunctionNode*)pFunc)) { + } else if (scanPathOptNeedDynOptimize((SFunctionNode*)pFunc)) { code = nodesListMakeStrictAppend(&pTmpDsoFuncs, nodesCloneNode(pFunc)); } else { otherFunc = true; @@ -180,15 +179,15 @@ static int32_t osdGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs, return TSDB_CODE_SUCCESS; } -static int32_t osdMatch(SOptimizeContext* pCxt, SLogicNode* pLogicNode, SOsdInfo* pInfo) { - pInfo->pScan = (SScanLogicNode*)optFindPossibleNode(pLogicNode, osdMayBeOptimized); +static int32_t scanPathOptMatch(SOptimizeContext* pCxt, SLogicNode* pLogicNode, SOsdInfo* pInfo) { + pInfo->pScan = (SScanLogicNode*)optFindPossibleNode(pLogicNode, scanPathOptMayBeOptimized); if (NULL == pInfo->pScan) { return TSDB_CODE_SUCCESS; } - return osdGetRelatedFuncs(pInfo->pScan, &pInfo->pSdrFuncs, &pInfo->pDsoFuncs); + return scanPathOptGetRelatedFuncs(pInfo->pScan, &pInfo->pSdrFuncs, &pInfo->pDsoFuncs); } -static EFuncDataRequired osdPromoteDataRequired(EFuncDataRequired l, EFuncDataRequired r) { +static EFuncDataRequired scanPathOptPromoteDataRequired(EFuncDataRequired l, EFuncDataRequired r) { switch (l) { case FUNC_DATA_REQUIRED_DATA_LOAD: return l; @@ -202,19 +201,19 @@ static EFuncDataRequired osdPromoteDataRequired(EFuncDataRequired l, EFuncDataRe return r; } -static int32_t osdGetDataRequired(SNodeList* pFuncs) { +static int32_t scanPathOptGetDataRequired(SNodeList* pFuncs) { if (NULL == pFuncs) { return FUNC_DATA_REQUIRED_DATA_LOAD; } EFuncDataRequired dataRequired = FUNC_DATA_REQUIRED_FILTEROUT; SNode* pFunc = NULL; FOREACH(pFunc, pFuncs) { - dataRequired = osdPromoteDataRequired(dataRequired, fmFuncDataRequired((SFunctionNode*)pFunc, NULL)); + dataRequired = scanPathOptPromoteDataRequired(dataRequired, fmFuncDataRequired((SFunctionNode*)pFunc, NULL)); } return dataRequired; } -static void setScanWindowInfo(SScanLogicNode* pScan) { +static void scanPathOptSetScanWin(SScanLogicNode* pScan) { SLogicNode* pParent = pScan->node.pParent; if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pParent) && pParent->pParent && QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pParent->pParent)) { @@ -233,23 +232,23 @@ static void setScanWindowInfo(SScanLogicNode* pScan) { } } -static int32_t osdOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { +static int32_t scanPathOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { SOsdInfo info = {0}; - int32_t code = osdMatch(pCxt, pLogicSubplan->pNode, &info); + int32_t code = scanPathOptMatch(pCxt, pLogicSubplan->pNode, &info); if (TSDB_CODE_SUCCESS == code && info.pScan) { - setScanWindowInfo((SScanLogicNode*)info.pScan); + scanPathOptSetScanWin((SScanLogicNode*)info.pScan); } if (TSDB_CODE_SUCCESS == code && (NULL != info.pDsoFuncs || NULL != info.pSdrFuncs)) { - info.pScan->dataRequired = osdGetDataRequired(info.pSdrFuncs); + info.pScan->dataRequired = scanPathOptGetDataRequired(info.pSdrFuncs); info.pScan->pDynamicScanFuncs = info.pDsoFuncs; - OPTIMIZE_FLAG_SET_MASK(info.pScan->node.optimizedFlag, OPTIMIZE_FLAG_OSD); + OPTIMIZE_FLAG_SET_MASK(info.pScan->node.optimizedFlag, OPTIMIZE_FLAG_SCAN_PATH); pCxt->optimized = true; } nodesDestroyList(info.pSdrFuncs); return code; } -static int32_t cpdMergeCond(SNode** pDst, SNode** pSrc) { +static int32_t pushDownCondOptMergeCond(SNode** pDst, SNode** pSrc) { SLogicConditionNode* pLogicCond = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); if (NULL == pLogicCond) { return TSDB_CODE_OUT_OF_MEMORY; @@ -270,7 +269,7 @@ static int32_t cpdMergeCond(SNode** pDst, SNode** pSrc) { return code; } -static int32_t cpdCondAppend(SNode** pCond, SNode** pAdditionalCond) { +static int32_t pushDownCondOptAppendCond(SNode** pCond, SNode** pAdditionalCond) { if (NULL == *pCond) { TSWAP(*pCond, *pAdditionalCond); return TSDB_CODE_SUCCESS; @@ -283,16 +282,16 @@ static int32_t cpdCondAppend(SNode** pCond, SNode** pAdditionalCond) { *pAdditionalCond = NULL; } } else { - code = cpdMergeCond(pCond, pAdditionalCond); + code = pushDownCondOptMergeCond(pCond, pAdditionalCond); } return code; } -static int32_t cpdCalcTimeRange(SOptimizeContext* pCxt, SScanLogicNode* pScan, SNode** pPrimaryKeyCond, - SNode** pOtherCond) { +static int32_t pushDownCondOptCalcTimeRange(SOptimizeContext* pCxt, SScanLogicNode* pScan, SNode** pPrimaryKeyCond, + SNode** pOtherCond) { int32_t code = TSDB_CODE_SUCCESS; if (pCxt->pPlanCxt->topicQuery || pCxt->pPlanCxt->streamQuery) { - code = cpdCondAppend(pOtherCond, pPrimaryKeyCond); + code = pushDownCondOptAppendCond(pOtherCond, pPrimaryKeyCond); } else { bool isStrict = false; code = filterGetTimeRange(*pPrimaryKeyCond, &pScan->scanRange, &isStrict); @@ -300,7 +299,7 @@ static int32_t cpdCalcTimeRange(SOptimizeContext* pCxt, SScanLogicNode* pScan, S if (isStrict) { nodesDestroyNode(*pPrimaryKeyCond); } else { - code = cpdCondAppend(pOtherCond, pPrimaryKeyCond); + code = pushDownCondOptAppendCond(pOtherCond, pPrimaryKeyCond); } *pPrimaryKeyCond = NULL; } @@ -308,8 +307,9 @@ static int32_t cpdCalcTimeRange(SOptimizeContext* pCxt, SScanLogicNode* pScan, S return code; } -static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan) { - if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD) || +static int32_t pushDownCondOptDealScan(SOptimizeContext* pCxt, SScanLogicNode* pScan) { + if (NULL == pScan->node.pConditions || + OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE) || TSDB_SYSTEM_TABLE == pScan->tableType) { return TSDB_CODE_SUCCESS; } @@ -319,14 +319,14 @@ static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* int32_t code = nodesPartitionCond(&pScan->node.pConditions, &pPrimaryKeyCond, &pScan->pTagIndexCond, &pScan->pTagCond, &pOtherCond); if (TSDB_CODE_SUCCESS == code && NULL != pPrimaryKeyCond) { - code = cpdCalcTimeRange(pCxt, pScan, &pPrimaryKeyCond, &pOtherCond); + code = pushDownCondOptCalcTimeRange(pCxt, pScan, &pPrimaryKeyCond, &pOtherCond); } if (TSDB_CODE_SUCCESS == code) { pScan->node.pConditions = pOtherCond; } if (TSDB_CODE_SUCCESS == code) { - OPTIMIZE_FLAG_SET_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD); + OPTIMIZE_FLAG_SET_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE); pCxt->optimized = true; } else { nodesDestroyNode(pPrimaryKeyCond); @@ -336,7 +336,7 @@ static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* return code; } -static bool cpdBelongThisTable(SNode* pCondCol, SNodeList* pTableCols) { +static bool pushDownCondOptBelongThisTable(SNode* pCondCol, SNodeList* pTableCols) { SNode* pTableCol = NULL; FOREACH(pTableCol, pTableCols) { if (nodesEqualNode(pCondCol, pTableCol)) { @@ -346,12 +346,12 @@ static bool cpdBelongThisTable(SNode* pCondCol, SNodeList* pTableCols) { return false; } -static EDealRes cpdIsMultiTableCondImpl(SNode* pNode, void* pContext) { +static EDealRes pushDownCondOptIsCrossTableCond(SNode* pNode, void* pContext) { SCpdIsMultiTableCondCxt* pCxt = pContext; if (QUERY_NODE_COLUMN == nodeType(pNode)) { - if (cpdBelongThisTable(pNode, pCxt->pLeftCols)) { + if (pushDownCondOptBelongThisTable(pNode, pCxt->pLeftCols)) { pCxt->havaLeftCol = true; - } else if (cpdBelongThisTable(pNode, pCxt->pRightCols)) { + } else if (pushDownCondOptBelongThisTable(pNode, pCxt->pRightCols)) { pCxt->haveRightCol = true; } return pCxt->havaLeftCol && pCxt->haveRightCol ? DEAL_RES_END : DEAL_RES_CONTINUE; @@ -359,10 +359,11 @@ static EDealRes cpdIsMultiTableCondImpl(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static ECondAction cpdCondAction(EJoinType joinType, SNodeList* pLeftCols, SNodeList* pRightCols, SNode* pNode) { +static ECondAction pushDownCondOptGetCondAction(EJoinType joinType, SNodeList* pLeftCols, SNodeList* pRightCols, + SNode* pNode) { SCpdIsMultiTableCondCxt cxt = { .pLeftCols = pLeftCols, .pRightCols = pRightCols, .havaLeftCol = false, .haveRightCol = false}; - nodesWalkExpr(pNode, cpdIsMultiTableCondImpl, &cxt); + nodesWalkExpr(pNode, pushDownCondOptIsCrossTableCond, &cxt); return (JOIN_TYPE_INNER != joinType ? COND_ACTION_STAY : (cxt.havaLeftCol && cxt.haveRightCol @@ -370,8 +371,8 @@ static ECondAction cpdCondAction(EJoinType joinType, SNodeList* pLeftCols, SNode : (cxt.havaLeftCol ? COND_ACTION_PUSH_LEFT_CHILD : COND_ACTION_PUSH_RIGHT_CHILD))); } -static int32_t cpdPartitionLogicCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, - SNode** pRightChildCond) { +static int32_t pushDownCondOptPartLogicCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, + SNode** pRightChildCond) { SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pJoin->node.pConditions; if (LOGIC_COND_TYPE_AND != pLogicCond->condType) { return TSDB_CODE_SUCCESS; @@ -387,7 +388,7 @@ static int32_t cpdPartitionLogicCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNo SNodeList* pRemainConds = NULL; SNode* pCond = NULL; FOREACH(pCond, pLogicCond->pParameterList) { - ECondAction condAction = cpdCondAction(pJoin->joinType, pLeftCols, pRightCols, pCond); + ECondAction condAction = pushDownCondOptGetCondAction(pJoin->joinType, pLeftCols, pRightCols, pCond); if (COND_ACTION_PUSH_JOIN == condAction) { code = nodesListMakeAppend(&pOnConds, nodesCloneNode(pCond)); } else if (COND_ACTION_PUSH_LEFT_CHILD == condAction) { @@ -439,11 +440,12 @@ static int32_t cpdPartitionLogicCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNo return code; } -static int32_t cpdPartitionOpCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, - SNode** pRightChildCond) { +static int32_t pushDownCondOptPartOpCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, + SNode** pRightChildCond) { SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; - ECondAction condAction = cpdCondAction(pJoin->joinType, pLeftCols, pRightCols, pJoin->node.pConditions); + ECondAction condAction = + pushDownCondOptGetCondAction(pJoin->joinType, pLeftCols, pRightCols, pJoin->node.pConditions); if (COND_ACTION_STAY == condAction) { return TSDB_CODE_SUCCESS; } else if (COND_ACTION_PUSH_JOIN == condAction) { @@ -457,35 +459,35 @@ static int32_t cpdPartitionOpCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode* return TSDB_CODE_SUCCESS; } -static int32_t cpdPartitionCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, - SNode** pRightChildCond) { +static int32_t pushDownCondOptPartCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, + SNode** pRightChildCond) { if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->node.pConditions)) { - return cpdPartitionLogicCond(pJoin, pOnCond, pLeftChildCond, pRightChildCond); + return pushDownCondOptPartLogicCond(pJoin, pOnCond, pLeftChildCond, pRightChildCond); } else { - return cpdPartitionOpCond(pJoin, pOnCond, pLeftChildCond, pRightChildCond); + return pushDownCondOptPartOpCond(pJoin, pOnCond, pLeftChildCond, pRightChildCond); } } -static int32_t cpdPushCondToOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode** pCond) { - return cpdCondAppend(&pJoin->pOnConditions, pCond); +static int32_t pushDownCondOptPushCondToOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode** pCond) { + return pushDownCondOptAppendCond(&pJoin->pOnConditions, pCond); } -static int32_t cpdPushCondToScan(SOptimizeContext* pCxt, SScanLogicNode* pScan, SNode** pCond) { - return cpdCondAppend(&pScan->node.pConditions, pCond); +static int32_t pushDownCondOptPushCondToScan(SOptimizeContext* pCxt, SScanLogicNode* pScan, SNode** pCond) { + return pushDownCondOptAppendCond(&pScan->node.pConditions, pCond); } -static int32_t cpdPushCondToChild(SOptimizeContext* pCxt, SLogicNode* pChild, SNode** pCond) { +static int32_t pushDownCondOptPushCondToChild(SOptimizeContext* pCxt, SLogicNode* pChild, SNode** pCond) { switch (nodeType(pChild)) { case QUERY_NODE_LOGIC_PLAN_SCAN: - return cpdPushCondToScan(pCxt, (SScanLogicNode*)pChild, pCond); + return pushDownCondOptPushCondToScan(pCxt, (SScanLogicNode*)pChild, pCond); default: break; } - planError("cpdPushCondToChild failed, invalid logic plan node %s", nodesNodeName(nodeType(pChild))); + planError("pushDownCondOptPushCondToChild failed, invalid logic plan node %s", nodesNodeName(nodeType(pChild))); return TSDB_CODE_PLAN_INTERNAL_ERROR; } -static bool cpdIsPrimaryKey(SNode* pNode, SNodeList* pTableCols) { +static bool pushDownCondOptIsPriKey(SNode* pNode, SNodeList* pTableCols) { if (QUERY_NODE_COLUMN != nodeType(pNode)) { return false; } @@ -493,10 +495,10 @@ static bool cpdIsPrimaryKey(SNode* pNode, SNodeList* pTableCols) { if (PRIMARYKEY_TIMESTAMP_COL_ID != pCol->colId) { return false; } - return cpdBelongThisTable(pNode, pTableCols); + return pushDownCondOptBelongThisTable(pNode, pTableCols); } -static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { +static bool pushDownCondOptIsPriKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { if (QUERY_NODE_OPERATOR != nodeType(pCond)) { return false; } @@ -508,15 +510,15 @@ static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; - if (cpdIsPrimaryKey(pOper->pLeft, pLeftCols)) { - return cpdIsPrimaryKey(pOper->pRight, pRightCols); - } else if (cpdIsPrimaryKey(pOper->pLeft, pRightCols)) { - return cpdIsPrimaryKey(pOper->pRight, pLeftCols); + if (pushDownCondOptIsPriKey(pOper->pLeft, pLeftCols)) { + return pushDownCondOptIsPriKey(pOper->pRight, pRightCols); + } else if (pushDownCondOptIsPriKey(pOper->pLeft, pRightCols)) { + return pushDownCondOptIsPriKey(pOper->pRight, pLeftCols); } return false; } -static bool cpdContainPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { +static bool pushDownCondOptContainPriKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { if (QUERY_NODE_LOGIC_CONDITION == nodeType(pCond)) { SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pCond; if (LOGIC_COND_TYPE_AND != pLogicCond->condType) { @@ -525,54 +527,56 @@ static bool cpdContainPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { bool hasPrimaryKeyEqualCond = false; SNode* pCond = NULL; FOREACH(pCond, pLogicCond->pParameterList) { - if (cpdContainPrimaryKeyEqualCond(pJoin, pCond)) { + if (pushDownCondOptContainPriKeyEqualCond(pJoin, pCond)) { hasPrimaryKeyEqualCond = true; break; } } return hasPrimaryKeyEqualCond; } else { - return cpdIsPrimaryKeyEqualCond(pJoin, pCond); + return pushDownCondOptIsPriKeyEqualCond(pJoin, pCond); } } -static int32_t cpdCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { +static int32_t pushDownCondOptCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (NULL == pJoin->pOnConditions) { return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN); } - if (!cpdContainPrimaryKeyEqualCond(pJoin, pJoin->pOnConditions)) { + if (!pushDownCondOptContainPriKeyEqualCond(pJoin, pJoin->pOnConditions)) { return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); } return TSDB_CODE_SUCCESS; } -static int32_t cpdPushJoinCondition(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { - if (OPTIMIZE_FLAG_TEST_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_CPD)) { +static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { + if (OPTIMIZE_FLAG_TEST_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) { return TSDB_CODE_SUCCESS; } if (NULL == pJoin->node.pConditions) { - return cpdCheckJoinOnCond(pCxt, pJoin); + return pushDownCondOptCheckJoinOnCond(pCxt, pJoin); } SNode* pOnCond = NULL; SNode* pLeftChildCond = NULL; SNode* pRightChildCond = NULL; - int32_t code = cpdPartitionCond(pJoin, &pOnCond, &pLeftChildCond, &pRightChildCond); + int32_t code = pushDownCondOptPartCond(pJoin, &pOnCond, &pLeftChildCond, &pRightChildCond); if (TSDB_CODE_SUCCESS == code && NULL != pOnCond) { - code = cpdPushCondToOnCond(pCxt, pJoin, &pOnCond); + code = pushDownCondOptPushCondToOnCond(pCxt, pJoin, &pOnCond); } if (TSDB_CODE_SUCCESS == code && NULL != pLeftChildCond) { - code = cpdPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0), &pLeftChildCond); + code = + pushDownCondOptPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0), &pLeftChildCond); } if (TSDB_CODE_SUCCESS == code && NULL != pRightChildCond) { - code = cpdPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1), &pRightChildCond); + code = + pushDownCondOptPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1), &pRightChildCond); } if (TSDB_CODE_SUCCESS == code) { - OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_CPD); + OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE); pCxt->optimized = true; - code = cpdCheckJoinOnCond(pCxt, pJoin); + code = pushDownCondOptCheckJoinOnCond(pCxt, pJoin); } else { nodesDestroyNode(pOnCond); nodesDestroyNode(pLeftChildCond); @@ -582,22 +586,22 @@ static int32_t cpdPushJoinCondition(SOptimizeContext* pCxt, SJoinLogicNode* pJoi return code; } -static int32_t cpdPushAggCondition(SOptimizeContext* pCxt, SAggLogicNode* pAgg) { +static int32_t pushDownCondOptDealAgg(SOptimizeContext* pCxt, SAggLogicNode* pAgg) { // todo return TSDB_CODE_SUCCESS; } -static int32_t cpdPushCondition(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { +static int32_t pushDownCondOptimizeImpl(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pLogicNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: - code = cpdOptimizeScanCondition(pCxt, (SScanLogicNode*)pLogicNode); + code = pushDownCondOptDealScan(pCxt, (SScanLogicNode*)pLogicNode); break; case QUERY_NODE_LOGIC_PLAN_JOIN: - code = cpdPushJoinCondition(pCxt, (SJoinLogicNode*)pLogicNode); + code = pushDownCondOptDealJoin(pCxt, (SJoinLogicNode*)pLogicNode); break; case QUERY_NODE_LOGIC_PLAN_AGG: - code = cpdPushAggCondition(pCxt, (SAggLogicNode*)pLogicNode); + code = pushDownCondOptDealAgg(pCxt, (SAggLogicNode*)pLogicNode); break; default: break; @@ -605,7 +609,7 @@ static int32_t cpdPushCondition(SOptimizeContext* pCxt, SLogicNode* pLogicNode) if (TSDB_CODE_SUCCESS == code) { SNode* pChild = NULL; FOREACH(pChild, pLogicNode->pChildren) { - code = cpdPushCondition(pCxt, (SLogicNode*)pChild); + code = pushDownCondOptimizeImpl(pCxt, (SLogicNode*)pChild); if (TSDB_CODE_SUCCESS != code) { break; } @@ -614,11 +618,11 @@ static int32_t cpdPushCondition(SOptimizeContext* pCxt, SLogicNode* pLogicNode) return code; } -static int32_t cpdOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { - return cpdPushCondition(pCxt, pLogicSubplan->pNode); +static int32_t pushDownCondOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + return pushDownCondOptimizeImpl(pCxt, pLogicSubplan->pNode); } -static bool opkIsPrimaryKeyOrderBy(SNodeList* pSortKeys) { +static bool sortPriKeyOptIsPriKeyOrderBy(SNodeList* pSortKeys) { if (1 != LIST_LENGTH(pSortKeys)) { return false; } @@ -626,17 +630,18 @@ static bool opkIsPrimaryKeyOrderBy(SNodeList* pSortKeys) { return (QUERY_NODE_COLUMN == nodeType(pNode) ? (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) : false); } -static bool opkSortMayBeOptimized(SLogicNode* pNode) { +static bool sortPriKeyOptMayBeOptimized(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SORT != nodeType(pNode)) { return false; } - if (OPTIMIZE_FLAG_TEST_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_OPK)) { - return false; + SSortLogicNode* pSort = (SSortLogicNode*)pNode; + if (pSort->groupSort || !sortPriKeyOptIsPriKeyOrderBy(pSort->pSortKeys) || 1 != LIST_LENGTH(pSort->node.pChildren)) { + return TSDB_CODE_SUCCESS; } return true; } -static int32_t opkGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeList** pScanNodes) { +static int32_t sortPriKeyOptGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeList** pScanNodes) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pNode)) { @@ -646,9 +651,11 @@ static int32_t opkGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeL } break; case QUERY_NODE_LOGIC_PLAN_JOIN: - code = opkGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes); + code = + sortPriKeyOptGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes); if (TSDB_CODE_SUCCESS == code) { - code = opkGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 1), pNotOptimize, pScanNodes); + code = + sortPriKeyOptGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 1), pNotOptimize, pScanNodes); } return code; case QUERY_NODE_LOGIC_PLAN_AGG: @@ -663,77 +670,58 @@ static int32_t opkGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeL return TSDB_CODE_SUCCESS; } - return opkGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes); + return sortPriKeyOptGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes); } -static int32_t opkGetScanNodes(SLogicNode* pNode, SNodeList** pScanNodes) { +static int32_t sortPriKeyOptGetScanNodes(SLogicNode* pNode, SNodeList** pScanNodes) { bool notOptimize = false; - int32_t code = opkGetScanNodesImpl(pNode, ¬Optimize, pScanNodes); + int32_t code = sortPriKeyOptGetScanNodesImpl(pNode, ¬Optimize, pScanNodes); if (TSDB_CODE_SUCCESS != code || notOptimize) { nodesClearList(*pScanNodes); } return code; } -static EOrder opkGetPrimaryKeyOrder(SSortLogicNode* pSort) { +static EOrder sortPriKeyOptGetPriKeyOrder(SSortLogicNode* pSort) { return ((SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0))->order; } -static SNode* opkRewriteDownNode(SSortLogicNode* pSort) { - SNode* pDownNode = nodesListGetNode(pSort->node.pChildren, 0); - // todo - NODES_CLEAR_LIST(pSort->node.pChildren); - return pDownNode; -} - -static int32_t opkDoOptimized(SOptimizeContext* pCxt, SSortLogicNode* pSort, SNodeList* pScanNodes) { - EOrder order = opkGetPrimaryKeyOrder(pSort); +static int32_t sortPriKeyOptApply(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SSortLogicNode* pSort, + SNodeList* pScanNodes) { + EOrder order = sortPriKeyOptGetPriKeyOrder(pSort); if (ORDER_DESC == order) { SNode* pScan = NULL; FOREACH(pScan, pScanNodes) { TSWAP(((SScanLogicNode*)pScan)->scanSeq[0], ((SScanLogicNode*)pScan)->scanSeq[1]); } } - if (NULL == pSort->node.pParent) { - // todo - return TSDB_CODE_SUCCESS; + int32_t code = + replaceLogicNode(pLogicSubplan, (SLogicNode*)pSort, (SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0)); + if (TSDB_CODE_SUCCESS == code) { + NODES_CLEAR_LIST(pSort->node.pChildren); + nodesDestroyNode((SNode*)pSort); } - - SNode* pDownNode = opkRewriteDownNode(pSort); - SNode* pNode; - FOREACH(pNode, pSort->node.pParent->pChildren) { - if (nodesEqualNode(pNode, (SNode*)pSort)) { - REPLACE_NODE(pDownNode); - ((SLogicNode*)pDownNode)->pParent = pSort->node.pParent; - break; - } - } - nodesDestroyNode((SNode*)pSort); - return TSDB_CODE_SUCCESS; + return code; } -static int32_t opkOptimizeImpl(SOptimizeContext* pCxt, SSortLogicNode* pSort) { - OPTIMIZE_FLAG_SET_MASK(pSort->node.optimizedFlag, OPTIMIZE_FLAG_OPK); - if (!opkIsPrimaryKeyOrderBy(pSort->pSortKeys) || 1 != LIST_LENGTH(pSort->node.pChildren)) { - return TSDB_CODE_SUCCESS; - } +static int32_t sortPrimaryKeyOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SSortLogicNode* pSort) { SNodeList* pScanNodes = NULL; - int32_t code = opkGetScanNodes((SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0), &pScanNodes); + int32_t code = sortPriKeyOptGetScanNodes((SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0), &pScanNodes); if (TSDB_CODE_SUCCESS == code && NULL != pScanNodes) { - code = opkDoOptimized(pCxt, pSort, pScanNodes); + code = sortPriKeyOptApply(pCxt, pLogicSubplan, pSort, pScanNodes); } nodesClearList(pScanNodes); return code; } -static int32_t opkOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { - SSortLogicNode* pSort = (SSortLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, opkSortMayBeOptimized); +static int32_t sortPrimaryKeyOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SSortLogicNode* pSort = (SSortLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, sortPriKeyOptMayBeOptimized); if (NULL == pSort) { return TSDB_CODE_SUCCESS; } - return opkOptimizeImpl(pCxt, pSort); + return sortPrimaryKeyOptimizeImpl(pCxt, pLogicSubplan, pSort); } -static bool smaOptMayBeOptimized(SLogicNode* pNode) { +static bool smaIndexOptMayBeOptimized(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode) || NULL == pNode->pParent || QUERY_NODE_LOGIC_PLAN_WINDOW != nodeType(pNode->pParent) || WINDOW_TYPE_INTERVAL != ((SWindowLogicNode*)pNode->pParent)->winType) { @@ -748,7 +736,8 @@ static bool smaOptMayBeOptimized(SLogicNode* pNode) { return true; } -static int32_t smaOptCreateMerge(SLogicNode* pChild, SNodeList* pMergeKeys, SNodeList* pTargets, SLogicNode** pOutput) { +static int32_t smaIndexOptCreateMerge(SLogicNode* pChild, SNodeList* pMergeKeys, SNodeList* pTargets, + SLogicNode** pOutput) { SMergeLogicNode* pMerge = (SMergeLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_MERGE); if (NULL == pMerge) { return TSDB_CODE_OUT_OF_MEMORY; @@ -767,8 +756,8 @@ static int32_t smaOptCreateMerge(SLogicNode* pChild, SNodeList* pMergeKeys, SNod return TSDB_CODE_SUCCESS; } -static int32_t smaOptRecombinationNode(SLogicSubplan* pLogicSubplan, SLogicNode* pInterval, SLogicNode* pMerge, - SLogicNode* pSmaScan) { +static int32_t smaIndexOptRecombinationNode(SLogicSubplan* pLogicSubplan, SLogicNode* pInterval, SLogicNode* pMerge, + SLogicNode* pSmaScan) { int32_t code = nodesListMakeAppend(&pMerge->pChildren, (SNode*)pInterval); if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeAppend(&pMerge->pChildren, (SNode*)pSmaScan); @@ -781,8 +770,8 @@ static int32_t smaOptRecombinationNode(SLogicSubplan* pLogicSubplan, SLogicNode* return code; } -static int32_t smaOptCreateSmaScan(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList* pCols, - SLogicNode** pOutput) { +static int32_t smaIndexOptCreateSmaScan(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList* pCols, + SLogicNode** pOutput) { SScanLogicNode* pSmaScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN); if (NULL == pSmaScan) { return TSDB_CODE_OUT_OF_MEMORY; @@ -811,7 +800,7 @@ static int32_t smaOptCreateSmaScan(SScanLogicNode* pScan, STableIndexInfo* pInde return TSDB_CODE_SUCCESS; } -static bool smaOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pWindow, STableIndexInfo* pIndex) { +static bool smaIndexOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pWindow, STableIndexInfo* pIndex) { if (pWindow->interval != pIndex->interval || pWindow->intervalUnit != pIndex->intervalUnit || pWindow->offset != pIndex->offset || pWindow->sliding != pIndex->sliding || pWindow->slidingUnit != pIndex->slidingUnit) { @@ -831,7 +820,7 @@ static bool smaOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pWindow return true; } -static SNode* smaOptCreateSmaCol(SNode* pFunc, uint64_t tableId, int32_t colId) { +static SNode* smaIndexOptCreateSmaCol(SNode* pFunc, uint64_t tableId, int32_t colId) { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { return NULL; @@ -846,7 +835,7 @@ static SNode* smaOptCreateSmaCol(SNode* pFunc, uint64_t tableId, int32_t colId) return (SNode*)pCol; } -static int32_t smaOptFindSmaFunc(SNode* pQueryFunc, SNodeList* pSmaFuncs) { +static int32_t smaIndexOptFindSmaFunc(SNode* pQueryFunc, SNodeList* pSmaFuncs) { int32_t index = 0; SNode* pSmaFunc = NULL; FOREACH(pSmaFunc, pSmaFuncs) { @@ -858,8 +847,8 @@ static int32_t smaOptFindSmaFunc(SNode* pQueryFunc, SNodeList* pSmaFuncs) { return -1; } -static int32_t smaOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, SNodeList** pOutput, - int32_t* pWStrartIndex) { +static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, SNodeList** pOutput, + int32_t* pWStrartIndex) { SNodeList* pCols = NULL; SNode* pFunc = NULL; int32_t code = TSDB_CODE_SUCCESS; @@ -870,11 +859,11 @@ static int32_t smaOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeLis if (FUNCTION_TYPE_WSTARTTS == ((SFunctionNode*)pFunc)->funcType) { *pWStrartIndex = index; } - smaFuncIndex = smaOptFindSmaFunc(pFunc, pSmaFuncs); + smaFuncIndex = smaIndexOptFindSmaFunc(pFunc, pSmaFuncs); if (smaFuncIndex < 0) { break; } else { - code = nodesListMakeStrictAppend(&pCols, smaOptCreateSmaCol(pFunc, tableId, smaFuncIndex + 2)); + code = nodesListMakeStrictAppend(&pCols, smaIndexOptCreateSmaCol(pFunc, tableId, smaFuncIndex + 2)); if (TSDB_CODE_SUCCESS != code) { break; } @@ -891,22 +880,22 @@ static int32_t smaOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeLis return code; } -static int32_t smaOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList** pCols, - int32_t* pWStrartIndex) { +static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList** pCols, + int32_t* pWStrartIndex) { SWindowLogicNode* pWindow = (SWindowLogicNode*)pScan->node.pParent; - if (!smaOptEqualInterval(pScan, pWindow, pIndex)) { + if (!smaIndexOptEqualInterval(pScan, pWindow, pIndex)) { return TSDB_CODE_SUCCESS; } SNodeList* pSmaFuncs = NULL; int32_t code = nodesStringToList(pIndex->expr, &pSmaFuncs); if (TSDB_CODE_SUCCESS == code) { - code = smaOptCreateSmaCols(pWindow->pFuncs, pIndex->dstTbUid, pSmaFuncs, pCols, pWStrartIndex); + code = smaIndexOptCreateSmaCols(pWindow->pFuncs, pIndex->dstTbUid, pSmaFuncs, pCols, pWStrartIndex); } nodesDestroyList(pSmaFuncs); return code; } -static SNode* smaOptCreateWStartTs() { +static SNode* smaIndexOptCreateWStartTs() { SFunctionNode* pWStart = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); if (NULL == pWStart) { return NULL; @@ -920,7 +909,7 @@ static SNode* smaOptCreateWStartTs() { return (SNode*)pWStart; } -static int32_t smaOptCreateMergeKey(SNode* pCol, SNodeList** pMergeKeys) { +static int32_t smaIndexOptCreateMergeKey(SNode* pCol, SNodeList** pMergeKeys) { SOrderByExprNode* pMergeKey = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); if (NULL == pMergeKey) { return TSDB_CODE_OUT_OF_MEMORY; @@ -935,9 +924,9 @@ static int32_t smaOptCreateMergeKey(SNode* pCol, SNodeList** pMergeKeys) { return nodesListMakeStrictAppend(pMergeKeys, (SNode*)pMergeKey); } -static int32_t smaOptRewriteInterval(SWindowLogicNode* pInterval, int32_t wstrartIndex, SNodeList** pMergeKeys) { +static int32_t smaIndexOptRewriteInterval(SWindowLogicNode* pInterval, int32_t wstrartIndex, SNodeList** pMergeKeys) { if (wstrartIndex < 0) { - SNode* pWStart = smaOptCreateWStartTs(); + SNode* pWStart = smaIndexOptCreateWStartTs(); if (NULL == pWStart) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -948,11 +937,11 @@ static int32_t smaOptRewriteInterval(SWindowLogicNode* pInterval, int32_t wstrar } wstrartIndex = LIST_LENGTH(pInterval->node.pTargets) - 1; } - return smaOptCreateMergeKey(nodesListGetNode(pInterval->node.pTargets, wstrartIndex), pMergeKeys); + return smaIndexOptCreateMergeKey(nodesListGetNode(pInterval->node.pTargets, wstrartIndex), pMergeKeys); } -static int32_t smaOptApplyIndexExt(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, - SNodeList* pSmaCols, int32_t wstrartIndex) { +static int32_t smaIndexOptApplyIndexExt(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, + SNodeList* pSmaCols, int32_t wstrartIndex) { SWindowLogicNode* pInterval = (SWindowLogicNode*)pScan->node.pParent; SNodeList* pMergeTargets = nodesCloneList(pInterval->node.pTargets); if (NULL == pMergeTargets) { @@ -961,42 +950,42 @@ static int32_t smaOptApplyIndexExt(SLogicSubplan* pLogicSubplan, SScanLogicNode* SLogicNode* pSmaScan = NULL; SLogicNode* pMerge = NULL; SNodeList* pMergeKeys = NULL; - int32_t code = smaOptRewriteInterval(pInterval, wstrartIndex, &pMergeKeys); + int32_t code = smaIndexOptRewriteInterval(pInterval, wstrartIndex, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { - code = smaOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); + code = smaIndexOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); } if (TSDB_CODE_SUCCESS == code) { - code = smaOptCreateMerge(pScan->node.pParent, pMergeKeys, pMergeTargets, &pMerge); + code = smaIndexOptCreateMerge(pScan->node.pParent, pMergeKeys, pMergeTargets, &pMerge); } if (TSDB_CODE_SUCCESS == code) { - code = smaOptRecombinationNode(pLogicSubplan, pScan->node.pParent, pMerge, pSmaScan); + code = smaIndexOptRecombinationNode(pLogicSubplan, pScan->node.pParent, pMerge, pSmaScan); } return code; } -static int32_t smaOptApplyIndex(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, - SNodeList* pSmaCols, int32_t wstrartIndex) { +static int32_t smaIndexOptApplyIndex(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, + SNodeList* pSmaCols, int32_t wstrartIndex) { SLogicNode* pSmaScan = NULL; - int32_t code = smaOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); + int32_t code = smaIndexOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); if (TSDB_CODE_SUCCESS == code) { code = replaceLogicNode(pLogicSubplan, pScan->node.pParent, pSmaScan); } return code; } -static void smaOptDestroySmaIndex(void* p) { taosMemoryFree(((STableIndexInfo*)p)->expr); } +static void smaIndexOptDestroySmaIndex(void* p) { taosMemoryFree(((STableIndexInfo*)p)->expr); } -static int32_t smaOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan) { +static int32_t smaIndexOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan) { int32_t code = TSDB_CODE_SUCCESS; int32_t nindexes = taosArrayGetSize(pScan->pSmaIndexes); for (int32_t i = 0; i < nindexes; ++i) { STableIndexInfo* pIndex = taosArrayGet(pScan->pSmaIndexes, i); SNodeList* pSmaCols = NULL; int32_t wstrartIndex = -1; - code = smaOptCouldApplyIndex(pScan, pIndex, &pSmaCols, &wstrartIndex); + code = smaIndexOptCouldApplyIndex(pScan, pIndex, &pSmaCols, &wstrartIndex); if (TSDB_CODE_SUCCESS == code && NULL != pSmaCols) { - code = smaOptApplyIndex(pLogicSubplan, pScan, pIndex, pSmaCols, wstrartIndex); - taosArrayDestroyEx(pScan->pSmaIndexes, smaOptDestroySmaIndex); + code = smaIndexOptApplyIndex(pLogicSubplan, pScan, pIndex, pSmaCols, wstrartIndex); + taosArrayDestroyEx(pScan->pSmaIndexes, smaIndexOptDestroySmaIndex); pScan->pSmaIndexes = NULL; break; } @@ -1004,12 +993,12 @@ static int32_t smaOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubp return code; } -static int32_t smaOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { - SScanLogicNode* pScan = (SScanLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, smaOptMayBeOptimized); +static int32_t smaIndexOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SScanLogicNode* pScan = (SScanLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, smaIndexOptMayBeOptimized); if (NULL == pScan) { return TSDB_CODE_SUCCESS; } - return smaOptimizeImpl(pCxt, pLogicSubplan, pScan); + return smaIndexOptimizeImpl(pCxt, pLogicSubplan, pScan); } static EDealRes partTagsOptHasColImpl(SNode* pNode, void* pContext) { @@ -1084,7 +1073,7 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub int32_t code = TSDB_CODE_SUCCESS; SScanLogicNode* pScan = (SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0); if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { - TSWAP(((SPartitionLogicNode*)pNode)->pPartitionKeys, pScan->pPartTags); + TSWAP(((SPartitionLogicNode*)pNode)->pPartitionKeys, pScan->pGroupTags); int32_t code = replaceLogicNode(pLogicSubplan, pNode, (SLogicNode*)pScan); if (TSDB_CODE_SUCCESS == code) { NODES_CLEAR_LIST(pNode->pChildren); @@ -1094,7 +1083,7 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub SNode* pGroupKey = NULL; FOREACH(pGroupKey, ((SAggLogicNode*)pNode)->pGroupKeys) { code = nodesListMakeStrictAppend( - &pScan->pPartTags, nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pGroupKey)->pParameterList, 0))); + &pScan->pGroupTags, nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pGroupKey)->pParameterList, 0))); if (TSDB_CODE_SUCCESS != code) { break; } @@ -1102,13 +1091,11 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub NODES_DESTORY_LIST(((SAggLogicNode*)pNode)->pGroupKeys); } if (TSDB_CODE_SUCCESS == code) { - code = partTagsOptRebuildTbanme(pScan->pPartTags); + code = partTagsOptRebuildTbanme(pScan->pGroupTags); } return code; } -//===================================================================================================================== -// eliminate project optimization static bool eliminateProjOptCheckProjColumnNames(SProjectLogicNode* pProjectNode) { SHashObj* pProjColNameHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); SNode* pProjection; @@ -1142,7 +1129,7 @@ static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) { return false; } - SNode* pProjection; + SNode* pProjection; FOREACH(pProjection, pProjectNode->pProjections) { SExprNode* pExprNode = (SExprNode*)pProjection; if (QUERY_NODE_COLUMN != nodeType(pExprNode)) { @@ -1190,31 +1177,169 @@ static int32_t eliminateProjOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLog return eliminateProjOptimizeImpl(pCxt, pLogicSubplan, pProjectNode); } -//===================================================================================================================== -// eliminate Set Operator optimization +static bool rewriteTailOptMayBeOptimized(SLogicNode* pNode) { + return QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC == nodeType(pNode) && ((SIndefRowsFuncLogicNode*)pNode)->isTailFunc; +} + +static SNode* rewriteTailOptCreateOrderByExpr(SNode* pSortKey) { + SOrderByExprNode* pOrder = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); + if (NULL == pOrder) { + return NULL; + } + pOrder->order = ORDER_DESC; + pOrder->pExpr = nodesCloneNode(pSortKey); + if (NULL == pOrder->pExpr) { + nodesDestroyNode((SNode*)pOrder); + return NULL; + } + return (SNode*)pOrder; +} + +static int32_t rewriteTailOptCreateLimit(SNode* pLimit, SNode* pOffset, SNode** pOutput) { + SLimitNode* pLimitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT); + if (NULL == pLimitNode) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pLimitNode->limit = NULL == pLimit ? -1 : ((SValueNode*)pLimit)->datum.i; + pLimitNode->offset = NULL == pOffset ? -1 : ((SValueNode*)pOffset)->datum.i; + *pOutput = (SNode*)pLimitNode; + return TSDB_CODE_SUCCESS; +} + +static bool rewriteTailOptNeedGroupSort(SIndefRowsFuncLogicNode* pIndef) { + if (1 != LIST_LENGTH(pIndef->node.pChildren)) { + return false; + } + SNode* pChild = nodesListGetNode(pIndef->node.pChildren, 0); + return QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pChild) || + (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && NULL != ((SScanLogicNode*)pChild)->pGroupTags); +} + +static int32_t rewriteTailOptCreateSort(SIndefRowsFuncLogicNode* pIndef, SLogicNode** pOutput) { + SSortLogicNode* pSort = (SSortLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SORT); + if (NULL == pSort) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pSort->groupSort = rewriteTailOptNeedGroupSort(pIndef); + TSWAP(pSort->node.pChildren, pIndef->node.pChildren); + pSort->node.precision = pIndef->node.precision; + + // tail(expr, [limit, offset,] _rowts) + SFunctionNode* pTail = (SFunctionNode*)nodesListGetNode(pIndef->pFuncs, 0); + int32_t rowtsIndex = LIST_LENGTH(pTail->pParameterList) - 1; + + int32_t code = nodesListMakeStrictAppend( + &pSort->pSortKeys, rewriteTailOptCreateOrderByExpr(nodesListGetNode(pTail->pParameterList, rowtsIndex))); + if (TSDB_CODE_SUCCESS == code) { + pSort->node.pTargets = nodesCloneList(((SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0))->pTargets); + if (NULL == pSort->node.pTargets) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pOutput = (SLogicNode*)pSort; + } else { + nodesDestroyNode((SNode*)pSort); + } + + return code; +} + +static SNode* rewriteTailOptCreateProjectExpr(SFunctionNode* pTail) { + SNode* pExpr = nodesCloneNode(nodesListGetNode(pTail->pParameterList, 0)); + if (NULL == pExpr) { + return NULL; + } + strcpy(((SExprNode*)pExpr)->aliasName, pTail->node.aliasName); + return pExpr; +} + +static int32_t rewriteTailOptCreateProject(SIndefRowsFuncLogicNode* pIndef, SLogicNode** pOutput) { + SProjectLogicNode* pProject = (SProjectLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_PROJECT); + if (NULL == pProject) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + TSWAP(pProject->node.pTargets, pIndef->node.pTargets); + pProject->node.precision = pIndef->node.precision; + + // tail(expr, [limit, offset,] _rowts) + SFunctionNode* pTail = (SFunctionNode*)nodesListGetNode(pIndef->pFuncs, 0); + int32_t limitIndex = LIST_LENGTH(pTail->pParameterList) > 2 ? 1 : -1; + int32_t offsetIndex = LIST_LENGTH(pTail->pParameterList) > 3 ? 2 : -1; + + int32_t code = nodesListMakeStrictAppend(&pProject->pProjections, rewriteTailOptCreateProjectExpr(pTail)); + if (TSDB_CODE_SUCCESS == code) { + code = rewriteTailOptCreateLimit(limitIndex < 0 ? NULL : nodesListGetNode(pTail->pParameterList, limitIndex), + offsetIndex < 0 ? NULL : nodesListGetNode(pTail->pParameterList, offsetIndex), + &pProject->node.pLimit); + } + if (TSDB_CODE_SUCCESS == code) { + *pOutput = (SLogicNode*)pProject; + } else { + nodesDestroyNode((SNode*)pProject); + } + return code; +} + +static int32_t rewriteTailOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, + SIndefRowsFuncLogicNode* pIndef) { + SLogicNode* pSort = NULL; + SLogicNode* pProject = NULL; + int32_t code = rewriteTailOptCreateSort(pIndef, &pSort); + if (TSDB_CODE_SUCCESS == code) { + code = rewriteTailOptCreateProject(pIndef, &pProject); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeAppend(&pProject->pChildren, (SNode*)pSort); + pSort->pParent = pProject; + pSort = NULL; + } + if (TSDB_CODE_SUCCESS == code) { + code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pIndef, pProject); + } + if (TSDB_CODE_SUCCESS == code) { + nodesDestroyNode((SNode*)pIndef); + } else { + nodesDestroyNode((SNode*)pSort); + nodesDestroyNode((SNode*)pProject); + } + return code; +} + +static int32_t rewriteTailOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SIndefRowsFuncLogicNode* pIndef = + (SIndefRowsFuncLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, rewriteTailOptMayBeOptimized); + + if (NULL == pIndef) { + return TSDB_CODE_SUCCESS; + } + + return rewriteTailOptimizeImpl(pCxt, pLogicSubplan, pIndef); +} static bool eliminateSetOpMayBeOptimized(SLogicNode* pNode) { SLogicNode* pParent = pNode->pParent; if (NULL == pParent || - QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pParent) && QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pParent) || + QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pParent) && QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pParent) || LIST_LENGTH(pParent->pChildren) < 2) { return false; } - if (nodeType(pNode) != nodeType(pNode->pParent) || - LIST_LENGTH(pNode->pChildren) < 2) { + if (nodeType(pNode) != nodeType(pNode->pParent) || LIST_LENGTH(pNode->pChildren) < 2) { return false; } return true; } -static int32_t eliminateSetOpOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SLogicNode* pSetOpNode) { +static int32_t eliminateSetOpOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, + SLogicNode* pSetOpNode) { SNode* pSibling; FOREACH(pSibling, pSetOpNode->pParent->pChildren) { if (nodesEqualNode(pSibling, (SNode*)pSetOpNode)) { SNode* pChild; - FOREACH(pChild, pSetOpNode->pChildren) { - ((SLogicNode*)pChild)->pParent = pSetOpNode->pParent; - } + FOREACH(pChild, pSetOpNode->pChildren) { ((SLogicNode*)pChild)->pParent = pSetOpNode->pParent; } INSERT_LIST(pSetOpNode->pParent->pChildren, pSetOpNode->pChildren); pSetOpNode->pChildren = NULL; @@ -1276,16 +1401,15 @@ static int32_t mergeProjectsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLog // clang-format off static const SOptimizeRule optimizeRuleSet[] = { - {.pName = "OptimizeScanData", .optimizeFunc = osdOptimize}, - {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, - {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}, - {.pName = "SmaIndex", .optimizeFunc = smaOptimize}, + {.pName = "ScanPath", .optimizeFunc = scanPathOptimize}, + {.pName = "PushDownCondition", .optimizeFunc = pushDownCondOptimize}, + {.pName = "SortPrimaryKey", .optimizeFunc = sortPrimaryKeyOptimize}, + {.pName = "SmaIndex", .optimizeFunc = smaIndexOptimize}, {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, {.pName = "MergeProjects", .optimizeFunc = mergeProjectsOptimize}, {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize}, - {.pName = "EliminateSetOperator", .optimizeFunc = eliminateSetOpOptimize} {.pName = "EliminateSetOperator", .optimizeFunc = eliminateSetOpOptimize}, - {.pName = "RewriteTail", .optimizeFunc = rewriteTailOptimize}, + {.pName = "RewriteTail", .optimizeFunc = rewriteTailOptimize} }; // clang-format on diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 4f46abd8da..46747af3a9 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -41,8 +41,12 @@ typedef struct SPhysiPlanContext { static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { SColumnNode* pCol = (SColumnNode*)pNode; - if (NULL != pStmtName && '\0' != pStmtName[0]) { - return sprintf(pKey, "%s.%s", pStmtName, pCol->node.aliasName); + if (NULL != pStmtName) { + if ('\0' != pStmtName[0]) { + return sprintf(pKey, "%s.%s", pStmtName, pCol->node.aliasName); + } else { + return sprintf(pKey, "%s", pCol->node.aliasName); + } } if ('\0' == pCol->tableAlias[0]) { return sprintf(pKey, "%s", pCol->colName); @@ -56,11 +60,13 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) { return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName); } -static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_t slotId, bool output, bool reserve) { +static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const char* pName, const SNode* pNode, int16_t slotId, + bool output, bool reserve) { SSlotDescNode* pSlot = (SSlotDescNode*)nodesMakeNode(QUERY_NODE_SLOT_DESC); if (NULL == pSlot) { return NULL; } + strcpy(pSlot->name, pName); pSlot->slotId = slotId; pSlot->dataType = ((SExprNode*)pNode)->resType; pSlot->reserve = reserve; @@ -99,10 +105,8 @@ static int32_t putSlotToHashImpl(int16_t dataBlockId, int16_t slotId, const char return taosHashPut(pHash, pName, len, &index, sizeof(SSlotIndex)); } -static int32_t putSlotToHash(int16_t dataBlockId, int16_t slotId, SNode* pNode, SHashObj* pHash) { - char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; - int32_t len = getSlotKey(pNode, NULL, name); - return putSlotToHashImpl(dataBlockId, slotId, name, len, pHash); +static int32_t putSlotToHash(const char* pName, int16_t dataBlockId, int16_t slotId, SNode* pNode, SHashObj* pHash) { + return putSlotToHashImpl(dataBlockId, slotId, pName, strlen(pName), pHash); } static int32_t createDataBlockDescHash(SPhysiPlanContext* pCxt, int32_t capacity, int16_t dataBlockId, @@ -131,9 +135,11 @@ static int32_t buildDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SD int16_t slotId = 0; SNode* pNode = NULL; FOREACH(pNode, pList) { - code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pNode, slotId, true, false)); + char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; + getSlotKey(pNode, NULL, name); + code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, name, pNode, slotId, true, false)); if (TSDB_CODE_SUCCESS == code) { - code = putSlotToHash(pDataBlockDesc->dataBlockId, slotId, pNode, pHash); + code = putSlotToHash(name, pDataBlockDesc->dataBlockId, slotId, pNode, pHash); } if (TSDB_CODE_SUCCESS == code) { pDataBlockDesc->totalRowSize += ((SExprNode*)pNode)->resType.bytes; @@ -196,7 +202,8 @@ static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList, int32_t len = getSlotKey(pExpr, pStmtName, name); SSlotIndex* pIndex = taosHashGet(pHash, name, len); if (NULL == pIndex) { - code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pExpr, nextSlotId, output, reserve)); + code = + nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, name, pExpr, nextSlotId, output, reserve)); if (TSDB_CODE_SUCCESS == code) { code = putSlotToHashImpl(pDataBlockDesc->dataBlockId, nextSlotId, name, len, pHash); } @@ -513,12 +520,13 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName); pTableScan->dataRequired = pScanLogicNode->dataRequired; pTableScan->pDynamicScanFuncs = nodesCloneList(pScanLogicNode->pDynamicScanFuncs); - pTableScan->pPartitionTags = nodesCloneList(pScanLogicNode->pPartTags); + pTableScan->pGroupTags = nodesCloneList(pScanLogicNode->pGroupTags); if ((NULL != pScanLogicNode->pDynamicScanFuncs && NULL == pTableScan->pDynamicScanFuncs) || - (NULL != pScanLogicNode->pPartTags && NULL == pTableScan->pPartitionTags)) { + (NULL != pScanLogicNode->pGroupTags && NULL == pTableScan->pGroupTags)) { nodesDestroyNode((SNode*)pTableScan); return TSDB_CODE_OUT_OF_MEMORY; } + pTableScan->groupSort = pScanLogicNode->groupSort; pTableScan->interval = pScanLogicNode->interval; pTableScan->offset = pScanLogicNode->offset; pTableScan->sliding = pScanLogicNode->sliding; @@ -1170,8 +1178,9 @@ static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildr static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SSortLogicNode* pSortLogicNode, SPhysiNode** pPhyNode) { - SSortPhysiNode* pSort = - (SSortPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pSortLogicNode, QUERY_NODE_PHYSICAL_PLAN_SORT); + SSortPhysiNode* pSort = (SSortPhysiNode*)makePhysiNode( + pCxt, (SLogicNode*)pSortLogicNode, + pSortLogicNode->groupSort ? QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT : QUERY_NODE_PHYSICAL_PLAN_SORT); if (NULL == pSort) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1185,7 +1194,7 @@ static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) { code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pSort->pExprs); if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockSlots(pCxt, pSort->pExprs, pChildTupe); + code = pushdownDataBlockSlots(pCxt, pSort->pExprs, pChildTupe); } } @@ -1322,6 +1331,7 @@ static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SMergeLogicNode* pM pMerge->numOfChannels = pMergeLogicNode->numOfChannels; pMerge->srcGroupId = pMergeLogicNode->srcGroupId; + pMerge->groupSort = pMergeLogicNode->groupSort; int32_t code = addDataBlockSlots(pCxt, pMergeLogicNode->pInputs, pMerge->node.pOutputDataBlockDesc); diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index b4a966f206..49ab50f913 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -362,7 +362,7 @@ static int32_t stbSplGetNumOfVgroups(SLogicNode* pNode) { } static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pSplitNode, - SNodeList* pMergeKeys, SLogicNode* pPartChild) { + SNodeList* pMergeKeys, SLogicNode* pPartChild, bool groupSort) { SMergeLogicNode* pMerge = (SMergeLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_MERGE); if (NULL == pMerge) { return TSDB_CODE_OUT_OF_MEMORY; @@ -371,6 +371,7 @@ static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubpla pMerge->srcGroupId = pCxt->groupId; pMerge->node.precision = pPartChild->precision; pMerge->pMergeKeys = pMergeKeys; + pMerge->groupSort = groupSort; int32_t code = TSDB_CODE_SUCCESS; pMerge->pInputs = nodesCloneList(pPartChild->pTargets); @@ -430,7 +431,7 @@ static int32_t stbSplSplitIntervalForBatch(SSplitContext* pCxt, SStableSplitInfo SNodeList* pMergeKeys = NULL; code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pInfo->pSplitNode)->pTspk, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { - code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, pMergeKeys, pPartWindow); + code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, pMergeKeys, pPartWindow, false); } if (TSDB_CODE_SUCCESS != code) { nodesDestroyList(pMergeKeys); @@ -497,12 +498,16 @@ static int32_t stbSplSplitSessionForStream(SSplitContext* pCxt, SStableSplitInfo return code; } -static void splSetTableScanType(SLogicNode* pNode, EScanType scanType) { +static void stbSplSetTableMergeScan(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { - ((SScanLogicNode*)pNode)->scanType = scanType; + SScanLogicNode* pScan = (SScanLogicNode*)pNode; + pScan->scanType = SCAN_TYPE_TABLE_MERGE; + if (NULL != pScan->pGroupTags) { + pScan->groupSort = true; + } } else { if (1 == LIST_LENGTH(pNode->pChildren)) { - splSetTableScanType((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), scanType); + stbSplSetTableMergeScan((SLogicNode*)nodesListGetNode(pNode->pChildren, 0)); } } } @@ -515,7 +520,7 @@ static int32_t stbSplSplitSessionOrStateForBatch(SSplitContext* pCxt, SStableSpl int32_t code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pWindow)->pTspk, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { - code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pChild, pMergeKeys, (SLogicNode*)pChild); + code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pChild, pMergeKeys, (SLogicNode*)pChild, true); } if (TSDB_CODE_SUCCESS == code) { @@ -524,13 +529,10 @@ static int32_t stbSplSplitSessionOrStateForBatch(SSplitContext* pCxt, SStableSpl } if (TSDB_CODE_SUCCESS == code) { - splSetTableScanType(pChild, SCAN_TYPE_TABLE_MERGE); - ++(pCxt->groupId); - } - - if (TSDB_CODE_SUCCESS == code) { + stbSplSetTableMergeScan(pChild); pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE; SPLIT_FLAG_SET_MASK(pInfo->pSubplan->splitFlag, SPLIT_FLAG_STABLE_SPLIT); + ++(pCxt->groupId); } else { nodesDestroyList(pMergeKeys); } @@ -560,7 +562,7 @@ static int32_t stbSplSplitState(SSplitContext* pCxt, SStableSplitInfo* pInfo) { static SNodeList* stbSplGetPartKeys(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { - return ((SScanLogicNode*)pNode)->pPartTags; + return ((SScanLogicNode*)pNode)->pGroupTags; } else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { return ((SPartitionLogicNode*)pNode)->pPartitionKeys; } else { @@ -775,6 +777,7 @@ static int32_t stbSplCreatePartSortNode(SSortLogicNode* pSort, SLogicNode** pOut pPartSort->node.pChildren = pChildren; splSetParent((SLogicNode*)pPartSort); pPartSort->pSortKeys = pSortKeys; + pPartSort->groupSort = pSort->groupSort; code = stbSplCreateMergeKeys(pPartSort->pSortKeys, pPartSort->node.pTargets, &pMergeKeys); } @@ -789,12 +792,29 @@ static int32_t stbSplCreatePartSortNode(SSortLogicNode* pSort, SLogicNode** pOut return code; } +static void stbSplSetScanPartSort(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + SScanLogicNode* pScan = (SScanLogicNode*)pNode; + if (NULL != pScan->pGroupTags) { + pScan->groupSort = true; + } + } else { + if (1 == LIST_LENGTH(pNode->pChildren)) { + stbSplSetScanPartSort((SLogicNode*)nodesListGetNode(pNode->pChildren, 0)); + } + } +} + static int32_t stbSplSplitSortNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { SLogicNode* pPartSort = NULL; SNodeList* pMergeKeys = NULL; + bool groupSort = ((SSortLogicNode*)pInfo->pSplitNode)->groupSort; int32_t code = stbSplCreatePartSortNode((SSortLogicNode*)pInfo->pSplitNode, &pPartSort, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { - code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, pMergeKeys, pPartSort); + code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, pMergeKeys, pPartSort, groupSort); + } + if (TSDB_CODE_SUCCESS == code && groupSort) { + stbSplSetScanPartSort(pPartSort); } if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, @@ -829,7 +849,7 @@ static int32_t stbSplSplitScanNodeForJoin(SSplitContext* pCxt, SLogicSubplan* pS SNodeList* pMergeKeys = NULL; int32_t code = stbSplCreateMergeKeysByPrimaryKey(stbSplFindPrimaryKeyFromScan(pScan), &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { - code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, (SLogicNode*)pScan); + code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, (SLogicNode*)pScan, false); } if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&pSubplan->pChildren, diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp index ea40567dae..8c19b52a09 100644 --- a/source/libs/planner/test/planBasicTest.cpp +++ b/source/libs/planner/test/planBasicTest.cpp @@ -67,6 +67,14 @@ TEST_F(PlanBasicTest, tailFunc) { run("SELECT TAIL(c1, 10) FROM t1"); run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10"); + + run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 PARTITION BY c1"); + + run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 ORDER BY 1"); + + run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 LIMIT 5"); + + run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 PARTITION BY c1 LIMIT 5"); } TEST_F(PlanBasicTest, interpFunc) { diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index 43b6bf5772..46c2f33048 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -76,6 +76,7 @@ static void parseArg(int argc, char* argv[]) { static struct option long_options[] = { {"dump", optional_argument, NULL, 'd'}, {"skipSql", required_argument, NULL, 's'}, + {"limitSql", required_argument, NULL, 'i'}, {"log", required_argument, NULL, 'l'}, {0, 0, 0, 0} }; @@ -88,6 +89,9 @@ static void parseArg(int argc, char* argv[]) { case 's': setSkipSqlNum(optarg); break; + case 'i': + setLimitSqlNum(optarg); + break; case 'l': setLogLevel(optarg); break; diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index c197a02dd1..d54859c296 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -51,6 +51,7 @@ enum DumpModule { DumpModule g_dumpModule = DUMP_MODULE_NOTHING; int32_t g_skipSql = 0; +int32_t g_limitSql = 0; int32_t g_logLevel = 131; void setDumpModule(const char* pModule) { @@ -76,28 +77,33 @@ void setDumpModule(const char* pModule) { } void setSkipSqlNum(const char* pNum) { g_skipSql = stoi(pNum); } - +void setLimitSqlNum(const char* pNum) { g_limitSql = stoi(pNum); } void setLogLevel(const char* pLogLevel) { g_logLevel = stoi(pLogLevel); } int32_t getLogLevel() { return g_logLevel; } class PlannerTestBaseImpl { public: - PlannerTestBaseImpl() : sqlNo_(0) {} + PlannerTestBaseImpl() : sqlNo_(0), sqlNum_(0) {} void useDb(const string& user, const string& db) { caseEnv_.acctId_ = 0; caseEnv_.user_ = user; caseEnv_.db_ = db; - caseEnv_.nsql_ = g_skipSql; + caseEnv_.numOfSkipSql_ = g_skipSql; + caseEnv_.numOfLimitSql_ = g_limitSql; } void run(const string& sql) { ++sqlNo_; - if (caseEnv_.nsql_ > 0) { - --(caseEnv_.nsql_); + if (caseEnv_.numOfSkipSql_ > 0) { + --(caseEnv_.numOfSkipSql_); return; } + if (caseEnv_.numOfLimitSql_ > 0 && caseEnv_.numOfLimitSql_ == sqlNum_) { + return; + } + ++sqlNum_; reset(); try { @@ -134,7 +140,7 @@ class PlannerTestBaseImpl { } void prepare(const string& sql) { - if (caseEnv_.nsql_ > 0) { + if (caseEnv_.numOfSkipSql_ > 0) { return; } @@ -148,7 +154,7 @@ class PlannerTestBaseImpl { } void bindParams(TAOS_MULTI_BIND* pParams, int32_t colIdx) { - if (caseEnv_.nsql_ > 0) { + if (caseEnv_.numOfSkipSql_ > 0) { return; } @@ -161,8 +167,8 @@ class PlannerTestBaseImpl { } void exec() { - if (caseEnv_.nsql_ > 0) { - --(caseEnv_.nsql_); + if (caseEnv_.numOfSkipSql_ > 0) { + --(caseEnv_.numOfSkipSql_); return; } @@ -197,9 +203,10 @@ class PlannerTestBaseImpl { int32_t acctId_; string user_; string db_; - int32_t nsql_; + int32_t numOfSkipSql_; + int32_t numOfLimitSql_; - caseEnv() : nsql_(0) {} + caseEnv() : numOfSkipSql_(0) {} }; struct stmtEnv { @@ -401,6 +408,7 @@ class PlannerTestBaseImpl { stmtEnv stmtEnv_; stmtRes res_; int32_t sqlNo_; + int32_t sqlNum_; }; PlannerTestBase::PlannerTestBase() : impl_(new PlannerTestBaseImpl()) {} diff --git a/source/libs/planner/test/planTestUtil.h b/source/libs/planner/test/planTestUtil.h index 7d2a9e533f..f9942c93a7 100644 --- a/source/libs/planner/test/planTestUtil.h +++ b/source/libs/planner/test/planTestUtil.h @@ -43,6 +43,7 @@ class PlannerTestBase : public testing::Test { extern void setDumpModule(const char* pModule); extern void setSkipSqlNum(const char* pNum); +extern void setLimitSqlNum(const char* pNum); extern void setLogLevel(const char* pLogLevel); extern int32_t getLogLevel(); diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index b130186516..37f18a6388 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -56,6 +56,31 @@ void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj); void syncEntryLog(const SSyncRaftEntry* pObj); void syncEntryLog2(char* s, const SSyncRaftEntry* pObj); +//----------------------------------- +typedef struct SRaftEntryCache { + SHashObj* pEntryHash; + int32_t maxCount; + int32_t currentCount; + TdThreadMutex mutex; + SSyncNode* pSyncNode; +} SRaftEntryCache; + +SRaftEntryCache* raftCacheCreate(SSyncNode* pSyncNode, int32_t maxCount); +void raftCacheDestroy(SRaftEntryCache* pCache); +int32_t raftCachePutEntry(struct SRaftEntryCache* pCache, SSyncRaftEntry* pEntry); +int32_t raftCacheGetEntry(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); +int32_t raftCacheGetEntryP(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); +int32_t raftCacheDelEntry(struct SRaftEntryCache* pCache, SyncIndex index); +int32_t raftCacheGetAndDel(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); +int32_t raftCacheClear(struct SRaftEntryCache* pCache); + +cJSON* raftCache2Json(SRaftEntryCache* pObj); +char* raftCache2Str(SRaftEntryCache* pObj); +void raftCachePrint(SRaftEntryCache* pObj); +void raftCachePrint2(char* s, SRaftEntryCache* pObj); +void raftCacheLog(SRaftEntryCache* pObj); +void raftCacheLog2(char* s, SRaftEntryCache* pObj); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 3996193037..60f6ca7e7b 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1355,11 +1355,16 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { int32_t userStrLen = strlen(str); SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; - if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { + if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); } - SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); - SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); + + SyncIndex logLastIndex = SYNC_INDEX_INVALID; + SyncIndex logBeginIndex = SYNC_INDEX_INVALID; + if (pSyncNode->pLogStore != NULL) { + logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); + logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); + } char* pCfgStr = syncCfg2SimpleStr(&(pSyncNode->pRaftCfg->cfg)); char* printStr = ""; diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index ff334a76bb..225360630c 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -180,3 +180,247 @@ void syncEntryLog2(char* s, const SSyncRaftEntry* pObj) { sTrace("syncEntryLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } + +//----------------------------------- +SRaftEntryCache* raftCacheCreate(SSyncNode* pSyncNode, int32_t maxCount) { + SRaftEntryCache* pCache = taosMemoryMalloc(sizeof(SRaftEntryCache)); + if (pCache == NULL) { + sError("vgId:%d raft cache create error", pSyncNode->vgId); + return NULL; + } + + pCache->pEntryHash = + taosHashInit(sizeof(SyncIndex), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (pCache->pEntryHash == NULL) { + sError("vgId:%d raft cache create hash error", pSyncNode->vgId); + return NULL; + } + + taosThreadMutexInit(&(pCache->mutex), NULL); + pCache->maxCount = maxCount; + pCache->currentCount = 0; + pCache->pSyncNode = pSyncNode; + + return pCache; +} + +void raftCacheDestroy(SRaftEntryCache* pCache) { + if (pCache != NULL) { + taosThreadMutexLock(&(pCache->mutex)); + taosHashCleanup(pCache->pEntryHash); + taosThreadMutexUnlock(&(pCache->mutex)); + taosThreadMutexDestroy(&(pCache->mutex)); + taosMemoryFree(pCache); + } +} + +// success, return 1 +// max count, return 0 +// error, return -1 +int32_t raftCachePutEntry(struct SRaftEntryCache* pCache, SSyncRaftEntry* pEntry) { + taosThreadMutexLock(&(pCache->mutex)); + + if (pCache->currentCount >= pCache->maxCount) { + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; + } + + taosHashPut(pCache->pEntryHash, &(pEntry->index), sizeof(pEntry->index), pEntry, pEntry->bytes); + ++(pCache->currentCount); + + do { + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "raft cache add, type:%s,%d, type2:%s,%d, index:%ld, bytes:%d", + TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType, + pEntry->index, pEntry->bytes); + syncNodeEventLog(pCache->pSyncNode, eventLog); + } while (0); + + taosThreadMutexUnlock(&(pCache->mutex)); + return 1; +} + +// success, return 0 +// error, return -1 +// not exist, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST +int32_t raftCacheGetEntry(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) { + if (ppEntry == NULL) { + return -1; + } + *ppEntry = NULL; + + taosThreadMutexLock(&(pCache->mutex)); + void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index)); + if (pTmp != NULL) { + SSyncRaftEntry* pEntry = pTmp; + *ppEntry = taosMemoryMalloc(pEntry->bytes); + memcpy(*ppEntry, pTmp, pEntry->bytes); + + do { + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "raft cache get, type:%s,%d, type2:%s,%d, index:%ld", + TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType), + (*ppEntry)->originalRpcType, (*ppEntry)->index); + syncNodeEventLog(pCache->pSyncNode, eventLog); + } while (0); + + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; + } + + taosThreadMutexUnlock(&(pCache->mutex)); + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + return -1; +} + +// success, return 0 +// error, return -1 +// not exist, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST +int32_t raftCacheGetEntryP(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) { + if (ppEntry == NULL) { + return -1; + } + *ppEntry = NULL; + + taosThreadMutexLock(&(pCache->mutex)); + void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index)); + if (pTmp != NULL) { + SSyncRaftEntry* pEntry = pTmp; + *ppEntry = pEntry; + + do { + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "raft cache get, type:%s,%d, type2:%s,%d, index:%ld", + TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType), + (*ppEntry)->originalRpcType, (*ppEntry)->index); + syncNodeEventLog(pCache->pSyncNode, eventLog); + } while (0); + + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; + } + + taosThreadMutexUnlock(&(pCache->mutex)); + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + return -1; +} + +int32_t raftCacheDelEntry(struct SRaftEntryCache* pCache, SyncIndex index) { + taosThreadMutexLock(&(pCache->mutex)); + taosHashRemove(pCache->pEntryHash, &index, sizeof(index)); + --(pCache->currentCount); + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; +} + +int32_t raftCacheGetAndDel(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) { + if (ppEntry == NULL) { + return -1; + } + *ppEntry = NULL; + + taosThreadMutexLock(&(pCache->mutex)); + void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index)); + if (pTmp != NULL) { + SSyncRaftEntry* pEntry = pTmp; + *ppEntry = taosMemoryMalloc(pEntry->bytes); + memcpy(*ppEntry, pTmp, pEntry->bytes); + + do { + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "raft cache get-and-del, type:%s,%d, type2:%s,%d, index:%ld", + TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType), + (*ppEntry)->originalRpcType, (*ppEntry)->index); + syncNodeEventLog(pCache->pSyncNode, eventLog); + } while (0); + + taosHashRemove(pCache->pEntryHash, &index, sizeof(index)); + --(pCache->currentCount); + + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; + } + + taosThreadMutexUnlock(&(pCache->mutex)); + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + return -1; +} + +int32_t raftCacheClear(struct SRaftEntryCache* pCache) { + taosThreadMutexLock(&(pCache->mutex)); + taosHashClear(pCache->pEntryHash); + pCache->currentCount = 0; + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; +} + +//----------------------------------- +cJSON* raftCache2Json(SRaftEntryCache* pCache) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pCache != NULL) { + taosThreadMutexLock(&(pCache->mutex)); + + snprintf(u64buf, sizeof(u64buf), "%p", pCache->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + cJSON_AddNumberToObject(pRoot, "currentCount", pCache->currentCount); + cJSON_AddNumberToObject(pRoot, "maxCount", pCache->maxCount); + cJSON* pEntries = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "entries", pEntries); + + SSyncRaftEntry* pIter = (SSyncRaftEntry*)taosHashIterate(pCache->pEntryHash, NULL); + if (pIter != NULL) { + SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter; + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + } + while (pIter) { + pIter = taosHashIterate(pCache->pEntryHash, pIter); + if (pIter != NULL) { + SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter; + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + } + } + + taosThreadMutexUnlock(&(pCache->mutex)); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SRaftEntryCache", pRoot); + return pJson; +} + +char* raftCache2Str(SRaftEntryCache* pCache) { + cJSON* pJson = raftCache2Json(pCache); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +void raftCachePrint(SRaftEntryCache* pCache) { + char* serialized = raftCache2Str(pCache); + printf("raftCachePrint | len:%lu | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftCachePrint2(char* s, SRaftEntryCache* pCache) { + char* serialized = raftCache2Str(pCache); + printf("raftCachePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftCacheLog(SRaftEntryCache* pCache) { + char* serialized = raftCache2Str(pCache); + sTrace("raftCacheLog | len:%lu | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void raftCacheLog2(char* s, SRaftEntryCache* pCache) { + if (gRaftDetailLog) { + char* serialized = raftCache2Str(pCache); + sTraceLong("raftCacheLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} \ No newline at end of file diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index 48c1b70a04..d2cbabe226 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -32,11 +32,13 @@ SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) { } void syncRespMgrDestroy(SSyncRespMgr *pObj) { - taosThreadMutexLock(&(pObj->mutex)); - taosHashCleanup(pObj->pRespHash); - taosThreadMutexUnlock(&(pObj->mutex)); - taosThreadMutexDestroy(&(pObj->mutex)); - taosMemoryFree(pObj); + if (pObj != NULL) { + taosThreadMutexLock(&(pObj->mutex)); + taosHashCleanup(pObj->pRespHash); + taosThreadMutexUnlock(&(pObj->mutex)); + taosThreadMutexDestroy(&(pObj->mutex)); + taosMemoryFree(pObj); + } } int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub) { diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 27084286da..bc63462a9e 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -17,6 +17,7 @@ add_executable(syncVotesRespondTest "") add_executable(syncIndexMgrTest "") add_executable(syncLogStoreTest "") add_executable(syncEntryTest "") +add_executable(syncEntryCacheTest "") add_executable(syncRequestVoteTest "") add_executable(syncRequestVoteReplyTest "") add_executable(syncAppendEntriesTest "") @@ -129,6 +130,10 @@ target_sources(syncEntryTest PRIVATE "syncEntryTest.cpp" ) +target_sources(syncEntryCacheTest + PRIVATE + "syncEntryCacheTest.cpp" +) target_sources(syncRequestVoteTest PRIVATE "syncRequestVoteTest.cpp" @@ -362,6 +367,11 @@ target_include_directories(syncEntryTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncEntryCacheTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_include_directories(syncRequestVoteTest PUBLIC "${TD_SOURCE_DIR}/include/libs/sync" @@ -610,6 +620,10 @@ target_link_libraries(syncEntryTest sync gtest_main ) +target_link_libraries(syncEntryCacheTest + sync + gtest_main +) target_link_libraries(syncRequestVoteTest sync gtest_main diff --git a/source/libs/sync/test/syncEntryCacheTest.cpp b/source/libs/sync/test/syncEntryCacheTest.cpp new file mode 100644 index 0000000000..787c08e507 --- /dev/null +++ b/source/libs/sync/test/syncEntryCacheTest.cpp @@ -0,0 +1,162 @@ +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SSyncRaftEntry* createEntry(int i) { + int32_t dataLen = 20; + SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); + assert(pEntry != NULL); + pEntry->msgType = 88; + pEntry->originalRpcType = 99; + pEntry->seqNum = 3; + pEntry->isWeak = true; + pEntry->term = 100 + i; + pEntry->index = i; + snprintf(pEntry->data, dataLen, "value%d", i); + + return pEntry; +} + +SSyncNode* createFakeNode() { + SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); + ASSERT(pSyncNode != NULL); + memset(pSyncNode, 0, sizeof(SSyncNode)); + + return pSyncNode; +} + +SRaftEntryCache* createCache(int maxCount) { + SSyncNode* pSyncNode = createFakeNode(); + ASSERT(pSyncNode != NULL); + + SRaftEntryCache* pCache = raftCacheCreate(pSyncNode, maxCount); + ASSERT(pCache != NULL); + + return pCache; +} + +void test1() { + int32_t code = 0; + SRaftEntryCache* pCache = createCache(5); + for (int i = 0; i < 5; ++i) { + SSyncRaftEntry* pEntry = createEntry(i); + code = raftCachePutEntry(pCache, pEntry); + ASSERT(code == 1); + syncEntryDestory(pEntry); + } + raftCacheLog2((char*)"==test1 write 5 entries==", pCache); + + SyncIndex index; + index = 1; + code = raftCacheDelEntry(pCache, index); + ASSERT(code == 0); + index = 3; + code = raftCacheDelEntry(pCache, index); + ASSERT(code == 0); + raftCacheLog2((char*)"==test1 delete 1,3==", pCache); + + code = raftCacheClear(pCache); + ASSERT(code == 0); + raftCacheLog2((char*)"==clear all==", pCache); +} + +void test2() { + int32_t code = 0; + SRaftEntryCache* pCache = createCache(5); + for (int i = 0; i < 5; ++i) { + SSyncRaftEntry* pEntry = createEntry(i); + code = raftCachePutEntry(pCache, pEntry); + ASSERT(code == 1); + syncEntryDestory(pEntry); + } + raftCacheLog2((char*)"==test2 write 5 entries==", pCache); + + SyncIndex index; + index = 1; + SSyncRaftEntry* pEntry; + code = raftCacheGetEntry(pCache, index, &pEntry); + ASSERT(code == 0); + syncEntryDestory(pEntry); + syncEntryLog2((char*)"==test2 get entry 1==", pEntry); + + index = 2; + code = raftCacheGetEntryP(pCache, index, &pEntry); + ASSERT(code == 0); + syncEntryLog2((char*)"==test2 get entry pointer 2==", pEntry); + + // not found + index = 8; + code = raftCacheGetEntry(pCache, index, &pEntry); + ASSERT(code == -1 && terrno == TSDB_CODE_WAL_LOG_NOT_EXIST); + sTrace("==test2 get entry 8 not found=="); + + // not found + index = 9; + code = raftCacheGetEntryP(pCache, index, &pEntry); + ASSERT(code == -1 && terrno == TSDB_CODE_WAL_LOG_NOT_EXIST); + sTrace("==test2 get entry pointer 9 not found=="); +} + +void test3() { + int32_t code = 0; + SRaftEntryCache* pCache = createCache(5); + for (int i = 0; i < 5; ++i) { + SSyncRaftEntry* pEntry = createEntry(i); + code = raftCachePutEntry(pCache, pEntry); + ASSERT(code == 1); + syncEntryDestory(pEntry); + } + for (int i = 6; i < 10; ++i) { + SSyncRaftEntry* pEntry = createEntry(i); + code = raftCachePutEntry(pCache, pEntry); + ASSERT(code == 0); + syncEntryDestory(pEntry); + } + raftCacheLog2((char*)"==test3 write 10 entries, max count is 5==", pCache); +} + +void test4() { + int32_t code = 0; + SRaftEntryCache* pCache = createCache(5); + for (int i = 0; i < 5; ++i) { + SSyncRaftEntry* pEntry = createEntry(i); + code = raftCachePutEntry(pCache, pEntry); + ASSERT(code == 1); + syncEntryDestory(pEntry); + } + raftCacheLog2((char*)"==test4 write 5 entries==", pCache); + + SyncIndex index; + index = 3; + SSyncRaftEntry* pEntry; + code = raftCacheGetAndDel(pCache, index, &pEntry); + ASSERT(code == 0); + syncEntryLog2((char*)"==test4 get-and-del entry 3==", pEntry); + raftCacheLog2((char*)"==test4 after get-and-del entry 3==", pCache); +} + +int main(int argc, char** argv) { + gRaftDetailLog = true; + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE + DEBUG_DEBUG; + + test1(); + test2(); + test3(); + test4(); + + return 0; +} diff --git a/tests/pytest/util/constant.py b/tests/pytest/util/constant.py index e5095c74dc..807398f70f 100644 --- a/tests/pytest/util/constant.py +++ b/tests/pytest/util/constant.py @@ -1,5 +1,76 @@ # -*- coding: utf-8 -*- +# basic type +TAOS_DATA_TYPE = [ + "INT", "BIGINT", "SMALLINT", "TINYINT", "INT UNSIGNED", "BIGINT UNSIGNED", "SMALLINT UNSIGNED", "TINYINT UNSIGNED", + "FLOAT", "DOUBLE", + "BOOL", + "BINARY", "NCHAR", "VARCHAR", + "TIMESTAMP", + # "MEDIUMBLOB", "BLOB", # add in 3.x + # "DECIMAL", "NUMERIC", # add in 3.x + "JSON", # only for tag +] + +TAOS_NUM_TYPE = [ + "INT", "BIGINT", "SMALLINT", "TINYINT", "INT UNSIGNED", "BIGINT UNSIGNED", "SMALLINT UNSIGNED", "TINYINT UNSIGNED", "FLOAT", "DOUBLE", + # "DECIMAL", "NUMERIC", # add in 3.x +] +TAOS_CHAR_TYPE = [ + "BINARY", "NCHAR", "VARCHAR", +] +TAOS_BOOL_TYPE = ["BOOL",] +TAOS_TS_TYPE = ["TIMESTAMP",] +TAOS_BIN_TYPE = [ + "MEDIUMBLOB", "BLOB", # add in 3.x +] + +TAOS_TIME_INIT = ["b", "u", "a", "s", "m", "h", "d", "w", "n", "y"] +TAOS_PRECISION = ["ms", "us", "ns"] +PRECISION_DEFAULT = "ms" +PRECISION = PRECISION_DEFAULT + +TAOS_KEYWORDS = [ + "ABORT", "CREATE", "IGNORE", "NULL", "STAR", + "ACCOUNT", "CTIME", "IMMEDIATE", "OF", "STATE", + "ACCOUNTS", "DATABASE", "IMPORT", "OFFSET", "STATEMENT", + "ADD", "DATABASES", "IN", "OR", "STATE_WINDOW", + "AFTER", "DAYS", "INITIALLY", "ORDER", "STORAGE", + "ALL", "DBS", "INSERT", "PARTITIONS", "STREAM", + "ALTER", "DEFERRED", "INSTEAD", "PASS", "STREAMS", + "AND", "DELIMITERS", "INT", "PLUS", "STRING", + "AS", "DESC", "INTEGER", "PPS", "SYNCDB", + "ASC", "DESCRIBE", "INTERVAL", "PRECISION", "TABLE", + "ATTACH", "DETACH", "INTO", "PREV", "TABLES", + "BEFORE", "DISTINCT", "IS", "PRIVILEGE", "TAG", + "BEGIN", "DIVIDE", "ISNULL", "QTIME", "TAGS", + "BETWEEN", "DNODE", "JOIN", "QUERIES", "TBNAME", + "BIGINT", "DNODES", "KEEP", "QUERY", "TIMES", + "BINARY", "DOT", "KEY", "QUORUM", "TIMESTAMP", + "BITAND", "DOUBLE", "KILL", "RAISE", "TINYINT", + "BITNOT", "DROP", "LE", "REM", "TOPIC", + "BITOR", "EACH", "LIKE", "REPLACE", "TOPICS", + "BLOCKS", "END", "LIMIT", "REPLICA", "TRIGGER", + "BOOL", "EQ", "LINEAR", "RESET", "TSERIES", + "BY", "EXISTS", "LOCAL", "RESTRICT", "UMINUS", + "CACHE", "EXPLAIN", "LP", "ROW", "UNION", + "CACHELAST", "FAIL", "LSHIFT", "RP", "UNSIGNED", + "CASCADE", "FILE", "LT", "RSHIFT", "UPDATE", + "CHANGE", "FILL", "MATCH", "SCORES", "UPLUS", + "CLUSTER", "FLOAT", "MAXROWS", "SELECT", "USE", + "COLON", "FOR", "MINROWS", "SEMI", "USER", + "COLUMN", "FROM", "MINUS", "SESSION", "USERS", + "COMMA", "FSYNC", "MNODES", "SET", "USING", + "COMP", "GE", "MODIFY", "SHOW", "VALUES", + "COMPACT", "GLOB", "MODULES", "SLASH", "VARIABLE", + "CONCAT", "GRANTS", "NCHAR", "SLIDING", "VARIABLES", + "CONFLICT", "GROUP", "NE", "SLIMIT", "VGROUPS", + "CONNECTION", "GT", "NONE", "SMALLINT", "VIEW", + "CONNECTIONS", "HAVING", "NOT", "SOFFSET", "VNODES", + "CONNS", "ID", "NOTNULL", "STABLE", "WAL", + "COPY", "IF", "NOW", "STABLES", "WHERE", +] + # basic data type boundary TINYINT_MAX = 127 TINYINT_MIN = -128 @@ -11,7 +82,7 @@ SMALLINT_MAX = 32767 SMALLINT_MIN = -32768 SMALLINT_UN_MAX = 65535 -MALLINT_UN_MIN = 0 +SMALLINT_UN_MIN = 0 INT_MAX = 2147483647 INT_MIN = -2147483648 @@ -33,8 +104,8 @@ DOUBLE_MIN = -1.7E+308 # schema boundary BINARY_LENGTH_MAX = 16374 -NCAHR_LENGTH_MAX_ = 4093 -DBNAME_LENGTH_MAX_ = 64 +NCAHR_LENGTH_MAX = 4093 +DBNAME_LENGTH_MAX = 64 STBNAME_LENGTH_MAX = 192 STBNAME_LENGTH_MIN = 1 @@ -66,4 +137,32 @@ MNODE_SHM_SIZE_DEFAULT = 6292480 VNODE_SHM_SIZE_MAX = 2147483647 VNODE_SHM_SIZE_MIN = 6292480 -VNODE_SHM_SIZE_DEFAULT = 31458304 \ No newline at end of file +VNODE_SHM_SIZE_DEFAULT = 31458304 + +# time_init +TIME_MS = 1 +TIME_US = TIME_MS/1000 +TIME_NS = TIME_US/1000 + +TIME_S = 1000 * TIME_MS +TIME_M = 60 * TIME_S +TIME_H = 60 * TIME_M +TIME_D = 24 * TIME_H +TIME_W = 7 * TIME_D +TIME_N = 30 * TIME_D +TIME_Y = 365 * TIME_D + + +# session parameters +INTERVAL_MIN = 1 * TIME_MS if PRECISION == PRECISION_DEFAULT else 1 * TIME_US + + +# streams and related agg-function +SMA_INDEX_FUNCTIONS = ["MIN", "MAX"] +ROLLUP_FUNCTIONS = ["AVG", "SUM", "MIN", "MAX", "LAST", "FIRST"] +SMA_WATMARK_MAXDELAY_INIT = ['a', "s", "m"] +WATERMARK_MAX = 900000 +WATERMARK_MIN = 0 + +MAX_DELAY_MAX = 900000 +MAX_DELAY_MIN = 1 \ No newline at end of file diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 580fc8ee47..fe802dd9a3 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -21,6 +21,7 @@ import psutil import shutil import pandas as pd from util.log import * +from util.constant import * def _parse_datetime(timestr): try: @@ -117,8 +118,7 @@ class TDSql: col_name_list = [] col_type_list = [] self.cursor.execute(sql) - self.queryCols = self.cursor.description - for query_col in self.queryCols: + for query_col in self.cursor.description: col_name_list.append(query_col[0]) col_type_list.append(query_col[1]) except Exception as e: @@ -301,6 +301,41 @@ class TDSql: args = (caller.filename, caller.lineno, self.sql, elm, expect_elm) tdLog.exit("%s(%d) failed: sql:%s, elm:%s == expect_elm:%s" % args) + def get_times(self, time_str, precision="ms"): + caller = inspect.getframeinfo(inspect.stack()[1][0]) + if time_str[-1] not in TAOS_TIME_INIT: + tdLog.exit(f"{caller.filename}({caller.lineno}) failed: {time_str} not a standard taos time init") + if precision not in TAOS_PRECISION: + tdLog.exit(f"{caller.filename}({caller.lineno}) failed: {precision} not a standard taos time precision") + + if time_str[-1] == TAOS_TIME_INIT[0]: + times = int(time_str[:-1]) * TIME_NS + if time_str[-1] == TAOS_TIME_INIT[1]: + times = int(time_str[:-1]) * TIME_US + if time_str[-1] == TAOS_TIME_INIT[2]: + times = int(time_str[:-1]) * TIME_MS + if time_str[-1] == TAOS_TIME_INIT[3]: + times = int(time_str[:-1]) * TIME_S + if time_str[-1] == TAOS_TIME_INIT[4]: + times = int(time_str[:-1]) * TIME_M + if time_str[-1] == TAOS_TIME_INIT[5]: + times = int(time_str[:-1]) * TIME_H + if time_str[-1] == TAOS_TIME_INIT[6]: + times = int(time_str[:-1]) * TIME_D + if time_str[-1] == TAOS_TIME_INIT[7]: + times = int(time_str[:-1]) * TIME_W + if time_str[-1] == TAOS_TIME_INIT[8]: + times = int(time_str[:-1]) * TIME_N + if time_str[-1] == TAOS_TIME_INIT[9]: + times = int(time_str[:-1]) * TIME_Y + + if precision == "ms": + return int(times) + elif precision == "us": + return int(times*1000) + elif precision == "ns": + return int(times*1000*1000) + def taosdStatus(self, state): tdLog.sleep(5) pstate = 0 diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 1b61bd1890..76d2546dac 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -29,7 +29,7 @@ #./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim #./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim #./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim -#./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim ./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim # ---- insert diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim index 9b11e16616..00c6e8a98d 100644 --- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim @@ -65,7 +65,7 @@ sql_error redistribute vgroup 3 dnode 6 dnode 3 dnode 4 # vgroup not exist sql_error redistribute vgroup 3 dnode 5 dnode 3 dnode 4 # un changed -sql_error redistribute vgroup 2 dnode 2 dnode 3 dnode 4 +# sql_error redistribute vgroup 2 dnode 2 dnode 3 dnode 4 # no enought vnodes sql_error redistribute vgroup 2 dnode 1 dnode 3 dnode 4 # offline vnodes @@ -176,8 +176,6 @@ if $rows != 1 then return -1 endi -return - print =============== step32: print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 diff --git a/tests/system-test/1-insert/create_retentions.py b/tests/system-test/1-insert/create_retentions.py index 5a0684e2ee..4b37eeb9a5 100644 --- a/tests/system-test/1-insert/create_retentions.py +++ b/tests/system-test/1-insert/create_retentions.py @@ -21,9 +21,9 @@ SINT_UN_COL = "c_sint_un" BINT_UN_COL = "c_bint_un" INT_UN_COL = "c_int_un" -BINARY_COL = "c8" -NCHAR_COL = "c9" -TS_COL = "c10" +BINARY_COL = "c_binary" +NCHAR_COL = "c_nchar" +TS_COL = "c_ts" NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] CHAR_COL = [ BINARY_COL, NCHAR_COL, ] @@ -51,12 +51,28 @@ class DataSet: binary_data : List[str] = None nchar_data : List[str] = None + def __post_init__(self): + self.ts_data = [] + self.int_data = [] + self.bint_data = [] + self.sint_data = [] + self.tint_data = [] + self.int_un_data = [] + self.bint_un_data = [] + self.sint_un_data = [] + self.tint_un_data = [] + self.float_data = [] + self.double_data = [] + self.bool_data = [] + self.binary_data = [] + self.nchar_data = [] + class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), True) + tdSql.init(conn.cursor(), False) @property def create_databases_sql_err(self): @@ -87,28 +103,28 @@ class TDTestCase: @property def create_stable_sql_err(self): return [ - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(ceil) watermark 1s maxdelay 1m", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(ceil) watermark 1s max_delay 1m", f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(count) watermark 1min", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay -1s", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay -1s", f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark -1m", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 1m ", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) maxdelay 1m ", + # f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 1m ", + # f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) max_delay 1m ", f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} binary(16)) tags (tag1 int) rollup(avg) watermark 1s", - f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) tags (tag1 int) rollup(avg) maxdelay 1m", - # f"create table ntb_1 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) rollup(avg) watermark 1s maxdelay 1s", + f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) tags (tag1 int) rollup(avg) max_delay 1m", + # f"create table ntb_1 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) rollup(avg) watermark 1s max_delay 1s", # f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) tags (tag1 int) " , # f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) " , # f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) " , # f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) " , - # watermark, maxdelay: [0, 900000], [ms, s, m, ?] - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 1u", + # watermark, max_delay: [0, 900000], [ms, s, m, ?] + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 1u", f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 1b", f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 900001ms", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 16m", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 901s", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 1h", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 0.2h", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 16m", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 901s", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 1h", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 0.2h", f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 0.002d", ] @@ -117,11 +133,11 @@ class TDTestCase: def create_stable_sql_current(self): return [ f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(avg)", - f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 5s maxdelay 1m", - f"create stable stb3 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(max) watermark 5s maxdelay 1m", - f"create stable stb4 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(sum) watermark 5s maxdelay 1m", - # f"create stable stb5 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(last) watermark 5s maxdelay 1m", - # f"create stable stb6 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s maxdelay 1m", + f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 5s max_delay 1m", + f"create stable stb3 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(max) watermark 5s max_delay 1m", + f"create stable stb4 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(sum) watermark 5s max_delay 1m", + # f"create stable stb5 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(last) watermark 5s max_delay 1m", + # f"create stable stb6 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s max_delay 1m", ] def test_create_stb(self): @@ -135,7 +151,7 @@ class TDTestCase: tdSql.checkRows(len(self.create_stable_sql_current)) # tdSql.execute("use db") # because db is a noraml database, not a rollup database, should not be able to create a rollup database - # tdSql.error(f"create stable nor_db_rollup_stb ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) file_factor 5.0") + # tdSql.error(f"create stable nor_db_rollup_stb ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 5s max_delay 1m") def test_create_databases(self): @@ -177,21 +193,6 @@ class TDTestCase: def __data_set(self, rows): data_set = DataSet() - # neg_data_set = DataSet() - data_set.ts_data = [] - data_set.int_data = [] - data_set.bint_data = [] - data_set.sint_data = [] - data_set.tint_data = [] - data_set.int_un_data = [] - data_set.bint_un_data = [] - data_set.sint_un_data = [] - data_set.tint_un_data = [] - data_set.float_data = [] - data_set.double_data = [] - data_set.bool_data = [] - data_set.binary_data = [] - data_set.nchar_data = [] for i in range(rows): data_set.ts_data.append(NOW + 1 * (rows - i)) @@ -226,6 +227,7 @@ class TDTestCase: return data_set def __insert_data(self): + tdLog.printNoPrefix("==========step: start inser data into tables now.....") data = self.__data_set(rows=self.rows) # now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) @@ -264,10 +266,10 @@ class TDTestCase: def run(self): self.rows = 10 - + tdSql.prepare() tdLog.printNoPrefix("==========step0:all check") - # self.all_test() + self.all_test() tdLog.printNoPrefix("==========step1:create table in normal database") tdSql.prepare() diff --git a/tests/system-test/1-insert/time_range_wise.py b/tests/system-test/1-insert/time_range_wise.py index 5387970d56..d4434987a6 100644 --- a/tests/system-test/1-insert/time_range_wise.py +++ b/tests/system-test/1-insert/time_range_wise.py @@ -17,25 +17,34 @@ TINT_COL = "c_tint" FLOAT_COL = "c_float" DOUBLE_COL = "c_double" BOOL_COL = "c_bool" -TINT_UN_COL = "c_tint_un" -SINT_UN_COL = "c_sint_un" -BINT_UN_COL = "c_bint_un" -INT_UN_COL = "c_int_un" +TINT_UN_COL = "c_utint" +SINT_UN_COL = "c_usint" +BINT_UN_COL = "c_ubint" +INT_UN_COL = "c_uint" BINARY_COL = "c_binary" NCHAR_COL = "c_nchar" TS_COL = "c_ts" - - NUM_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] CHAR_COL = [BINARY_COL, NCHAR_COL, ] BOOLEAN_COL = [BOOL_COL, ] TS_TYPE_COL = [TS_COL, ] +INT_TAG = "t_int" + +ALL_COL = [PRIMARY_COL, INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BINARY_COL, NCHAR_COL, BOOL_COL, TS_COL] +TAG_COL = [INT_TAG] + # insert data args: TIME_STEP = 10000 NOW = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) +# init db/table +DBNAME = "db" +STBNAME = "stb1" +CTBNAME = "ct1" +NTBNAME = "nt1" + @dataclass class DataSet: @@ -73,19 +82,25 @@ class DataSet: @dataclass class SMAschema: - creation : str = "CREATE" - index_name : str = "sma_index_1" - index_flag : str = "SMA INDEX" - operator : str = "ON" - tbname : str = None - watermark : str = None - maxdelay : str = None - func : Tuple[str] = None - interval : Tuple[str] = None - sliding : str = None - other : Any = None - drop : str = "DROP" - drop_flag : str = "INDEX" + creation : str = "CREATE" + index_name : str = "sma_index_1" + index_flag : str = "SMA INDEX" + operator : str = "ON" + tbname : str = None + watermark : str = "5s" + max_delay : str = "6m" + func : Tuple[str] = None + interval : Tuple[str] = ("6m", "10s") + sliding : str = "6m" + other : Any = None + drop : str = "DROP" + drop_flag : str = "INDEX" + querySmaOptimize : int = 1 + show : str = "SHOW" + show_msg : str = "INDEXES" + show_oper : str = "FROM" + dbname : str = None + rollup_db : bool = False def __post_init__(self): if isinstance(self.other, dict): @@ -111,8 +126,8 @@ class SMAschema: self.watermark = v del self.other[k] - if k.lower() == "maxdelay" and isinstance(v, str) and not self.maxdelay: - self.maxdelay = v + if k.lower() == "max_delay" and isinstance(v, str) and not self.max_delay: + self.max_delay = v del self.other[k] if k.lower() == "functions" and isinstance(v, tuple) and not self.func: @@ -131,12 +146,36 @@ class SMAschema: self.drop_flag = v del self.other[k] + if k.lower() == "show_msg" and isinstance(v, str) and not self.show_msg: + self.show_msg = v + del self.other[k] + + if k.lower() == "dbname" and isinstance(v, str) and not self.dbname: + self.dbname = v + del self.other[k] + + if k.lower() == "show_oper" and isinstance(v, str) and not self.show_oper: + self.show_oper = v + del self.other[k] + + if k.lower() == "rollup_db" and isinstance(v, bool) and not self.rollup_db: + self.rollup_db = v + del self.other[k] + + + +# from ...pytest.util.sql import * +# from ...pytest.util.constant import * class TDTestCase: + updatecfgDict = {"querySmaOptimize": 1} def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) + self.precision = "ms" + self.sma_count = 0 + self.sma_created_index = [] """ create sma index : @@ -155,13 +194,17 @@ class TDTestCase: if sma.func: sql += f" function({', '.join(sma.func)})" if sma.interval: - sql += f" interval({', '.join(sma.interval)})" + interval, offset = self.__get_interval_offset(sma.interval) + if offset: + sql += f" interval({interval}, {offset})" + else: + sql += f" interval({interval})" if sma.sliding: sql += f" sliding({sma.sliding})" if sma.watermark: sql += f" watermark {sma.watermark}" - if sma.maxdelay: - sql += f" maxdelay {sma.maxdelay}" + if sma.max_delay: + sql += f" max_delay {sma.max_delay}" if isinstance(sma.other, dict): for k,v in sma.other.items(): if isinstance(v,tuple) or isinstance(v, list): @@ -171,53 +214,305 @@ class TDTestCase: if isinstance(sma.other, tuple) or isinstance(sma.other, list): sql += " ".join(sma.other) if isinstance(sma.other, int) or isinstance(sma.other, float) or isinstance(sma.other, str): - sql += sma.other + sql += f" {sma.other}" return sql - def sma_create_check(self, sma:SMAschema): + def __get_sma_func_col(self, func): + cols = [] + if isinstance(func, str): + cols.append( func.split("(")[-1].split(")")[0] ) + elif isinstance(func, tuple) or isinstance(func, list): + for func_col in func: + cols.append(func_col.split("(")[-1].split(")")[0]) + else: + cols = [] + return cols + + def __check_sma_func(self, func:tuple): + if not isinstance(func, str) and not isinstance(func, tuple) and not isinstance(func, list): + return False + if isinstance(func, str) : + if "(" not in func or ")" not in func: + return False + if func.split("(")[0].upper() not in SMA_INDEX_FUNCTIONS: + return False + if func.split("(")[1].split(")")[0] not in ALL_COL and func.split("(")[1].split(")")[0] not in TAG_COL : + return False + if isinstance(func, tuple) or isinstance(func, list): + for arg in func: + if not isinstance(arg, str): + return False + if "(" not in arg or ")" not in arg: + return False + if arg.split("(")[0].upper() not in SMA_INDEX_FUNCTIONS: + return False + if arg.split("(")[1].split(")")[0] not in ALL_COL and arg.split("(")[1].split(")")[0] not in TAG_COL : + return False + return True + + def __check_sma_watermark(self, arg): + if not arg: + return False + if not isinstance(arg, str): + return False + if arg[-1] not in SMA_WATMARK_MAXDELAY_INIT: + return False + if len(arg) == 1: + return False + if not arg[:-1].isdecimal(): + return False + if tdSql.get_times(arg) > WATERMARK_MAX: + return False + if tdSql.get_times(arg) < WATERMARK_MIN: + return False + + return True + + def __check_sma_max_delay(self, arg): + if not self.__check_sma_watermark(arg): + return False + if tdSql.get_times(arg) < MAX_DELAY_MIN: + return False + + return True + + def __check_sma_sliding(self, arg): + if not isinstance(arg, str): + return False + if arg[-1] not in TAOS_TIME_INIT: + return False + if len(arg) == 1: + return False + if not arg[:-1].isdecimal(): + return False + + return True + + def __get_interval_offset(self, args): + if isinstance(args, str): + interval, offset = args, None + elif isinstance(args,tuple) or isinstance(args, list): + if len(args) == 1: + interval, offset = args[0], None + elif len(args) == 2: + interval, offset = args + else: + interval, offset = False, False + else: + interval, offset = False, False + + return interval, offset + + def __check_sma_interval(self, args): + if not isinstance(args, tuple) and not isinstance(args,str): + return False + interval, offset = self.__get_interval_offset(args) + if not interval: + return False + if not self.__check_sma_sliding(interval): + return False + if tdSql.get_times(interval) < INTERVAL_MIN: + return False + if offset: + if not self.__check_sma_sliding(offset): + return False + if tdSql.get_times(interval) <= tdSql.get_times(offset) : + return False + + return True + + def __sma_create_check(self, sma:SMAschema): + if self.updatecfgDict["querySmaOptimize"] == 0: + return False + # # TODO: if database is a rollup-db, can not create sma index + # tdSql.query("select database()") + # if sma.rollup_db : + # return False tdSql.query("show stables") + if not sma.tbname: + return False stb_in_list = False for row in tdSql.queryResult: if sma.tbname == row[0]: stb_in_list = True - break if not stb_in_list: - tdSql.error(self.__create_sma_index(sma)) - if not sma.creation: - tdSql.error(self.__create_sma_index(sma)) - if not sma.index_flag: - tdSql.error(self.__create_sma_index(sma)) - if not sma.index_name: - tdSql.error(self.__create_sma_index(sma)) - if not sma.operator: - tdSql.error(self.__create_sma_index(sma)) - if not sma.tbname: - tdSql.error(self.__create_sma_index(sma)) - if not sma.func: - tdSql.error(self.__create_sma_index(sma)) - if not sma.interval: - tdSql.error(self.__create_sma_index(sma)) - if not sma.sliding: - tdSql.error(self.__create_sma_index(sma)) + return False + if not sma.creation or not isinstance(sma.creation, str) or sma.creation.upper() != "CREATE": + return False + if not sma.index_flag or not isinstance(sma.index_flag, str) or sma.index_flag.upper() != "SMA INDEX" : + return False + if not sma.index_name or not isinstance(sma.index_name, str) or sma.index_name.upper() in TAOS_KEYWORDS: + return False + if not sma.operator or not isinstance(sma.operator, str) or sma.operator.upper() != "ON": + return False + + if not sma.func or not self.__check_sma_func(sma.func): + return False + tdSql.query(f"desc {sma.tbname}") + _col_list = [] + for col_row in tdSql.queryResult: + _col_list.append(col_row[0]) + _sma_func_cols = self.__get_sma_func_col(sma.func) + for _sma_func_col in _sma_func_cols: + if _sma_func_col not in _col_list: + return False + + if sma.sliding and not self.__check_sma_sliding(sma.sliding): + return False + interval, _ = self.__get_interval_offset(sma.interval) + if not sma.interval or not self.__check_sma_interval(sma.interval) : + return False + if sma.sliding and tdSql.get_times(interval) < tdSql.get_times(sma.sliding): + return False + if sma.watermark and not self.__check_sma_watermark(sma.watermark): + return False + if sma.max_delay and not self.__check_sma_max_delay(sma.max_delay): + return False if sma.other: + return False + + return True + + def sma_create_check(self, sma:SMAschema): + if self.__sma_create_check(sma): + tdSql.query(self.__create_sma_index(sma)) + self.sma_count += 1 + self.sma_created_index.append(sma.index_name) + tdSql.query("show streams") + tdSql.checkRows(self.sma_count) + + else: tdSql.error(self.__create_sma_index(sma)) + def __drop_sma_index(self, sma:SMAschema): + sql = f"{sma.drop} {sma.drop_flag} {sma.index_name}" + return sql + + def __sma_drop_check(self, sma:SMAschema): + if not sma.drop: + return False + if not sma.drop_flag: + return False + if not sma.index_name: + return False + + return True + + def sma_drop_check(self, sma:SMAschema): + if self.__sma_drop_check(sma): + tdSql.query(self.__drop_sma_index(sma)) + print(self.__drop_sma_index(sma)) + self.sma_count -= 1 + self.sma_created_index = list(filter(lambda x: x != sma.index_name, self.sma_created_index)) + tdSql.query("show streams") + tdSql.checkRows(self.sma_count) + + else: + tdSql.error(self.__drop_sma_index(sma)) + + def __show_sma_index(self, sma:SMAschema): + sql = f"{sma.show} {sma.show_msg} {sma.show_oper} {sma.tbname}" + return sql + + def __sma_show_check(self, sma:SMAschema): + if not sma.show: + return False + if not sma.show_msg: + return False + if not sma.show_oper: + return False + if not sma.tbname: + return False + + return True + + def sma_show_check(self, sma:SMAschema): + if self.__sma_show_check(sma): + tdSql.query(self.__show_sma_index(sma)) + tdSql.checkRows(self.sma_count) + else: + tdSql.error(self.__show_sma_index(sma)) + + @property + def __create_sma_sql(self): + err_sqls = [] + cur_sqls = [] + # err_set + # # case 1: required fields check + err_sqls.append( SMAschema(creation="", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(index_name="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(index_flag="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(operator="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(tbname="", func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(func=("",),tbname=STBNAME ) ) + err_sqls.append( SMAschema(interval=(""),tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + + # # case 2: err fields + err_sqls.append( SMAschema(creation="show",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(creation="alter",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(creation="select",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + + err_sqls.append( SMAschema(index_flag="SMA INDEXES", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(index_flag="SMA INDEX ,", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(index_name="tbname", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + + + # current_set + + cur_sqls.append( SMAschema(max_delay="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + cur_sqls.append( SMAschema(watermark="",index_name="sma_index_2",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + cur_sqls.append( SMAschema(sliding="",index_name='sma_index_3',tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + + + return err_sqls, cur_sqls + + def test_create_sma(self): + err_sqls , cur_sqls = self.__create_sma_sql + for err_sql in err_sqls: + self.sma_create_check(err_sql) + for cur_sql in cur_sqls: + self.sma_create_check(cur_sql) + + @property + def __drop_sma_sql(self): + err_sqls = [] + cur_sqls = [] + # err_set + ## case 1: required fields check + err_sqls.append( SMAschema(drop="") ) + err_sqls.append( SMAschema(drop_flag="") ) + err_sqls.append( SMAschema(index_name="") ) + + for index in self.sma_created_index: + cur_sqls.append(SMAschema(index_name=index)) + + return err_sqls, cur_sqls + + def test_drop_sma(self): + err_sqls , cur_sqls = self.__drop_sma_sql + for err_sql in err_sqls: + self.sma_drop_check(err_sql) + # for cur_sql in cur_sqls: + # self.sma_drop_check(cur_sql) def all_test(self): + self.test_create_sma() + self.test_drop_sma() + pass def __create_tb(self): tdLog.printNoPrefix("==========step: create table") - create_stb_sql = f'''create table stb1( + create_stb_sql = f'''create table {STBNAME}( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp, {TINT_UN_COL} tinyint unsigned, {SINT_UN_COL} smallint unsigned, {INT_UN_COL} int unsigned, {BINT_UN_COL} bigint unsigned - ) tags (tag1 int) + ) tags ({INT_TAG} int) ''' - create_ntb_sql = f'''create table t1( + create_ntb_sql = f'''create table {NTBNAME}( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp, @@ -253,6 +548,7 @@ class TDTestCase: return data_set def __insert_data(self): + tdLog.printNoPrefix("==========step: start inser data into tables now.....") data = self.__data_set(rows=self.rows) # now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) @@ -278,7 +574,7 @@ class TDTestCase: tdSql.execute( f"insert into ct4 values ( {NOW - i * int(TIME_STEP * 0.8) }, {row_data} )") tdSql.execute( - f"insert into t1 values ( {NOW - i * int(TIME_STEP * 1.2)}, {row_data} )") + f"insert into {NTBNAME} values ( {NOW - i * int(TIME_STEP * 1.2)}, {row_data} )") tdSql.execute( f"insert into ct2 values ( {NOW + int(TIME_STEP * 0.6)}, {null_data} )") @@ -295,28 +591,31 @@ class TDTestCase: f"insert into ct4 values ( {NOW - self.rows * int(TIME_STEP * 0.39)}, {null_data} )") tdSql.execute( - f"insert into t1 values ( {NOW + int(TIME_STEP * 1.2)}, {null_data} )") + f"insert into {NTBNAME} values ( {NOW + int(TIME_STEP * 1.2)}, {null_data} )") tdSql.execute( - f"insert into t1 values ( {NOW - (self.rows + 1) * int(TIME_STEP * 1.2)}, {null_data} )") + f"insert into {NTBNAME} values ( {NOW - (self.rows + 1) * int(TIME_STEP * 1.2)}, {null_data} )") tdSql.execute( - f"insert into t1 values ( {NOW - self.rows * int(TIME_STEP * 0.59)}, {null_data} )") + f"insert into {NTBNAME} values ( {NOW - self.rows * int(TIME_STEP * 0.59)}, {null_data} )") def run(self): - sma1 = SMAschema(func=("min(c1)","max(c2)")) - sql1 = self.__create_sma_index(sma1) - print("================") - print(sql1) - # a = DataSet() - # return self.rows = 10 tdLog.printNoPrefix("==========step0:all check") - # self.all_test() tdLog.printNoPrefix("==========step1:create table in normal database") tdSql.prepare() self.__create_tb() - self.__insert_data() + # self.__insert_data() + self.all_test() + + # drop databases, create same name db、stb and sma index + # tdSql.prepare() + # self.__create_tb() + # self.__insert_data() + # self.all_test() + + + return tdLog.printNoPrefix("==========step2:create table in rollup database") diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 5ff11c84dd..df6390f59c 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), True) + tdSql.init(conn.cursor(), False) def __query_condition(self,tbname): query_condition = [] diff --git a/tests/system-test/2-query/join2.py b/tests/system-test/2-query/join2.py index 40da41eee7..5533cb840e 100644 --- a/tests/system-test/2-query/join2.py +++ b/tests/system-test/2-query/join2.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), True) + tdSql.init(conn.cursor(), False) def __query_condition(self,tbname): query_condition = [] diff --git a/tests/system-test/2-query/substr.py b/tests/system-test/2-query/substr.py index 79b5ac515b..f833a42b57 100644 --- a/tests/system-test/2-query/substr.py +++ b/tests/system-test/2-query/substr.py @@ -31,7 +31,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(),False) def __substr_condition(self): # sourcery skip: extract-method substr_condition = [] diff --git a/tests/system-test/2-query/tail.py b/tests/system-test/2-query/tail.py index 0e2110f912..a4c85cc5ed 100644 --- a/tests/system-test/2-query/tail.py +++ b/tests/system-test/2-query/tail.py @@ -188,8 +188,8 @@ class TDTestCase: def check_tail_table(self , tbname , col_name , tail_rows , offset): tail_sql = f"select tail({col_name} , {tail_rows} , {offset}) from {tbname}" - equal_sql = f"select {col_name} from (select ts , {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}) order by ts" - #equal_sql = f"select {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}" + #equal_sql = f"select {col_name} from (select ts , {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}) order by ts" + equal_sql = f"select {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}" tdSql.query(tail_sql) tail_result = tdSql.queryResult @@ -404,7 +404,7 @@ class TDTestCase: f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) - tdSql.query("select tail(c2,2) from sub1_bound") + tdSql.query("select tail(c2,2) from sub1_bound order by 1 desc") tdSql.checkRows(2) tdSql.checkData(0,0,9223372036854775803)