diff --git a/Jenkinsfile2 b/Jenkinsfile2 index f582461fb2..f618912295 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -338,6 +338,14 @@ pipeline { changeRequest() } steps { + script { + def linux_node_ip = sh ( + script: 'ip addr|grep 192|grep -v virbr|awk "{print \\\$2}"|sed "s/\\/.*//"', + returnStdout: true + ).trim() + echo "${linux_node_ip}" + echo "${WKDIR}/restore.sh -p ${BRANCH_NAME} -n ${BUILD_ID} -c {container name}" + } catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { timeout(time: 120, unit: 'MINUTES'){ pre_test() diff --git a/include/client/taos.h b/include/client/taos.h index 5e7f12de0a..d31d5c582c 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -128,6 +128,7 @@ typedef struct setConfRet { DLL_EXPORT void taos_cleanup(void); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); +DLL_EXPORT setConfRet taos_set_config(const char *config); DLL_EXPORT int taos_init(void); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 2e2c7d1700..6653fdd9cd 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -236,7 +236,7 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* int8_t needCompress); const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData); -void blockDebugShowData(const SArray* dataBlocks, const char* flag); +void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag); // for debug char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf); diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index f9ede63f7f..8f7f22a6a0 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -78,7 +78,6 @@ int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag); int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag); int32_t tTagToValArray(const STag *pTag, SArray **ppArray); void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remove -void debugCheckTags(STag *pTag); // TODO: remove // STRUCT ================= struct STColumn { diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a5688af18a..c5b0b89311 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1886,7 +1886,7 @@ typedef struct SVCreateStbReq { int8_t rollup; SSchemaWrapper schemaRow; SSchemaWrapper schemaTag; - SRSmaParam pRSmaParam; + SRSmaParam rsmaParam; } SVCreateStbReq; int tEncodeSVCreateStbReq(SEncoder* pCoder, const SVCreateStbReq* pReq); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index e23b84c1f8..0bd917a9c6 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -24,6 +24,8 @@ extern "C" { #include "querynodes.h" #include "tname.h" +#define SLOT_NAME_LEN TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN + typedef struct SLogicNode { ENodeType type; SNodeList* pTargets; // SColumnNode @@ -74,8 +76,8 @@ typedef struct SScanLogicNode { int16_t tsColId; double filesFactor; SArray* pSmaIndexes; - SNodeList* pPartTags; - bool partSort; + SNodeList* pGroupTags; + bool groupSort; } SScanLogicNode; typedef struct SJoinLogicNode { @@ -100,6 +102,7 @@ typedef struct SProjectLogicNode { typedef struct SIndefRowsFuncLogicNode { SLogicNode node; SNodeList* pFuncs; + bool isTailFunc; } SIndefRowsFuncLogicNode; typedef struct SInterpFuncLogicNode { @@ -138,6 +141,7 @@ typedef struct SMergeLogicNode { SNodeList* pInputs; int32_t numOfChannels; int32_t srcGroupId; + bool groupSort; } SMergeLogicNode; typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType; @@ -184,6 +188,7 @@ typedef struct SFillLogicNode { typedef struct SSortLogicNode { SLogicNode node; SNodeList* pSortKeys; + bool groupSort; } SSortLogicNode; typedef struct SPartitionLogicNode { @@ -230,6 +235,7 @@ typedef struct SSlotDescNode { bool reserve; bool output; bool tag; + char name[SLOT_NAME_LEN]; } SSlotDescNode; typedef struct SDataBlockDescNode { @@ -279,7 +285,8 @@ typedef struct STableScanPhysiNode { double ratio; int32_t dataRequired; SNodeList* pDynamicScanFuncs; - SNodeList* pPartitionTags; + SNodeList* pGroupTags; + bool groupSort; int64_t interval; int64_t offset; int64_t sliding; @@ -353,6 +360,7 @@ typedef struct SMergePhysiNode { SNodeList* pTargets; int32_t numOfChannels; int32_t srcGroupId; + bool groupSort; } SMergePhysiNode; typedef struct SWinodwPhysiNode { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index a6e466e73e..39c569acef 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -259,6 +259,7 @@ typedef struct SSelectStmt { bool hasTailFunc; bool hasInterpFunc; bool hasLastRowFunc; + bool groupSort; } SSelectStmt; typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType; diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index c2c1a3534d..2b8c6a895e 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -69,7 +69,7 @@ typedef struct SRpcMsg { } SRpcMsg; typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *rf); -typedef bool (*RpcRfp)(int32_t code); +typedef bool (*RpcRfp)(int32_t code, tmsg_t msgType); typedef struct SRpcInit { char localFqdn[TSDB_FQDN_LEN]; diff --git a/include/os/osSystem.h b/include/os/osSystem.h index 6770be6e46..581e688ccb 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -29,6 +29,9 @@ extern "C" { #define tcgetattr TCGETATTR_FUNC_TAOS_FORBID #endif +#define TAOS_CONSOLE_PROMPT_HEADER "taos> " +#define TAOS_CONSOLE_PROMPT_CONTINUE " -> " + typedef struct TdCmd *TdCmdPtr; TdCmdPtr taosOpenCmd(const char* cmd); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index d7bf4b60f1..8e0556125a 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -84,9 +84,12 @@ void closeTransporter(STscObj *pTscObj) { rpcClose(pTscObj->pAppInfo->pTransporter); } -static bool clientRpcRfp(int32_t code) { +static bool clientRpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { + if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) { + return false; + } return true; } else { return false; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index bbd477fa3b..52574dcc9f 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -81,6 +81,19 @@ void taos_cleanup(void) { taosCloseLog(); } +static setConfRet taos_set_config_imp(const char *config){ + setConfRet ret = {SET_CONF_RET_SUCC, {0}}; + // TODO: need re-implementation + return ret; +} + +setConfRet taos_set_config(const char *config){ +// TODO pthread_mutex_lock(&setConfMutex); + setConfRet ret = taos_set_config_imp(config); +// pthread_mutex_unlock(&setConfMutex); + return ret; +} + TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { tscDebug("try to connect to %s:%u, user:%s db:%s", ip, port, user, db); if (user == NULL) { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 9f89d72172..1ec298ee15 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1605,7 +1605,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { return buf; } -void blockDebugShowData(const SArray* dataBlocks, const char* flag) { +void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) { char pBuf[128] = {0}; int32_t sz = taosArrayGetSize(dataBlocks); for (int32_t i = 0; i < sz; i++) { @@ -1613,7 +1613,7 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) { size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); int32_t rows = pDataBlock->info.rows; - printf("%s |block type %d |child id %d|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId); + printf("%s |block type %d |child id %d|group id %zX\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId); for (int32_t j = 0; j < rows; j++) { printf("%s |", flag); for (int32_t k = 0; k < numOfCols; k++) { diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 8460a27a0e..7c1b31b6e4 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -862,21 +862,6 @@ void debugPrintSTag(STag *pTag, const char *tag, int32_t ln) { printf("\n"); } -void debugCheckTags(STag *pTag) { - switch (pTag->flags) { - case 0x0: - case 0x20: - case 0x40: - case 0x60: - break; - default: - ASSERT(0); - } - - ASSERT(pTag->nTag <= 128 && pTag->nTag >= 0); - ASSERT(pTag->ver <= 512 && pTag->ver >= 0); // temp condition for pTag->ver -} - static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { int32_t n = 0; @@ -999,7 +984,6 @@ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag) { debugPrintSTag(*ppTag, __func__, __LINE__); #endif - debugCheckTags(*ppTag); // TODO: remove this line after debug return code; _err: diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 8a052026f2..e9b5c67d76 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4763,7 +4763,7 @@ int tEncodeSVCreateStbReq(SEncoder *pCoder, const SVCreateStbReq *pReq) { if (tEncodeSSchemaWrapper(pCoder, &pReq->schemaRow) < 0) return -1; if (tEncodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1; if (pReq->rollup) { - if (tEncodeSRSmaParam(pCoder, &pReq->pRSmaParam) < 0) return -1; + if (tEncodeSRSmaParam(pCoder, &pReq->rsmaParam) < 0) return -1; } tEndEncode(pCoder); @@ -4779,7 +4779,7 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) { if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaRow) < 0) return -1; if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1; if (pReq->rollup) { - if (tDecodeSRSmaParam(pCoder, &pReq->pRSmaParam) < 0) return -1; + if (tDecodeSRSmaParam(pCoder, &pReq->rsmaParam) < 0) return -1; } tEndDecode(pCoder); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index a4745abd5b..7e31cc3144 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -248,9 +248,12 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { } } -static bool rpcRfp(int32_t code) { +static bool rpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { + if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) { + return false; + } return true; } else { return false; diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index bc2d5c82b1..1497bba11c 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -39,8 +39,10 @@ typedef struct { int32_t id; int32_t errCode; int32_t acceptableCode; - ETrnStage stage; + int32_t retryCode; ETrnAct actionType; + ETrnStage stage; + int8_t reserved; int8_t rawWritten; int8_t msgSent; int8_t msgReceived; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 029a1e6f8c..164bcc7d60 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -234,7 +234,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { if (retrieveReq.user[0] != 0) { memcpy(pReq->info.conn.user, retrieveReq.user, TSDB_USER_LEN); } else { - memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, TSDB_USER_LEN); + memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, strlen(TSDB_DEFAULT_USER) + 1); } if (mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db) != 0) { return -1; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 92f85ecd04..dd01a0fa16 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -427,17 +427,17 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.schemaTag.pSchema = pStb->pTags; if (req.rollup) { - req.pRSmaParam.maxdelay[0] = pStb->maxdelay[0]; - req.pRSmaParam.maxdelay[1] = pStb->maxdelay[1]; + req.rsmaParam.maxdelay[0] = pStb->maxdelay[0]; + req.rsmaParam.maxdelay[1] = pStb->maxdelay[1]; if (pStb->ast1Len > 0) { - if (mndConvertRsmaTask(&req.pRSmaParam.qmsg[0], &req.pRSmaParam.qmsgLen[0], pStb->pAst1, pStb->uid, - STREAM_TRIGGER_WINDOW_CLOSE, req.pRSmaParam.watermark[0]) < 0) { + if (mndConvertRsmaTask(&req.rsmaParam.qmsg[0], &req.rsmaParam.qmsgLen[0], pStb->pAst1, pStb->uid, + STREAM_TRIGGER_WINDOW_CLOSE, req.rsmaParam.watermark[0]) < 0) { goto _err; } } if (pStb->ast2Len > 0) { - if (mndConvertRsmaTask(&req.pRSmaParam.qmsg[1], &req.pRSmaParam.qmsgLen[1], pStb->pAst2, pStb->uid, - STREAM_TRIGGER_WINDOW_CLOSE, req.pRSmaParam.watermark[1]) < 0) { + if (mndConvertRsmaTask(&req.rsmaParam.qmsg[1], &req.rsmaParam.qmsgLen[1], pStb->pAst2, pStb->uid, + STREAM_TRIGGER_WINDOW_CLOSE, req.rsmaParam.watermark[1]) < 0) { goto _err; } } @@ -470,12 +470,12 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt tEncoderClear(&encoder); *pContLen = contLen; - taosMemoryFreeClear(req.pRSmaParam.qmsg[0]); - taosMemoryFreeClear(req.pRSmaParam.qmsg[1]); + taosMemoryFreeClear(req.rsmaParam.qmsg[0]); + taosMemoryFreeClear(req.rsmaParam.qmsg[1]); return pHead; _err: - taosMemoryFreeClear(req.pRSmaParam.qmsg[0]); - taosMemoryFreeClear(req.pRSmaParam.qmsg[1]); + taosMemoryFreeClear(req.rsmaParam.qmsg[0]); + taosMemoryFreeClear(req.rsmaParam.qmsg[1]); return NULL; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index a9de1a05a7..b09ee5f608 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -15,15 +15,15 @@ #define _DEFAULT_SOURCE #include "mndTrans.h" -#include "mndPrivilege.h" #include "mndConsumer.h" #include "mndDb.h" +#include "mndPrivilege.h" #include "mndShow.h" #include "mndSync.h" #include "mndUser.h" -#define TRANS_VER_NUMBER 1 -#define TRANS_ARRAY_SIZE 8 +#define TRANS_VER_NUMBER 1 +#define TRANS_ARRAY_SIZE 8 #define TRANS_RESERVE_SIZE 64 static SSdbRaw *mndTransActionEncode(STrans *pTrans); @@ -55,7 +55,7 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans); static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsMaster(pMnode); } static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans); -static int32_t mndProcessTransReq(SRpcMsg *pReq); +static int32_t mndProcessTransTimer(SRpcMsg *pReq); static int32_t mndProcessTtl(SRpcMsg *pReq); static int32_t mndProcessKillTransReq(SRpcMsg *pReq); @@ -73,7 +73,7 @@ int32_t mndInitTrans(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndTransActionDelete, }; - mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransReq); + mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransTimer); mndSetMsgHandle(pMnode, TDMT_MND_KILL_TRANS, mndProcessKillTransReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TRANS, mndRetrieveTrans); @@ -139,8 +139,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) + SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER) if (pAction->actionType == TRANS_ACTION_RAW) { int32_t len = sdbGetRawTotalSize(pAction->pRaw); SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER) @@ -163,8 +165,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) + SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER) if (pAction->actionType == TRANS_ACTION_RAW) { int32_t len = sdbGetRawTotalSize(pAction->pRaw); SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER) @@ -187,8 +191,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) + SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER) if (pAction->actionType == TRANS_ACTION_RAW) { int32_t len = sdbGetRawTotalSize(pAction->pRaw); SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER) @@ -291,10 +297,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER) SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) action.actionType = actionType; SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) action.stage = stage; + SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER) if (action.actionType == TRANS_ACTION_RAW) { SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER) SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) @@ -324,10 +332,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER) SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) action.actionType = actionType; SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) action.stage = stage; + SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER) if (action.actionType == TRANS_ACTION_RAW) { SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER) SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) @@ -357,10 +367,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER) SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) action.actionType = actionType; SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) action.stage = stage; + SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER) if (action.actionType) { SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER) SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) @@ -463,15 +475,25 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { if (fp) { (*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen); } + pTrans->startFunc = 0; } return 0; } static void mndTransDropData(STrans *pTrans) { - mndTransDropActions(pTrans->redoActions); - mndTransDropActions(pTrans->undoActions); - mndTransDropActions(pTrans->commitActions); + if (pTrans->redoActions != NULL) { + mndTransDropActions(pTrans->redoActions); + pTrans->redoActions = NULL; + } + if (pTrans->undoActions != NULL) { + mndTransDropActions(pTrans->undoActions); + pTrans->undoActions = NULL; + } + if (pTrans->commitActions != NULL) { + mndTransDropActions(pTrans->commitActions); + pTrans->commitActions = NULL; + } if (pTrans->rpcRsp != NULL) { taosMemoryFree(pTrans->rpcRsp); pTrans->rpcRsp = NULL; @@ -492,6 +514,7 @@ static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) { if (fp) { (*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen); } + pTrans->stopFunc = 0; } mndTransDropData(pTrans); @@ -805,7 +828,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { sendRsp = true; } } else { - if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 3) { + if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 6) { if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR; sendRsp = true; } @@ -875,8 +898,8 @@ int32_t mndTransProcessRsp(SRpcMsg *pRsp) { pAction->errCode = pRsp->code; } - mDebug("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x", transId, mndTransStr(pAction->stage), action, - pRsp->code, pAction->acceptableCode); + mDebug("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId, + mndTransStr(pAction->stage), action, pRsp->code, pAction->acceptableCode, pAction->retryCode); mndTransExecute(pMnode, pTrans); _OVER: @@ -884,6 +907,21 @@ _OVER: return 0; } +static void mndTransResetAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { + pAction->rawWritten = 0; + pAction->msgSent = 0; + pAction->msgReceived = 0; + if (pAction->errCode == TSDB_CODE_RPC_REDIRECT || pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR || + pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) { + pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps; + mDebug("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage), + pAction->id, pAction->epSet.inUse); + } else { + mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), pAction->id); + } + pAction->errCode = 0; +} + static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) { int32_t numOfActions = taosArrayGetSize(pArray); @@ -894,18 +932,7 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) continue; if (pAction->rawWritten && (pAction->errCode == 0 || pAction->errCode == pAction->acceptableCode)) continue; - pAction->rawWritten = 0; - pAction->msgSent = 0; - pAction->msgReceived = 0; - if (pAction->errCode == TSDB_CODE_RPC_REDIRECT || pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR || - pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) { - pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps; - mDebug("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage), - action, pAction->epSet.inUse); - } else { - mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), action); - } - pAction->errCode = 0; + mndTransResetAction(pMnode, pTrans, pAction); } } @@ -1112,9 +1139,9 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) if (pAction->msgReceived) { if (pAction->errCode != 0 && pAction->errCode != pAction->acceptableCode) { code = pAction->errCode; - pAction->msgSent = 0; - pAction->msgReceived = 0; - mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), action); + mndTransResetAction(pMnode, pTrans, pAction); + } else { + mDebug("trans:%d, %s:%d execute successfully", pTrans->id, mndTransStr(pAction->stage), action); } } else { code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -1123,6 +1150,8 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) if (pAction->rawWritten) { if (pAction->errCode != 0 && pAction->errCode != pAction->acceptableCode) { code = pAction->errCode; + } else { + mDebug("trans:%d, %s:%d write successfully", pTrans->id, mndTransStr(pAction->stage), action); } } } @@ -1156,9 +1185,16 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) { mDebug("trans:%d, %s:%d is in progress and wait it finish", pTrans->id, mndTransStr(pAction->stage), pAction->id); break; + } else if (code == pAction->retryCode) { + mDebug("trans:%d, %s:%d receive code:0x%x and retry", pTrans->id, mndTransStr(pAction->stage), pAction->id, code); + taosMsleep(300); + action--; + continue; } else { terrno = code; pTrans->code = code; + mDebug("trans:%d, %s:%d receive code:0x%x and wait another schedule, failedTimes:%d", pTrans->id, + mndTransStr(pAction->stage), pAction->id, code, pTrans->failedTimes); break; } } @@ -1343,7 +1379,7 @@ void mndTransExecute(SMnode *pMnode, STrans *pTrans) { mndTransSendRpcRsp(pMnode, pTrans); } -static int32_t mndProcessTransReq(SRpcMsg *pReq) { +static int32_t mndProcessTransTimer(SRpcMsg *pReq) { mndTransPullup(pReq->info.node); return 0; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 0e931e0a9c..d42016e6a0 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -15,10 +15,10 @@ #define _DEFAULT_SOURCE #include "mndVgroup.h" -#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" +#include "mndPrivilege.h" #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" @@ -896,6 +896,8 @@ int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD action.pCont = pHead; action.contLen = contLen; action.msgType = TDMT_VND_ALTER_CONFIRM; + // incorrect redirect result will cause this erro + action.retryCode = TSDB_CODE_VND_INVALID_VGROUP_ID; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pHead); @@ -942,6 +944,8 @@ static int32_t mndAddSetVnodeStandByAction(SMnode *pMnode, STrans *pTrans, SDbOb action.contLen = contLen; action.msgType = TDMT_SYNC_SET_VNODE_STANDBY; action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + // Keep retrying until the target vnode is not the leader + action.retryCode = TSDB_CODE_SYN_IS_LEADER; if (isRedo) { if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -1003,7 +1007,7 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, mInfo("vgId:%d, will add 1 vnodes", pVgroup->vgId); if (mndAddVnodeToVgroup(pMnode, &newVg, pArray) != 0) return -1; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[1], true) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[newVg.replica - 1], true) != 0) return -1; if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg, TDMT_VND_ALTER_REPLICA) != 0) return -1; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; @@ -1017,10 +1021,19 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; - SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) return -1; - sdbSetRawStatus(pRaw, SDB_STATUS_READY); - pRaw = NULL; + { + SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) return -1; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + pRaw = NULL; + } + + { + SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) return -1; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + pRaw = NULL; + } mInfo("vgId:%d, vgroup info after move, replica:%d", newVg.vgId, newVg.replica); for (int32_t i = 0; i < newVg.replica; ++i) { @@ -1168,10 +1181,19 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld3->id) != 0) goto _OVER; } - pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; - sdbSetRawStatus(pRaw, SDB_STATUS_READY); - pRaw = NULL; + { + pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + pRaw = NULL; + } + + { + pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + pRaw = NULL; + } mInfo("vgId:%d, vgroup info after redistribute, replica:%d", newVg.vgId, newVg.replica); for (int32_t i = 0; i < newVg.replica; ++i) { @@ -1229,7 +1251,8 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { } if (req.dnodeId1 == pVgroup->vnodeGid[0].dnodeId) { - terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; + // terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; + code = 0; goto _OVER; } @@ -1351,7 +1374,8 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { } if (pNew1 == NULL && pOld1 == NULL && pNew2 == NULL && pOld2 == NULL && pNew3 == NULL && pOld3 == NULL) { - terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; + // terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; + code = 0; goto _OVER; } @@ -1424,13 +1448,25 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, S } else { } - SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); - if (pVgRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) { - sdbFreeRaw(pVgRaw); - return -1; + { + SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { + sdbFreeRaw(pVgRaw); + return -1; + } + sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); + } + + { + SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) { + sdbFreeRaw(pVgRaw); + return -1; + } + sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); } - sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); } return 0; @@ -1538,12 +1574,23 @@ static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SD if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pDst->id) != 0) return -1; if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pSrc->id) != 0) return -1; - SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) { - sdbFreeRaw(pRaw); - return -1; + { + SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) { + sdbFreeRaw(pRaw); + return -1; + } + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + } + + { + SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) { + sdbFreeRaw(pRaw); + return -1; + } + sdbSetRawStatus(pRaw, SDB_STATUS_READY); } - sdbSetRawStatus(pRaw, SDB_STATUS_READY); mInfo("vgId:%d, vgroup info after balance, replica:%d", newVg.vgId, newVg.replica); for (int32_t i = 0; i < newVg.replica; ++i) { @@ -1552,7 +1599,8 @@ static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SD return 0; } -static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDnodeObj *pSrc, SDnodeObj *pDst) { +static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDnodeObj *pSrc, SDnodeObj *pDst, + SHashObj *pBalancedVgroups) { void *pIter = NULL; int32_t code = -1; SSdb *pSdb = pMnode->pSdb; @@ -1561,6 +1609,10 @@ static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDno SVgObj *pVgroup = NULL; pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; + if (taosHashGet(pBalancedVgroups, &pVgroup->vgId, sizeof(int32_t)) != NULL) { + sdbRelease(pSdb, pVgroup); + continue; + } bool existInSrc = false; bool existInDst = false; @@ -1577,6 +1629,9 @@ static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDno SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName); code = mndSetBalanceVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, pSrc, pDst); + if (code == 0) { + code = taosHashPut(pBalancedVgroups, &pVgroup->vgId, sizeof(int32_t), &pVgroup->vgId, sizeof(int32_t)); + } mndReleaseDb(pMnode, pDb); sdbRelease(pSdb, pVgroup); sdbCancelFetch(pSdb, pIter); @@ -1590,6 +1645,10 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) { int32_t code = -1; int32_t numOfVgroups = 0; STrans *pTrans = NULL; + SHashObj *pBalancedVgroups = NULL; + + pBalancedVgroups = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (pBalancedVgroups == NULL) goto _OVER; pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); if (pTrans == NULL) goto _OVER; @@ -1613,18 +1672,18 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) { pDst->id, dstScore); if (srcScore > dstScore - 0.000001) { - code = mndBalanceVgroupBetweenDnode(pMnode, pTrans, pSrc, pDst); + code = mndBalanceVgroupBetweenDnode(pMnode, pTrans, pSrc, pDst, pBalancedVgroups); if (code == 0) { pSrc->numOfVnodes--; pDst->numOfVnodes++; numOfVgroups++; continue; } else { - mError("trans:%d, failed to balance vgroup from dnode:%d to dnode:%d", pTrans->id, pSrc->id, pDst->id); - return -1; + mDebug("trans:%d, no vgroup need to balance from dnode:%d to dnode:%d", pTrans->id, pSrc->id, pDst->id); + break; } } else { - mDebug("trans:%d, no vgroup need to balance vgroup any more", pTrans->id); + mDebug("trans:%d, no vgroup need to balance any more", pTrans->id); break; } } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index c49b33beb2..5c2d2cd712 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -199,15 +199,20 @@ typedef struct { uint64_t groupId; } STableKeyInfo; +#define TABLE_ROLLUP_ON ((int8_t)0x1) +#define TABLE_IS_ROLLUP(FLG) (((FLG) & (TABLE_ROLLUP_ON)) != 0) +#define TABLE_SET_ROLLUP(FLG) ((FLG) |= TABLE_ROLLUP_ON) struct SMetaEntry { int64_t version; int8_t type; + int8_t flags; // TODO: need refactor? tb_uid_t uid; char *name; union { struct { SSchemaWrapper schemaRow; SSchemaWrapper schemaTag; + SRSmaParam rsmaParam; } stbEntry; struct { int64_t ctime; diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 956e451b58..4b42ab5263 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -176,15 +176,18 @@ static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) { static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType); +void *tdFreeRSmaInfo(SRSmaInfo *pInfo); -void *tdFreeRSmaInfo(SRSmaInfo *pInfo); - +int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); +int32_t tdProcessRSmaRestoreImpl(SSma *pSma); int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg); int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg); int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); // smaFileUtil ================ +#define TD_FILE_HEAD_SIZE 512 + typedef struct STFInfo STFInfo; typedef struct STFile STFile; @@ -220,12 +223,14 @@ int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte); int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence); int64_t tdWriteTFile(STFile *pTFile, void *buf, int64_t nbyte); int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset); +int64_t tdGetTFileSize(STFile *pTFile, int64_t *size); int32_t tdRemoveTFile(STFile *pTFile); int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo); int32_t tdUpdateTFileHeader(STFile *pTFile); void tdUpdateTFileMagic(STFile *pTFile, void *pCksm); void tdCloseTFile(STFile *pTFile); -void tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName); + +void tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index acf5b0b613..23d7665ba3 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -24,22 +24,25 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { if (tEncodeCStr(pCoder, pME->name) < 0) return -1; if (pME->type == TSDB_SUPER_TABLE) { + if (tEncodeI8(pCoder, pME->flags) < 0) return -1; // TODO: need refactor? if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaRow) < 0) return -1; if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1; + if (TABLE_IS_ROLLUP(pME->flags)) { + if (tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1; + } } else if (pME->type == TSDB_CHILD_TABLE) { if (tEncodeI64(pCoder, pME->ctbEntry.ctime) < 0) return -1; if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1; - if (tEncodeI32(pCoder, pME->ctbEntry.commentLen) < 0) return -1; + if (tEncodeI32v(pCoder, pME->ctbEntry.commentLen) < 0) return -1; if (pME->ctbEntry.commentLen > 0){ if (tEncodeCStr(pCoder, pME->ctbEntry.comment) < 0) return -1; } if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1; - debugCheckTags((STag*)pME->ctbEntry.pTags); // TODO: remove after debug if (tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags) < 0) return -1; } else if (pME->type == TSDB_NORMAL_TABLE) { if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1; if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1; - if (tEncodeI32(pCoder, pME->ntbEntry.commentLen) < 0) return -1; + if (tEncodeI32v(pCoder, pME->ntbEntry.commentLen) < 0) return -1; if (pME->ntbEntry.commentLen > 0){ if (tEncodeCStr(pCoder, pME->ntbEntry.comment) < 0) return -1; } @@ -64,23 +67,26 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { if (tDecodeCStr(pCoder, &pME->name) < 0) return -1; if (pME->type == TSDB_SUPER_TABLE) { + if (tDecodeI8(pCoder, &pME->flags) < 0) return -1; // TODO: need refactor? if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaRow) < 0) return -1; if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag) < 0) return -1; + if (TABLE_IS_ROLLUP(pME->flags)) { + if (tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1; + } } else if (pME->type == TSDB_CHILD_TABLE) { if (tDecodeI64(pCoder, &pME->ctbEntry.ctime) < 0) return -1; if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1; - if (tDecodeI32(pCoder, &pME->ctbEntry.commentLen) < 0) return -1; + if (tDecodeI32v(pCoder, &pME->ctbEntry.commentLen) < 0) return -1; if (pME->ctbEntry.commentLen > 0){ if (tDecodeCStr(pCoder, &pME->ctbEntry.comment) < 0) return -1; } if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1; if (tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags) < 0) return -1; // (TODO) - debugCheckTags((STag*)pME->ctbEntry.pTags); // TODO: remove after debug } else if (pME->type == TSDB_NORMAL_TABLE) { if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1; if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1; - if (tDecodeI32(pCoder, &pME->ntbEntry.commentLen) < 0) return -1; + if (tDecodeI32v(pCoder, &pME->ntbEntry.commentLen) < 0) return -1; if (pME->ntbEntry.commentLen > 0){ if (tDecodeCStr(pCoder, &pME->ntbEntry.comment) < 0) return -1; } diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index f57ee54400..85106f46c2 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -342,6 +342,7 @@ SMStbCursor *metaOpenStbCursor(SMeta *pMeta, tb_uid_t suid) { pStbCur = (SMStbCursor *)taosMemoryCalloc(1, sizeof(*pStbCur)); if (pStbCur == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -351,6 +352,7 @@ SMStbCursor *metaOpenStbCursor(SMeta *pMeta, tb_uid_t suid) { ret = tdbTbcOpen(pMeta->pSuidIdx, &pStbCur->pCur, NULL); if (ret < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; metaULock(pMeta); taosMemoryFree(pStbCur); return NULL; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index ea425ca7de..a621b4ddb0 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -139,6 +139,10 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { me.name = pReq->name; me.stbEntry.schemaRow = pReq->schemaRow; me.stbEntry.schemaTag = pReq->schemaTag; + if (pReq->rollup) { + TABLE_SET_ROLLUP(me.flags); + me.stbEntry.rsmaParam = pReq->rsmaParam; + } if (metaHandleEntry(pMeta, &me) < 0) goto _err; diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 1e8832615e..7f115633b9 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -156,6 +156,7 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS static void tdDestroyTSmaStat(STSmaStat *pStat) { if (pStat) { + smaDebug("destroy tsma stat"); tDestroyTSma(pStat->pTSma); taosMemoryFreeClear(pStat->pTSma); taosMemoryFreeClear(pStat->pTSchema); @@ -170,15 +171,12 @@ static void *tdFreeTSmaStat(STSmaStat *pStat) { static void tdDestroyRSmaStat(SRSmaStat *pStat) { if (pStat) { - smaDebug("vgId:%d, %s:%d free rsma stat", SMA_VID(pStat->pSma), __func__, __LINE__); + smaDebug("vgId:%d destroy rsma stat", SMA_VID(pStat->pSma)); // step 1: set persistence task cancelled atomic_store_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_CANCELLED); - // step 2: clean timer + // step 2: stop the persistence timer taosTmrStopA(&RSMA_TMR_ID(pStat)); - if (RSMA_TMR_HANDLE(pStat)) { - taosTmrCleanUp(RSMA_TMR_HANDLE(pStat)); - } // step 3: wait the persistence thread to finish int32_t nLoops = 0; @@ -194,7 +192,6 @@ static void tdDestroyRSmaStat(SRSmaStat *pStat) { sched_yield(); nLoops = 0; } - taosMsleep(1000); // TODO: remove this line when release } } @@ -219,7 +216,11 @@ static void tdDestroyRSmaStat(SRSmaStat *pStat) { sched_yield(); nLoops = 0; } - taosMsleep(1000); // TODO: remove this line when release + } + + // step 6: cleanup the timer handle + if (RSMA_TMR_HANDLE(pStat)) { + taosTmrCleanUp(RSMA_TMR_HANDLE(pStat)); } } } @@ -245,16 +246,12 @@ void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) { int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { if (pSmaStat) { if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { - smaDebug("%s:%d destroy tsma stat", __func__, __LINE__); tdDestroyTSmaStat(SMA_TSMA_STAT(pSmaStat)); } else if (smaType == TSDB_SMA_TYPE_ROLLUP) { - smaDebug("%s:%d destroy rsma stat", __func__, __LINE__); tdDestroyRSmaStat(SMA_RSMA_STAT(pSmaStat)); } else { ASSERT(0); } - } else { - smaDebug("%s:%d no need to destroy rsma stat", __func__, __LINE__); } return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index 2f40df8b45..88ed7426f7 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -18,6 +18,7 @@ static int32_t smaEvalDays(SRetention *r, int8_t precision); static int32_t smaSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type); +static int32_t rsmaRestore(SSma *pSma); #define SMA_SET_KEEP_CFG(l) \ do { \ @@ -100,6 +101,9 @@ int32_t smaOpen(SVnode *pVnode) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + + pVnode->pSma = pSma; + pSma->pVnode = pVnode; taosThreadMutexInit(&pSma->mutex, NULL); pSma->locked = false; @@ -117,17 +121,22 @@ int32_t smaOpen(SVnode *pVnode) { ASSERT(0); } } + + // restore the rsma +#if 0 + if (rsmaRestore(pSma) < 0) { + goto _err; + } +#endif } - pVnode->pSma = pSma; return 0; _err: - taosMemoryFreeClear(pSma); return -1; } int32_t smaCloseEnv(SSma *pSma) { - if(pSma) { + if (pSma) { SMA_TSMA_ENV(pSma) = tdFreeSmaEnv(SMA_TSMA_ENV(pSma)); SMA_RSMA_ENV(pSma) = tdFreeSmaEnv(SMA_RSMA_ENV(pSma)); } @@ -153,13 +162,12 @@ int32_t smaClose(SSma *pSma) { /** * @brief rsma env restore - * - * @param pSma - * @return int32_t + * + * @param pSma + * @return int32_t */ -int32_t smaRestore(SSma *pSma) { - if (!pSma) return 0; - // iterate all stables to restore the rsma env - - return TSDB_CODE_SUCCESS; +static int32_t rsmaRestore(SSma *pSma) { + ASSERT(VND_IS_RSMA(pSma->pVnode)); + + return tdProcessRSmaRestoreImpl(pSma); } \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 020ee38db9..ed5b6f4055 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -15,10 +15,14 @@ #include "sma.h" -#define RSMA_QTASK_PERSIST_MS 7200000 +#define RSMA_QTASKINFO_PERSIST_MS 7200000 +#define RSMA_QTASKINFO_BUFSIZE 32768 typedef enum { TD_QTASK_TMP_FILE = 0, TD_QTASK_CUR_FILE } TD_QTASK_FILE_T; static const char *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"}; +typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem; +typedef struct SRSmaQTaskFIter SRSmaQTaskFIter; + static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids); static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo *pRSmaInfo, SReadHandle *handle, @@ -27,6 +31,13 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType tb_uid_t suid, int8_t level); static void tdRSmaFetchTrigger(void *param, void *tmrId); static void tdRSmaPersistTrigger(void *param, void *tmrId); +static void *tdRSmaPersistExec(void *param); +static void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char *outputName); + +static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskFIter *pIter, STFile *pTFile); +static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskFIter *pIter, bool *isFinish); +static int32_t tdRSmaQTaskInfoIterNext(SRSmaQTaskFIter *pIter, SRSmaQTaskInfoItem *pItem, bool *isEnd); +static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *infoItem); struct SRSmaInfoItem { SRSmaInfo *pRsmaInfo; @@ -45,14 +56,38 @@ struct SRSmaInfo { SRSmaInfoItem items[TSDB_RETENTION_L2]; }; +struct SRSmaQTaskInfoItem { + int32_t len; + int8_t type; + int64_t suid; + void *qTaskInfo; +}; + +struct SRSmaQTaskFIter { + STFile *pTFile; + int64_t offset; + int64_t fsize; + int32_t nBytes; + int32_t nAlloc; + char *buf; + // ------------ + int32_t nBufPos; +}; + +static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) { + return lenWithHead - sizeof(int32_t) - sizeof(int8_t) - sizeof(int64_t); +} + +static FORCE_INLINE void tdRSmaQTaskInfoIterDestroy(SRSmaQTaskFIter *pIter) { taosMemoryFreeClear(pIter->buf); } + static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) { // Note: free/kill may in RC qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { - smaDebug("vgId:%d, %s:%d free qTaskInfo_t %p of level %d", vgId, __func__, __LINE__, otaskHandle, level); + smaDebug("vgId:%d, free qTaskInfo_t %p of level %d", vgId, otaskHandle, level); qDestroyTask(otaskHandle); } else { - smaDebug("vgId:%d, %s:%d not free qTaskInfo_t %p of level %d", vgId, __func__, __LINE__, otaskHandle, level); + smaDebug("vgId:%d, not free qTaskInfo_t %p of level %d", vgId, otaskHandle, level); } } @@ -89,7 +124,7 @@ static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) { return TSDB_CODE_SUCCESS; } -static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) { +static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) { SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; @@ -230,26 +265,21 @@ _err: } /** - * @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam. + * @brief for rsam create or restore * - * @param pTsdb - * @param pMeta - * @param pReq + * @param pSma + * @param param + * @param suid + * @param tbName * @return int32_t */ -int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { - SSma *pSma = pVnode->pSma; - if (!pReq->rollup) { - smaTrace("vgId:%d, return directly since no rollup for stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); - return TSDB_CODE_SUCCESS; - } - - SMeta *pMeta = pVnode->pMeta; - SMsgCb *pMsgCb = &pVnode->msgCb; - SRSmaParam *param = &pReq->pRSmaParam; +int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName) { + SVnode *pVnode = pSma->pVnode; + SMeta *pMeta = pVnode->pMeta; + SMsgCb *pMsgCb = &pVnode->msgCb; if ((param->qmsgLen[0] == 0) && (param->qmsgLen[1] == 0)) { - smaWarn("vgId:%d, no qmsg1/qmsg2 for rollup stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); + smaDebug("vgId:%d, no qmsg1/qmsg2 for rollup table %s %" PRIi64, SMA_VID(pSma), tbName, suid); return TSDB_CODE_SUCCESS; } @@ -262,10 +292,10 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; - pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t)); + pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (pRSmaInfo) { ASSERT(0); // TODO: free original pRSmaInfo is exists abnormally - smaWarn("vgId:%d, rsma info already exists for stb: %s, %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); + smaDebug("vgId:%d, rsma info already exists for table %s, %" PRIi64, SMA_VID(pSma), tbName, suid); return TSDB_CODE_SUCCESS; } @@ -289,14 +319,14 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { .vnode = pVnode, }; - STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), pReq->suid, -1); + STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), suid, -1); if (!pTSchema) { terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; goto _err; } pRSmaInfo->pTSchema = pTSchema; pRSmaInfo->pSma = pSma; - pRSmaInfo->suid = pReq->suid; + pRSmaInfo->suid = suid; if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 0) < 0) { goto _err; @@ -306,16 +336,16 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { goto _err; } - if (taosHashPut(RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) { + if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) { goto _err; } else { - smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), pReq->suid); + smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), suid); } // start the persist timer if (TASK_TRIGGER_STAT_INIT == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_INIT, TASK_TRIGGER_STAT_ACTIVE)) { - taosTmrStart(tdRSmaPersistTrigger, RSMA_QTASK_PERSIST_MS, pStat, RSMA_TMR_HANDLE(pStat)); + taosTmrStart(tdRSmaPersistTrigger, RSMA_QTASKINFO_PERSIST_MS, pStat, RSMA_TMR_HANDLE(pStat)); } return TSDB_CODE_SUCCESS; @@ -325,6 +355,24 @@ _err: return TSDB_CODE_FAILED; } +/** + * @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam. + * + * @param pTsdb + * @param pMeta + * @param pReq + * @return int32_t + */ +int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { + SSma *pSma = pVnode->pSma; + if (!pReq->rollup) { + smaTrace("vgId:%d, return directly since no rollup for stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); + return TSDB_CODE_SUCCESS; + } + + return tdProcessRSmaCreateImpl(pSma, &pReq->rsmaParam, pReq->suid, pReq->name); +} + /** * @brief store suid/[uids], prefer to use array and then hash * @@ -487,7 +535,7 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) #if 1 char flag[10] = {0}; snprintf(flag, 10, "level %" PRIi8, pItem->level); - blockDebugShowData(pResult, flag); + blockDebugShowDataBlocks(pResult, flag); #endif STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2); SSubmitReq *pReq = NULL; @@ -502,8 +550,10 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) } taosMemoryFreeClear(pReq); + } else if (terrno == 0) { + smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched yet", SMA_VID(pSma), pItem->level); } else { - smaDebug("vgId:%d, no rsma %" PRIi8 " data generated since %s", SMA_VID(pSma), pItem->level, tstrerror(terrno)); + smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched since %s", SMA_VID(pSma), pItem->level, tstrerror(terrno)); } tdDestroySDataBlockArray(pResult); @@ -526,16 +576,16 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat)); if (rsmaTriggerStat == TASK_TRIGGER_STAT_CANCELLED || rsmaTriggerStat == TASK_TRIGGER_STAT_FINISHED) { - smaDebug("vgId:%d, %s:%d level %" PRIi8 " not fetch since stat is cancelled for table suid:%" PRIi64, SMA_VID(pSma), - __func__, __LINE__, pItem->level, pItem->pRsmaInfo->suid); + smaDebug("vgId:%d, level %" PRIi8 " not fetch since stat is cancelled for table suid:%" PRIi64, SMA_VID(pSma), + pItem->level, pItem->pRsmaInfo->suid); return; } int8_t fetchTriggerStat = atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE); if (fetchTriggerStat == TASK_TRIGGER_STAT_ACTIVE) { - smaDebug("vgId:%d, %s:%d level %" PRIi8 " stat is active for table suid:%" PRIi64, SMA_VID(pSma), __func__, - __LINE__, pItem->level, pItem->pRsmaInfo->suid); + smaDebug("vgId:%d, level %" PRIi8 " stat is active for table suid:%" PRIi64, SMA_VID(pSma), pItem->level, + pItem->pRsmaInfo->suid); tdRefSmaStat(pSma, (SSmaStat *)pStat); @@ -546,13 +596,13 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { tdUnRefSmaStat(pSma, (SSmaStat *)pStat); } else { - smaDebug("vgId:%d, %s:%d level %" PRIi8 " stat is inactive for table suid:%" PRIi64, SMA_VID(pSma), __func__, - __LINE__, pItem->level, pItem->pRsmaInfo->suid); + smaDebug("vgId:%d, level %" PRIi8 " stat is inactive for table suid:%" PRIi64, SMA_VID(pSma), pItem->level, + pItem->pRsmaInfo->suid); } } -static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, - tb_uid_t suid, int8_t level) { +static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, tb_uid_t suid, + int8_t level) { if (!pItem || !pItem->taskInfo) { smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid); return TSDB_CODE_SUCCESS; @@ -568,7 +618,7 @@ static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int3 tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SUBMIT_BLOCK); atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE); - smaDebug("vgId:%d, %s:%d process rsma insert", SMA_VID(pSma), __func__, __LINE__); + smaDebug("vgId:%d, process rsma insert", SMA_VID(pSma)); SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pStat = SMA_RSMA_STAT(pEnv->pStat); @@ -645,7 +695,7 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { return TSDB_CODE_SUCCESS; } -void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char *outputName) { +static void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char *outputName) { tdGetVndFileName(vid, "rsma", tdQTaskInfoFname[ftype], outputName); } @@ -661,18 +711,6 @@ static void *tdRSmaPersistExec(void *param) { goto _end; } -#if 0 - SArray *suidList = taosArrayInit(1, sizeof(tb_uid_t)); - if (tsdbGetStbIdList(SMA_META(pSma), 0, suidList) < 0) { - ASSERT(0); - } else { - for (int32_t i = 0; i < taosArrayGetSize(suidList); ++i) { - tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); - smaDebug("suid [%d] is %" PRIi64, i, suid); - } - } -#endif - void *infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL); if (!infoHash) { goto _end; @@ -700,7 +738,7 @@ static void *tdRSmaPersistExec(void *param) { } char *pOutput = NULL; int32_t len = 0; - int8_t type = 0; + int8_t type = (int8_t)(i + 1); if (qSerializeTaskStatus(taskInfo, &pOutput, &len) < 0) { smaError("vgId:%d, table %" PRIi64 " level %d serialize rsma task failed since %s", vid, pRSmaInfo->suid, i + 1, terrstr(terrno)); @@ -736,8 +774,9 @@ static void *tdRSmaPersistExec(void *param) { isFileCreated = true; } - len += (sizeof(len) + sizeof(pRSmaInfo->suid)); + len += (sizeof(len) + sizeof(type) + sizeof(pRSmaInfo->suid)); tdAppendTFile(&tFile, &len, sizeof(len), &toffset); + tdAppendTFile(&tFile, &type, sizeof(type), &toffset); tdAppendTFile(&tFile, &pRSmaInfo->suid, sizeof(pRSmaInfo->suid), &toffset); tdAppendTFile(&tFile, pOutput, len, &toffset); @@ -833,25 +872,275 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) { if (TASK_TRIGGER_STAT_CANCELLED != atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_CANCELLED, TASK_TRIGGER_STAT_FINISHED)) { - smaDebug("%s:%d rsma persistence start since active", __func__, __LINE__); + smaDebug("rsma persistence start since active"); + + // start persist task tdRSmaPersistTask(pRSmaStat); - taosTmrReset(tdRSmaPersistTrigger, RSMA_QTASK_PERSIST_MS, pRSmaStat, pRSmaStat->tmrHandle, &pRSmaStat->tmrId); + + taosTmrReset(tdRSmaPersistTrigger, RSMA_QTASKINFO_PERSIST_MS, pRSmaStat, pRSmaStat->tmrHandle, + &pRSmaStat->tmrId); } else { atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); } } break; case TASK_TRIGGER_STAT_CANCELLED: { atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_FINISHED); - smaDebug("%s:%d rsma persistence not start since cancelled and finished", __func__, __LINE__); + smaDebug("rsma persistence not start since cancelled and finished"); } break; case TASK_TRIGGER_STAT_INACTIVE: { - smaDebug("%s:%d rsma persistence not start since inactive", __func__, __LINE__); + smaDebug("rsma persistence not start since inactive"); } break; case TASK_TRIGGER_STAT_INIT: { - smaDebug("%s:%d rsma persistence not start since init", __func__, __LINE__); + smaDebug("rsma persistence not start since init"); } break; default: { - smaWarn("%s:%d rsma persistence not start since unknown stat %" PRIi8, __func__, __LINE__, tmrStat); + smaWarn("rsma persistence not start since unknown stat %" PRIi8, tmrStat); + ASSERT(0); } break; } -} \ No newline at end of file +} + +int32_t tdProcessRSmaRestoreImpl(SSma *pSma) { + SVnode *pVnode = pSma->pVnode; + + // step 1: iterate all stables to restore the rsma env + + SArray *suidList = taosArrayInit(1, sizeof(tb_uid_t)); + if (tsdbGetStbIdList(SMA_META(pSma), 0, suidList) < 0) { + smaError("vgId:%d, failed to restore rsma since get stb id list error: %s", TD_VID(pVnode), terrstr()); + return TSDB_CODE_FAILED; + } + + if (taosArrayGetSize(suidList) == 0) { + smaDebug("vgId:%d no need to restore rsma since empty stb id list", TD_VID(pVnode)); + return TSDB_CODE_SUCCESS; + } + + SMetaReader mr = {0}; + metaReaderInit(&mr, SMA_META(pSma), 0); + for (int32_t i = 0; i < taosArrayGetSize(suidList); ++i) { + tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); + smaDebug("suid [%d] is %" PRIi64, i, suid); + if (metaGetTableEntryByUid(&mr, suid) < 0) { + smaError("vgId:%d failed to get table meta for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr()); + goto _err; + } + ASSERT(mr.me.type == TSDB_SUPER_TABLE); + ASSERT(mr.me.uid == suid); + if (TABLE_IS_ROLLUP(mr.me.flags)) { + SRSmaParam *param = &mr.me.stbEntry.rsmaParam; + for (int i = 0; i < 2; ++i) { + smaDebug("vgId: %d table:%" PRIi64 " maxdelay[%d]:%" PRIi64 " watermark[%d]:%" PRIi64, TD_VID(pSma->pVnode), + suid, i, param->maxdelay[i], i, param->watermark[i]); + } + if (tdProcessRSmaCreateImpl(pSma, &mr.me.stbEntry.rsmaParam, suid, mr.me.name) < 0) { + smaError("vgId:%d failed to retore rsma env for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr()); + goto _err; + } + } + } + + // step 2: retrieve qtaskinfo object from the rsma/qtaskinfo file and restore + STFile tFile = {0}; + char qTaskInfoFName[TSDB_FILENAME_LEN]; + + tdRSmaQTaskGetFName(TD_VID(pVnode), TD_QTASK_CUR_FILE, qTaskInfoFName); + if (tdInitTFile(&tFile, pVnode->pTfs, qTaskInfoFName) < 0) { + goto _err; + } + if (tdOpenTFile(&tFile, TD_FILE_READ) < 0) { + goto _err; + } + SRSmaQTaskFIter fIter = {0}; + if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) { + goto _err; + } + SRSmaQTaskInfoItem infoItem = {0}; + bool isEnd = false; + int32_t code = 0; + while ((code = tdRSmaQTaskInfoIterNext(&fIter, &infoItem, &isEnd)) == 0) { + if (isEnd) { + break; + } + if ((code = tdRSmaQTaskInfoItemRestore(pSma, &infoItem)) < 0) { + break; + } + } + tdRSmaQTaskInfoIterDestroy(&fIter); + + if (code < 0) { + goto _err; + } + + metaReaderClear(&mr); + taosArrayDestroy(suidList); + return TSDB_CODE_SUCCESS; +_err: + ASSERT(0); + metaReaderClear(&mr); + taosArrayDestroy(suidList); + smaError("failed to restore rsma info since %s", terrstr()); + return TSDB_CODE_FAILED; +} + +static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *infoItem) { + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)pSma->pRSmaEnv); + SRSmaInfo *pRSmaInfo = NULL; + void *qTaskInfo = NULL; + + pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &infoItem->suid, sizeof(infoItem->suid)); + + if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { + smaDebug("vgId:%d, no restore as no rsma info for suid:%" PRIu64, SMA_VID(pSma), infoItem->suid); + return TSDB_CODE_SUCCESS; + } + + if (infoItem->type == 1) { + qTaskInfo = pRSmaInfo->items[0].taskInfo; + } else if (infoItem->type == 2) { + qTaskInfo = pRSmaInfo->items[1].taskInfo; + } else { + ASSERT(0); + } + + if (!qTaskInfo) { + smaDebug("vgId:%d, no restore as NULL rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), infoItem->suid); + return TSDB_CODE_SUCCESS; + } + + if (qDeserializeTaskStatus(qTaskInfo, infoItem->qTaskInfo, infoItem->len) < 0) { + smaError("vgId:%d, restore rsma failed for suid:%" PRIi64 " level %d since %s", SMA_VID(pSma), infoItem->suid, + infoItem->type, terrstr(terrno)); + return TSDB_CODE_FAILED; + } + smaDebug("vgId:%d, restore rsma success for suid:%" PRIi64 " level %d", SMA_VID(pSma), infoItem->suid, + infoItem->type); + + return TSDB_CODE_SUCCESS; +} + +static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskFIter *pIter, STFile *pTFile) { + memset(pIter, 0, sizeof(*pIter)); + pIter->pTFile = pTFile; + pIter->offset = TD_FILE_HEAD_SIZE; + + if (tdGetTFileSize(pTFile, &pIter->fsize) < 0) { + return TSDB_CODE_FAILED; + } + + if ((pIter->fsize - TD_FILE_HEAD_SIZE) < RSMA_QTASKINFO_BUFSIZE) { + pIter->nAlloc = pIter->fsize - TD_FILE_HEAD_SIZE; + } else { + pIter->nAlloc = RSMA_QTASKINFO_BUFSIZE; + } + + if (pIter->nAlloc < TD_FILE_HEAD_SIZE) { + pIter->nAlloc = TD_FILE_HEAD_SIZE; + } + + pIter->buf = taosMemoryMalloc(pIter->nAlloc); + if (!pIter->buf) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskFIter *pIter, bool *isFinish) { + STFile *pTFile = pIter->pTFile; + int64_t nBytes = RSMA_QTASKINFO_BUFSIZE; + + if (pIter->offset >= pIter->fsize) { + *isFinish = true; + return TSDB_CODE_SUCCESS; + } + + if ((pIter->fsize - pIter->offset) < RSMA_QTASKINFO_BUFSIZE) { + nBytes = pIter->fsize - pIter->offset; + } + + if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET) < 0) { + ASSERT(0); + return TSDB_CODE_FAILED; + } + + if (tdReadTFile(pTFile, pIter->buf, nBytes) != nBytes) { + ASSERT(0); + return TSDB_CODE_FAILED; + } + + int32_t infoLen = 0; + taosDecodeFixedI32(pIter->buf, &infoLen); + if (infoLen > nBytes) { + ASSERT(infoLen > RSMA_QTASKINFO_BUFSIZE); + pIter->nAlloc = infoLen; + void *pBuf = taosMemoryRealloc(pIter->buf, infoLen); + if (!pBuf) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + pIter->buf = pBuf; + nBytes = infoLen; + + if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET)) { + ASSERT(0); + return TSDB_CODE_FAILED; + } + + if (tdReadTFile(pTFile, pIter->buf, nBytes) != nBytes) { + ASSERT(0); + return TSDB_CODE_FAILED; + } + } + + pIter->offset += nBytes; + pIter->nBytes = nBytes; + pIter->nBufPos = 0; + + return TSDB_CODE_SUCCESS; +} + +static int32_t tdRSmaQTaskInfoIterNext(SRSmaQTaskFIter *pIter, SRSmaQTaskInfoItem *pItem, bool *isEnd) { + while (1) { + // block iter + bool isFinish = false; + if (tdRSmaQTaskInfoIterNextBlock(pIter, &isFinish) < 0) { + ASSERT(0); + return TSDB_CODE_FAILED; + } + if (isFinish) { + *isEnd = true; + return TSDB_CODE_SUCCESS; + } + + // consume the block + int32_t qTaskInfoLenWithHead = 0; + pIter->buf = taosDecodeFixedI32(pIter->buf, &qTaskInfoLenWithHead); + if (qTaskInfoLenWithHead < 0) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return TSDB_CODE_FAILED; + } + while (1) { + if ((pIter->nBufPos + qTaskInfoLenWithHead) <= pIter->nBytes) { + pIter->buf = taosDecodeFixedI8(pIter->buf, &pItem->type); + pIter->buf = taosDecodeFixedI64(pIter->buf, &pItem->suid); + pItem->qTaskInfo = pIter->buf; + pItem->len = tdRSmaQTaskInfoContLen(qTaskInfoLenWithHead); + // do the restore job + printf("%s:%d ###### restore the qtask info offset:%" PRIi64 "\n", __func__, __LINE__, pIter->offset); + + pIter->buf = POINTER_SHIFT(pIter->buf, pItem->len); + pIter->nBufPos += qTaskInfoLenWithHead; + + pIter->buf = taosDecodeFixedI32(pIter->buf, &qTaskInfoLenWithHead); + continue; + } + // prepare and load next block in the file + pIter->offset -= (pIter->nBytes - pIter->nBufPos); + break; + } + } + + return TSDB_CODE_SUCCESS; +} diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 85fddebf55..1f60da0b0a 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -17,8 +17,6 @@ // smaFileUtil ================ -#define TD_FILE_HEAD_SIZE 512 - #define TD_FILE_STATE_OK 0 #define TD_FILE_STATE_BAD 1 @@ -71,6 +69,11 @@ int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence) { return loffset; } +int64_t tdGetTFileSize(STFile *pTFile, int64_t *size) { + ASSERT(TD_FILE_OPENED(pTFile)); + return taosFStatFile(pTFile->pFile, size, NULL); +} + int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte) { ASSERT(TD_FILE_OPENED(pTFile)); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index bd40fd4a1f..03f18cc766 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2886,6 +2886,9 @@ int32_t tsdbGetCtbIdList(SMeta* pMeta, int64_t suid, SArray* list) { */ int32_t tsdbGetStbIdList(SMeta* pMeta, int64_t suid, SArray* list) { SMStbCursor* pCur = metaOpenStbCursor(pMeta, suid); + if(!pCur) { + return TSDB_CODE_FAILED; + } while (1) { tb_uid_t id = metaStbCursorNext(pCur); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 2834e0a2e7..4d73dbc406 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -172,9 +172,9 @@ void vnodeClose(SVnode *pVnode) { vnodeSyncClose(pVnode); vnodeQueryClose(pVnode); walClose(pVnode->pWal); - smaCloseEx(pVnode->pSma); tqClose(pVnode->pTq); if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb); + smaCloseEx(pVnode->pSma); metaClose(pVnode->pMeta); vnodeCloseBufPool(pVnode); // destroy handle diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 88bdea3ae7..72c766e0ae 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -299,7 +299,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, int64_t version, SRpcMsg *pMsg, SRp void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { // TODO - blockDebugShowData(data, __func__); + blockDebugShowDataBlocks(data, __func__); tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data); } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 5b0dae00cf..1cc3f9b874 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -403,7 +403,7 @@ typedef struct SIntervalAggOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode SOptrBasicInfo binfo; // basic info SAggSupporter aggSup; // aggregate supporter - + SExprSupp scalarSupp; // supporter for perform scalar function SGroupResInfo groupResInfo; // multiple results build supporter SInterval interval; // interval info int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. @@ -730,15 +730,12 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode *pNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams, SSDataBlock* pInputBlock, - SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pColMatchColInfo, - SExecTaskInfo* pTaskInfo); +SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo, bool isStream); + STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream); SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, @@ -811,7 +808,7 @@ int32_t getMaximumIdleDurationSec(); * nOptrWithVal: *nOptrWithVal save the number of optr with value * return: result code, 0 means success */ -int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t *length); +int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t *length, int32_t *nOptrWithVal); /* * ops: root operator, created by caller diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h index 35bdd4ee55..972f4469e4 100644 --- a/source/libs/executor/inc/tsort.h +++ b/source/libs/executor/inc/tsort.h @@ -136,6 +136,13 @@ bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colId); */ void* tsortGetValue(STupleHandle* pVHandle, int32_t colId); +/** + * + * @param pVHandle + * @return + */ +uint64_t tsortGetGroupId(STupleHandle* pVHandle); + /** * * @param pSortHandle diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 374a3a736d..be6ca5145a 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -159,7 +159,12 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) { } SArray* createSortInfo(SNodeList* pNodeList) { - size_t numOfCols = LIST_LENGTH(pNodeList); + size_t numOfCols = 0; + if (pNodeList != NULL) { + numOfCols = LIST_LENGTH(pNodeList); + } else { + numOfCols = 0; + } SArray* pList = taosArrayInit(numOfCols, sizeof(SBlockOrderInfo)); if (pList == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index a9e1e03178..e4c0959185 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -222,7 +222,13 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) { return TSDB_CODE_INVALID_PARA; } - return encodeOperator(pTaskInfo->pRoot, pOutput, len); + int32_t nOptrWithVal = 0; + int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal); + if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal = 0)) { + taosMemoryFreeClear(*pOutput); + *len = 0; + } + return code; } int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 56cf6a5a72..17e5482b60 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4224,8 +4224,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; bool isStream = (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type); - pOptr = - createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pTaskInfo, isStream); + pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pIntervalPhyNode, + pTaskInfo, isStream); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) { SMergeAlignedIntervalPhysiNode* pIntervalPhyNode = (SMergeAlignedIntervalPhysiNode*)pPhyNode; @@ -4269,17 +4269,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createGroupSortOperatorInfo(ops[0], (SGroupSortPhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE == type) { SMergePhysiNode* pMergePhyNode = (SMergePhysiNode*)pPhyNode; - - SDataBlockDescNode* pDescNode = pPhyNode->pOutputDataBlockDesc; - SSDataBlock* pResBlock = createResDataBlock(pDescNode); - - SArray* sortInfo = createSortInfo(pMergePhyNode->pMergeKeys); - int32_t numOfOutputCols = 0; - SArray* pColList = - extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); - SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0); - SSDataBlock* pInputDataBlock = createResDataBlock(pChildNode->pOutputDataBlockDesc); - pOptr = createMultiwaySortMergeOperatorInfo(ops, size, pInputDataBlock, pResBlock, sortInfo, pColList, pTaskInfo); + pOptr = createMultiwayMergeOperatorInfo(ops, size, pMergePhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION == type) { SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; @@ -4472,12 +4462,12 @@ int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHa return 0; } -int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) { +int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length, int32_t* nOptrWithVal) { int32_t code = TDB_CODE_SUCCESS; char* pCurrent = NULL; int32_t currLength = 0; if (ops->fpSet.encodeResultRow) { - if (result == NULL || length == NULL) { + if (result == NULL || length == NULL || nOptrWithVal == NULL) { return TSDB_CODE_TSC_INVALID_INPUT; } code = ops->fpSet.encodeResultRow(ops, &pCurrent, &currLength); @@ -4488,8 +4478,13 @@ int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) { *result = NULL; } return code; + } else if (currLength == 0) { + ASSERT(!pCurrent); + goto _downstream; } - + + ++(*nOptrWithVal); + ASSERT(currLength >= 0); if (*result == NULL) { @@ -4516,8 +4511,10 @@ int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) { taosMemoryFree(pCurrent); *length = *(int32_t*)(*result); } + +_downstream: for (int32_t i = 0; i < ops->numOfDownstream; ++i) { - code = encodeOperator(ops->pDownstream[i], result, length); + code = encodeOperator(ops->pDownstream[i], result, length, nOptrWithVal); if (code != TDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ac57d3a9da..c51ef44154 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -448,7 +448,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) { qDebug("%s start to repeat ascending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) { STimeWindow* pWin = &pTableScanInfo->cond.twindows[i]; - qDebug("%s\t qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); + qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); } // do prepare for the next round table scan operation tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); @@ -467,7 +467,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) { qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) { STimeWindow* pWin = &pTableScanInfo->cond.twindows[i]; - qDebug("%s\t qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); + qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); } while (pTableScanInfo->scanTimes < total) { @@ -492,7 +492,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) { GET_TASKID(pTaskInfo)); for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) { STimeWindow* pWin = &pTableScanInfo->cond.twindows[i]; - qDebug("%s\t qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); + qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); } tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); pTableScanInfo->curTWinIdx = 0; @@ -2051,7 +2051,7 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle qDebug("no table qualified for query, TID:0x%" PRIx64 ", QID:0x%" PRIx64, taskId, queryId); return TSDB_CODE_SUCCESS; } - code = generateGroupIdMap(pTableListInfo, pHandle, pTableScanNode->pPartitionTags); + code = generateGroupIdMap(pTableListInfo, pHandle, pTableScanNode->pGroupTags); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2455,7 +2455,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN if (pInfo == NULL || pOperator == NULL) { goto _error; } - if (pTableScanNode->pPartitionTags) { + if (pTableScanNode->pGroupTags) { taosArraySort(pTableListInfo->pTableList, compareTableKeyInfoByGid); } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 09cc4e47be..44ff4c1c90 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -142,7 +142,8 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i SSDataBlock* loadNextDataBlock(void* param) { SOperatorInfo* pOperator = (SOperatorInfo*)param; - return pOperator->fpSet.getNextFn(pOperator); + SSDataBlock* pBlock = pOperator->fpSet.getNextFn(pOperator); + return pBlock; } // todo refactor: merged with fetch fp @@ -442,7 +443,7 @@ void destroyGroupSortOperatorInfo(void* param, int32_t numOfOutput) { SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo) { SGroupSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupSortOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL /* || rowSize > 100 * 1024 * 1024*/) { goto _error; } @@ -474,8 +475,8 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort pOperator->exprSupp.numOfExprs = numOfCols; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyGroupSortOperatorInfo, NULL, - NULL, getGroupSortExplainExecInfo); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyGroupSortOperatorInfo, + NULL, NULL, getGroupSortExplainExecInfo); int32_t code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -493,7 +494,7 @@ _error: //===================================================================================== // Multiway Sort Merge operator -typedef struct SMultiwaySortMergeOperatorInfo { +typedef struct SMultiwayMergeOperatorInfo { SOptrBasicInfo binfo; int32_t bufPageSize; @@ -503,14 +504,17 @@ typedef struct SMultiwaySortMergeOperatorInfo { SSortHandle* pSortHandle; SArray* pColMatchInfo; // for index map from table scan output - SSDataBlock* pInputBlock; - int64_t startTs; // sort start time - uint64_t groupId; -} SMultiwaySortMergeOperatorInfo; + SSDataBlock* pInputBlock; + int64_t startTs; // sort start time + bool groupSort; + bool hasGroupId; + uint64_t groupId; + STupleHandle* prefetchedTuple; +} SMultiwayMergeOperatorInfo; -int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { - SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; +int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) { + SMultiwayMergeOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; if (OPTR_IS_OPENED(pOperator)) { return TSDB_CODE_SUCCESS; @@ -524,8 +528,9 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { pInfo->pInputBlock, pTaskInfo->id.str); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, NULL, NULL); - tsortSetCompareGroupId(pInfo->pSortHandle, true); - + + tsortSetCompareGroupId(pInfo->pSortHandle, pInfo->groupSort); + for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) { SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); ps->param = pOperator->pDownstream[i]; @@ -547,8 +552,8 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo, SOperatorInfo* pOperator) { - SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SMultiwayMergeOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; blockDataCleanup(pDataBlock); @@ -560,18 +565,47 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData blockDataEnsureCapacity(p, capacity); while (1) { - STupleHandle* pTupleHandle = tsortNextTuple(pHandle); + STupleHandle* pTupleHandle = NULL; + if (pInfo->groupSort) { + if (pInfo->prefetchedTuple == NULL) { + pTupleHandle = tsortNextTuple(pHandle); + } else { + pTupleHandle = pInfo->prefetchedTuple; + pInfo->groupId = tsortGetGroupId(pTupleHandle); + pInfo->prefetchedTuple = NULL; + } + } + else { + pTupleHandle = tsortNextTuple(pHandle); + pInfo->groupId = 0; + } + if (pTupleHandle == NULL) { break; } - appendOneRowToDataBlock(p, pTupleHandle); + if (pInfo->groupSort) + { + uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle); + if (!pInfo->hasGroupId) { + pInfo->groupId = tupleGroupId; + pInfo->hasGroupId = true; + appendOneRowToDataBlock(p, pTupleHandle); + } else if (pInfo->groupId == tupleGroupId) { + appendOneRowToDataBlock(p, pTupleHandle); + } else { + pInfo->prefetchedTuple = pTupleHandle; + break; + } + } else { + appendOneRowToDataBlock(p, pTupleHandle); + } if (p->info.rows >= capacity) { break; } } - if (p->info.rows > 0) {// todo extract method + if (p->info.rows > 0) { // todo extract method blockDataEnsureCapacity(pDataBlock, p->info.rows); int32_t numOfCols = taosArrayGetSize(pColMatchInfo); for (int32_t i = 0; i < numOfCols; ++i) { @@ -593,13 +627,13 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; } -SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) { +SSDataBlock* doMultiwayMerge(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SMultiwayMergeOperatorInfo* pInfo = pOperator->info; int32_t code = pOperator->fpSet._openFn(pOperator); if (code != TSDB_CODE_SUCCESS) { @@ -608,7 +642,6 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) { SSDataBlock* pBlock = getMultiwaySortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo, pOperator); - if (pBlock != NULL) { pOperator->resultInfo.totalRows += pBlock->info.rows; } else { @@ -617,8 +650,8 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) { return pBlock; } -void destroyMultiwaySortMergeOperatorInfo(void* param, int32_t numOfOutput) { - SMultiwaySortMergeOperatorInfo* pInfo = (SMultiwaySortMergeOperatorInfo*)param; +void destroyMultiwayMergeOperatorInfo(void* param, int32_t numOfOutput) { + SMultiwayMergeOperatorInfo* pInfo = (SMultiwayMergeOperatorInfo*)param; pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); pInfo->pInputBlock = blockDataDestroy(pInfo->pInputBlock); @@ -626,11 +659,11 @@ void destroyMultiwaySortMergeOperatorInfo(void* param, int32_t numOfOutput) { taosArrayDestroy(pInfo->pColMatchInfo); } -int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { +int32_t getMultiwayMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { ASSERT(pOptr != NULL); SSortExecInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo)); - SMultiwaySortMergeOperatorInfo* pOperatorInfo = (SMultiwaySortMergeOperatorInfo*)pOptr->info; + SMultiwayMergeOperatorInfo* pOperatorInfo = (SMultiwayMergeOperatorInfo*)pOptr->info; *pInfo = tsortGetSortExecInfo(pOperatorInfo->pSortHandle); *pOptrExplain = pInfo; @@ -638,24 +671,35 @@ int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrEx return TSDB_CODE_SUCCESS; } -SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams, - SSDataBlock* pInputBlock, SSDataBlock* pResBlock, SArray* pSortInfo, - SArray* pColMatchColInfo, SExecTaskInfo* pTaskInfo) { - SMultiwaySortMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwaySortMergeOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - int32_t rowSize = pResBlock->info.rowSize; +SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size_t numStreams, + SMergePhysiNode* pMergePhyNode, SExecTaskInfo* pTaskInfo) { + SPhysiNode* pPhyNode = (SPhysiNode*)pMergePhyNode; + + SMultiwayMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwayMergeOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SDataBlockDescNode* pDescNode = pPhyNode->pOutputDataBlockDesc; + SSDataBlock* pResBlock = createResDataBlock(pDescNode); + + int32_t rowSize = pResBlock->info.rowSize; if (pInfo == NULL || pOperator == NULL || rowSize > 100 * 1024 * 1024) { goto _error; } + SArray* pSortInfo = createSortInfo(pMergePhyNode->pMergeKeys); + int32_t numOfOutputCols = 0; + SArray* pColMatchColInfo = + extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); + SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0); + SSDataBlock* pInputBlock = createResDataBlock(pChildNode->pOutputDataBlockDesc); initResultSizeInfo(pOperator, 1024); + pInfo->groupSort = pMergePhyNode->groupSort; pInfo->binfo.pRes = pResBlock; pInfo->pSortInfo = pSortInfo; pInfo->pColMatchInfo = pColMatchColInfo; pInfo->pInputBlock = pInputBlock; - pOperator->name = "MultiwaySortMerge"; + pOperator->name = "MultiwayMerge"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE; pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; @@ -667,9 +711,8 @@ SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, // one additional is reserved for merged result. pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1); - pOperator->fpSet = - createOperatorFpSet(doOpenMultiwaySortMergeOperator, doMultiwaySortMerge, NULL, NULL, - destroyMultiwaySortMergeOperatorInfo, NULL, NULL, getMultiwaySortMergeExplainExecInfo); + pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL, NULL, + destroyMultiwayMergeOperatorInfo, NULL, NULL, getMultiwayMergeExplainExecInfo); int32_t code = appendDownstream(pOperator, downStreams, numStreams); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index b7d39207d8..963e714972 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -969,6 +969,12 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { getTableScanInfo(pOperator, &pInfo->order, &scanFlag); + if (pInfo->scalarSupp.pExprInfo != NULL) { + SExprSupp* pExprSup =& pInfo->scalarSupp; + projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, + pExprSup->numOfExprs, NULL); + } + // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, scanFlag, true); hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag, NULL); @@ -1381,6 +1387,11 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { continue; } + if (pInfo->scalarSupp.pExprInfo != NULL) { + SExprSupp* pExprSup = &pInfo->scalarSupp; + projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + } + // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the // caller. Note that all the time window are not close till now. // the pDataBlock are always the same one, no need to call this again @@ -1498,18 +1509,27 @@ void increaseTs(SqlFunctionCtx* pCtx) { SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo, bool isStream) { + STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream) { SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->win = pTaskInfo->window; - pInfo->order = TSDB_ORDER_ASC; - pInfo->interval = *pInterval; + pInfo->win = pTaskInfo->window; + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; pInfo->execModel = pTaskInfo->execModel; - pInfo->twAggSup = *pTwAggSupp; + pInfo->twAggSup = *pTwAggSupp; + + if (pPhyNode->window.pExprs != NULL) { + int32_t numOfScalar = 0; + SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar); + int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } pInfo->primaryTsIndex = primaryTsSlotId; @@ -2473,7 +2493,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { if (IS_FINAL_OP(pInfo)) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); - SIntervalAggOperatorInfo* pChildInfo = pChildOp->info; + SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info; SExprSupp* pChildSup = &pChildOp->exprSupp; doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildInfo->primaryTsIndex, diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 50474ff62e..7d44d41b55 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -752,6 +752,10 @@ void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) { } } +uint64_t tsortGetGroupId(STupleHandle* pVHandle) { + return pVHandle->pBlock->info.groupId; +} + SSortExecInfo tsortGetSortExecInfo(SSortHandle* pHandle) { SSortExecInfo info = {0}; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 983cffe9dc..364ee0692f 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -110,7 +110,7 @@ static void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf); static int32_t udfdConnectToMnode(); static int32_t udfdLoadUdf(char *udfName, SUdf *udf); -static bool udfdRpcRfp(int32_t code); +static bool udfdRpcRfp(int32_t code, tmsg_t msgType); static int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static int32_t udfdOpenClientRpc(); static int32_t udfdCloseClientRpc(); @@ -546,9 +546,12 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { } return 0; } -static bool udfdRpcRfp(int32_t code) { +static bool udfdRpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { + if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) { + return false; + } return true; } else { return false; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index ac197912cf..9d8baf472b 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -351,7 +351,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_SCALAR_FIELD(watermark); COPY_SCALAR_FIELD(tsColId); COPY_SCALAR_FIELD(filesFactor); - CLONE_NODE_LIST_FIELD(pPartTags); + CLONE_NODE_LIST_FIELD(pGroupTags); return TSDB_CODE_SUCCESS; } @@ -401,6 +401,7 @@ static int32_t logicMergeCopy(const SMergeLogicNode* pSrc, SMergeLogicNode* pDst CLONE_NODE_LIST_FIELD(pInputs); COPY_SCALAR_FIELD(numOfChannels); COPY_SCALAR_FIELD(srcGroupId); + COPY_SCALAR_FIELD(groupSort); return TSDB_CODE_SUCCESS; } @@ -436,6 +437,7 @@ static int32_t logicFillCopy(const SFillLogicNode* pSrc, SFillLogicNode* pDst) { static int32_t logicSortCopy(const SSortLogicNode* pSrc, SSortLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pSortKeys); + COPY_SCALAR_FIELD(groupSort); return TSDB_CODE_SUCCESS; } @@ -500,7 +502,7 @@ static int32_t physiTableScanCopy(const STableScanPhysiNode* pSrc, STableScanPhy COPY_SCALAR_FIELD(ratio); COPY_SCALAR_FIELD(dataRequired); CLONE_NODE_LIST_FIELD(pDynamicScanFuncs); - CLONE_NODE_LIST_FIELD(pPartitionTags); + CLONE_NODE_LIST_FIELD(pGroupTags); COPY_SCALAR_FIELD(interval); COPY_SCALAR_FIELD(offset); COPY_SCALAR_FIELD(sliding); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 514d65d52e..df7429bd88 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -234,6 +234,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiMerge"; case QUERY_NODE_PHYSICAL_PLAN_SORT: return "PhysiSort"; + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: + return "PhysiGroupSort"; case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: return "PhysiHashInterval"; case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: @@ -539,7 +541,7 @@ static const char* jkScanLogicPlanScanPseudoCols = "ScanPseudoCols"; static const char* jkScanLogicPlanTableId = "TableId"; static const char* jkScanLogicPlanTableType = "TableType"; static const char* jkScanLogicPlanTagCond = "TagCond"; -static const char* jkScanLogicPlanPartTags = "PartTags"; +static const char* jkScanLogicPlanGroupTags = "GroupTags"; static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { const SScanLogicNode* pNode = (const SScanLogicNode*)pObj; @@ -561,7 +563,7 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddObject(pJson, jkScanLogicPlanTagCond, nodeToJson, pNode->pTagCond); } if (TSDB_CODE_SUCCESS == code) { - code = nodeListToJson(pJson, jkScanLogicPlanPartTags, pNode->pPartTags); + code = nodeListToJson(pJson, jkScanLogicPlanGroupTags, pNode->pGroupTags); } return code; @@ -588,7 +590,7 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { code = jsonToNodeObject(pJson, jkScanLogicPlanTagCond, &pNode->pTagCond); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeList(pJson, jkScanLogicPlanPartTags, &pNode->pPartTags); + code = jsonToNodeList(pJson, jkScanLogicPlanGroupTags, &pNode->pGroupTags); } return code; @@ -1430,7 +1432,8 @@ static const char* jkTableScanPhysiPlanTriggerType = "triggerType"; static const char* jkTableScanPhysiPlanWatermark = "watermark"; static const char* jkTableScanPhysiPlanTsColId = "tsColId"; static const char* jkTableScanPhysiPlanFilesFactor = "FilesFactor"; -static const char* jkTableScanPhysiPlanPartitionTags = "PartitionTags"; +static const char* jkTableScanPhysiPlanGroupTags = "GroupTags"; +static const char* jkTableScanPhysiPlanGroupSort = "GroupSort"; static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj; @@ -1485,7 +1488,10 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddDoubleToObject(pJson, jkTableScanPhysiPlanFilesFactor, pNode->filesFactor); } if (TSDB_CODE_SUCCESS == code) { - code = nodeListToJson(pJson, jkTableScanPhysiPlanPartitionTags, pNode->pPartitionTags); + code = nodeListToJson(pJson, jkTableScanPhysiPlanGroupTags, pNode->pGroupTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanGroupSort, pNode->groupSort); } return code; @@ -1544,7 +1550,10 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { code = tjsonGetDoubleValue(pJson, jkTableScanPhysiPlanFilesFactor, &pNode->filesFactor); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeList(pJson, jkTableScanPhysiPlanPartitionTags, &pNode->pPartitionTags); + code = jsonToNodeList(pJson, jkTableScanPhysiPlanGroupTags, &pNode->pGroupTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanGroupSort, &pNode->groupSort); } return code; @@ -1725,6 +1734,7 @@ static const char* jkMergePhysiPlanMergeKeys = "MergeKeys"; static const char* jkMergePhysiPlanTargets = "Targets"; static const char* jkMergePhysiPlanNumOfChannels = "NumOfChannels"; static const char* jkMergePhysiPlanSrcGroupId = "SrcGroupId"; +static const char* jkMergePhysiPlanGroupSort = "GroupSort"; static int32_t physiMergeNodeToJson(const void* pObj, SJson* pJson) { const SMergePhysiNode* pNode = (const SMergePhysiNode*)pObj; @@ -1742,6 +1752,9 @@ static int32_t physiMergeNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkMergePhysiPlanSrcGroupId, pNode->srcGroupId); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkMergePhysiPlanGroupSort, pNode->groupSort); + } return code; } @@ -1762,6 +1775,9 @@ static int32_t jsonToPhysiMergeNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkMergePhysiPlanSrcGroupId, &pNode->srcGroupId); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkMergePhysiPlanGroupSort, &pNode->groupSort); + } return code; } @@ -3369,6 +3385,7 @@ static const char* jkSlotDescSlotId = "SlotId"; static const char* jkSlotDescDataType = "DataType"; static const char* jkSlotDescReserve = "Reserve"; static const char* jkSlotDescOutput = "Output"; +static const char* jkSlotDescName = "Name"; static int32_t slotDescNodeToJson(const void* pObj, SJson* pJson) { const SSlotDescNode* pNode = (const SSlotDescNode*)pObj; @@ -3383,6 +3400,9 @@ static int32_t slotDescNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkSlotDescOutput, pNode->output); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkSlotDescName, pNode->name); + } return code; } @@ -3400,6 +3420,9 @@ static int32_t jsonToSlotDescNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkSlotDescOutput, &pNode->output); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkSlotDescName, pNode->name); + } return code; } @@ -4137,6 +4160,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN_MERGE: return physiMergeNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_SORT: + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: return physiSortNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: @@ -4280,6 +4304,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { case QUERY_NODE_PHYSICAL_PLAN_MERGE: return jsonToPhysiMergeNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_SORT: + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: return jsonToPhysiSortNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index 1fb2db9f23..3747dde9ed 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -500,7 +500,8 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk } break; } - case QUERY_NODE_PHYSICAL_PLAN_SORT: { + case QUERY_NODE_PHYSICAL_PLAN_SORT: + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: { SSortPhysiNode* pSort = (SSortPhysiNode*)pNode; res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext); if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 734f287cfb..be6faa92cb 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -288,6 +288,8 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SMergePhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_SORT: return makeNode(type, sizeof(SSortPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: + return makeNode(type, sizeof(SGroupSortPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: return makeNode(type, sizeof(SIntervalPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: @@ -709,7 +711,7 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pLogicNode->pTagCond); nodesDestroyNode(pLogicNode->pTagIndexCond); taosArrayDestroy(pLogicNode->pSmaIndexes); - nodesDestroyList(pLogicNode->pPartTags); + nodesDestroyList(pLogicNode->pGroupTags); break; } case QUERY_NODE_LOGIC_PLAN_JOIN: { @@ -813,7 +815,7 @@ void nodesDestroyNode(SNode* pNode) { STableScanPhysiNode* pPhyNode = (STableScanPhysiNode*)pNode; destroyScanPhysiNode((SScanPhysiNode*)pNode); nodesDestroyList(pPhyNode->pDynamicScanFuncs); - nodesDestroyList(pPhyNode->pPartitionTags); + nodesDestroyList(pPhyNode->pGroupTags); break; } case QUERY_NODE_PHYSICAL_PLAN_PROJECT: { @@ -850,7 +852,8 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyList(pPhyNode->pTargets); break; } - case QUERY_NODE_PHYSICAL_PLAN_SORT: { + case QUERY_NODE_PHYSICAL_PLAN_SORT: + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: { SSortPhysiNode* pPhyNode = (SSortPhysiNode*)pNode; destroyPhysiNode((SPhysiNode*)pPhyNode); nodesDestroyList(pPhyNode->pExprs); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b68a4cc568..45240536e3 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2456,90 +2456,6 @@ static int32_t rewriteUniqueStmt(STranslateContext* pCxt, SSelectStmt* pSelect) return cxt.pTranslateCxt->errCode; } -typedef struct SRwriteTailCxt { - STranslateContext* pTranslateCxt; - int64_t limit; - int64_t offset; -} SRwriteTailCxt; - -static EDealRes rewriteTailFunc(SNode** pNode, void* pContext) { - SRwriteTailCxt* pCxt = pContext; - if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { - SFunctionNode* pFunc = (SFunctionNode*)*pNode; - if (FUNCTION_TYPE_TAIL == pFunc->funcType) { - pCxt->limit = ((SValueNode*)nodesListGetNode(pFunc->pParameterList, 1))->datum.i; - if (3 == LIST_LENGTH(pFunc->pParameterList)) { - pCxt->offset = ((SValueNode*)nodesListGetNode(pFunc->pParameterList, 2))->datum.i; - } - SNode* pExpr = nodesListGetNode(pFunc->pParameterList, 0); - strcpy(((SExprNode*)pExpr)->aliasName, ((SExprNode*)*pNode)->aliasName); - NODES_CLEAR_LIST(pFunc->pParameterList); - nodesDestroyNode(*pNode); - *pNode = pExpr; - return DEAL_RES_IGNORE_CHILD; - } - } - return DEAL_RES_CONTINUE; -} - -static int32_t createLimieNode(SRwriteTailCxt* pCxt, SLimitNode** pOutput) { - *pOutput = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT); - if (NULL == *pOutput) { - return TSDB_CODE_OUT_OF_MEMORY; - } - (*pOutput)->limit = pCxt->limit; - (*pOutput)->offset = pCxt->offset; - return TSDB_CODE_SUCCESS; -} - -static SNode* createOrderByExpr(STranslateContext* pCxt) { - SOrderByExprNode* pOrder = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); - if (NULL == pOrder) { - return NULL; - } - pCxt->errCode = createPrimaryKeyCol(pCxt, &pOrder->pExpr); - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - nodesDestroyNode((SNode*)pOrder); - return NULL; - } - pOrder->order = ORDER_DESC; - pOrder->nullOrder = NULL_ORDER_FIRST; - return (SNode*)pOrder; -} - -/* case 1: - * in: select tail(expr, k, f) from t where_clause - * out: select expr from t where_clause order by _rowts desc limit k offset f - * - * case 2: - * in: select tail(expr, k, f) from t where_clause partition_by_clause - * out: select expr from t where_clause partition_by_clause sort by _rowts desc limit k offset f - * - * case 3: - * in: select tail(expr, k, f) from t where_clause order_by_clause limit_clause - * out: select expr from ( - * select expr from t where_clause order by _rowts desc limit k offset f - * ) order_by_clause limit_clause - * - * case 4: - * in: select tail(expr, k, f) from t where_clause partition_by_clause limit_clause - * out: - */ -static int32_t rewriteTailStmt(STranslateContext* pCxt, SSelectStmt* pSelect) { - if (!pSelect->hasTailFunc) { - return TSDB_CODE_SUCCESS; - } - - SRwriteTailCxt cxt = {.pTranslateCxt = pCxt, .limit = -1, .offset = -1}; - nodesRewriteExprs(pSelect->pProjectionList, rewriteTailFunc, &cxt); - int32_t code = nodesListMakeStrictAppend(&pSelect->pOrderByList, createOrderByExpr(pCxt)); - if (TSDB_CODE_SUCCESS == code) { - code = createLimieNode(&cxt, &pSelect->pLimit); - } - pSelect->hasIndefiniteRowsFunc = false; - return code; -} - typedef struct SReplaceOrderByAliasCxt { STranslateContext* pTranslateCxt; SNodeList* pProjectionList; @@ -2616,9 +2532,6 @@ static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect if (TSDB_CODE_SUCCESS == code) { code = rewriteUniqueStmt(pCxt, pSelect); } - // if (TSDB_CODE_SUCCESS == code) { - // code = rewriteTailStmt(pCxt, pSelect); - // } if (TSDB_CODE_SUCCESS == code) { code = rewriteTimelineFunc(pCxt, pSelect); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index c4f4624355..202dd96581 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -337,7 +337,11 @@ int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) { static bool isValidateTag(char* input) { if (!input) return false; for (size_t i = 0; i < strlen(input); ++i) { + #ifdef WINDOWS + if (input[i] < 0x20 || input[i] > 0x7E) return false; + #else if (isprint(input[i]) == 0) return false; + #endif } return true; } @@ -377,6 +381,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, SMs char* jsonKey = item->string; if (!isValidateTag(jsonKey)) { + fprintf(stdout,"%s(%d) %s %08" PRId64 "\n", __FILE__, __LINE__,__func__,taosGetSelfPthreadId());fflush(stdout); retCode = buildSyntaxErrMsg(pMsgBuf, "json key not validate", jsonKey); goto end; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index a4cdcd35d3..396a7b6193 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -507,6 +507,8 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt return TSDB_CODE_OUT_OF_MEMORY; } + pIdfRowsFunc->isTailFunc = pSelect->hasTailFunc; + // indefinite rows functions and _select_values functions int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsVectorFunc, &pIdfRowsFunc->pFuncs); if (TSDB_CODE_SUCCESS == code) { @@ -733,6 +735,8 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect return TSDB_CODE_OUT_OF_MEMORY; } + pSort->groupSort = pSelect->groupSort; + int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, COLLECT_COL_TYPE_ALL, &pSort->node.pTargets); if (TSDB_CODE_SUCCESS == code && NULL == pSort->node.pTargets) { code = nodesListMakeStrictAppend(&pSort->node.pTargets, diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index f1cad6010c..53c9ef8dd0 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -20,9 +20,8 @@ #define OPTIMIZE_FLAG_MASK(n) (1 << n) -#define OPTIMIZE_FLAG_OSD OPTIMIZE_FLAG_MASK(0) -#define OPTIMIZE_FLAG_CPD OPTIMIZE_FLAG_MASK(1) -#define OPTIMIZE_FLAG_OPK OPTIMIZE_FLAG_MASK(2) +#define OPTIMIZE_FLAG_SCAN_PATH OPTIMIZE_FLAG_MASK(0) +#define OPTIMIZE_FLAG_PUSH_DOWN_CONDE OPTIMIZE_FLAG_MASK(1) #define OPTIMIZE_FLAG_SET_MASK(val, mask) (val) |= (mask) #define OPTIMIZE_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0) @@ -76,7 +75,7 @@ static SLogicNode* optFindPossibleNode(SLogicNode* pNode, FMayBeOptimized func) return NULL; } -EDealRes osdHaveNormalColImpl(SNode* pNode, void* pContext) { +EDealRes scanPathOptHaveNormalColImpl(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { // *((bool*)pContext) = (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType); *((bool*)pContext) = true; @@ -85,14 +84,14 @@ EDealRes osdHaveNormalColImpl(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static bool osdHaveNormalCol(SNodeList* pList) { +static bool scanPathOptHaveNormalCol(SNodeList* pList) { bool res = false; - nodesWalkExprsPostOrder(pList, osdHaveNormalColImpl, &res); + nodesWalkExprsPostOrder(pList, scanPathOptHaveNormalColImpl, &res); return res; } -static bool osdMayBeOptimized(SLogicNode* pNode) { - if (OPTIMIZE_FLAG_TEST_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_OSD)) { +static bool scanPathOptMayBeOptimized(SLogicNode* pNode) { + if (OPTIMIZE_FLAG_TEST_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_SCAN_PATH)) { return false; } if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode)) { @@ -108,10 +107,10 @@ static bool osdMayBeOptimized(SLogicNode* pNode) { QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent->pParent))) { return true; } - return !osdHaveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys); + return !scanPathOptHaveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys); } -static SNodeList* osdGetAllFuncs(SLogicNode* pNode) { +static SNodeList* scanPathOptGetAllFuncs(SLogicNode* pNode) { switch (nodeType(pNode)) { case QUERY_NODE_LOGIC_PLAN_WINDOW: return ((SWindowLogicNode*)pNode)->pFuncs; @@ -123,7 +122,7 @@ static SNodeList* osdGetAllFuncs(SLogicNode* pNode) { return NULL; } -static bool needOptimizeDataRequire(const SFunctionNode* pFunc) { +static bool scanPathOptNeedOptimizeDataRequire(const SFunctionNode* pFunc) { if (!fmIsSpecialDataRequiredFunc(pFunc->funcId)) { return false; } @@ -136,7 +135,7 @@ static bool needOptimizeDataRequire(const SFunctionNode* pFunc) { return true; } -static bool needOptimizeDynamicScan(const SFunctionNode* pFunc) { +static bool scanPathOptNeedDynOptimize(const SFunctionNode* pFunc) { if (!fmIsDynamicScanOptimizedFunc(pFunc->funcId)) { return false; } @@ -149,17 +148,17 @@ static bool needOptimizeDynamicScan(const SFunctionNode* pFunc) { return true; } -static int32_t osdGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs, SNodeList** pDsoFuncs) { - SNodeList* pAllFuncs = osdGetAllFuncs(pScan->node.pParent); +static int32_t scanPathOptGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs, SNodeList** pDsoFuncs) { + SNodeList* pAllFuncs = scanPathOptGetAllFuncs(pScan->node.pParent); SNodeList* pTmpSdrFuncs = NULL; SNodeList* pTmpDsoFuncs = NULL; SNode* pFunc = NULL; bool otherFunc = false; FOREACH(pFunc, pAllFuncs) { int32_t code = TSDB_CODE_SUCCESS; - if (needOptimizeDataRequire((SFunctionNode*)pFunc)) { + if (scanPathOptNeedOptimizeDataRequire((SFunctionNode*)pFunc)) { code = nodesListMakeStrictAppend(&pTmpSdrFuncs, nodesCloneNode(pFunc)); - } else if (needOptimizeDynamicScan((SFunctionNode*)pFunc)) { + } else if (scanPathOptNeedDynOptimize((SFunctionNode*)pFunc)) { code = nodesListMakeStrictAppend(&pTmpDsoFuncs, nodesCloneNode(pFunc)); } else { otherFunc = true; @@ -180,15 +179,15 @@ static int32_t osdGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs, return TSDB_CODE_SUCCESS; } -static int32_t osdMatch(SOptimizeContext* pCxt, SLogicNode* pLogicNode, SOsdInfo* pInfo) { - pInfo->pScan = (SScanLogicNode*)optFindPossibleNode(pLogicNode, osdMayBeOptimized); +static int32_t scanPathOptMatch(SOptimizeContext* pCxt, SLogicNode* pLogicNode, SOsdInfo* pInfo) { + pInfo->pScan = (SScanLogicNode*)optFindPossibleNode(pLogicNode, scanPathOptMayBeOptimized); if (NULL == pInfo->pScan) { return TSDB_CODE_SUCCESS; } - return osdGetRelatedFuncs(pInfo->pScan, &pInfo->pSdrFuncs, &pInfo->pDsoFuncs); + return scanPathOptGetRelatedFuncs(pInfo->pScan, &pInfo->pSdrFuncs, &pInfo->pDsoFuncs); } -static EFuncDataRequired osdPromoteDataRequired(EFuncDataRequired l, EFuncDataRequired r) { +static EFuncDataRequired scanPathOptPromoteDataRequired(EFuncDataRequired l, EFuncDataRequired r) { switch (l) { case FUNC_DATA_REQUIRED_DATA_LOAD: return l; @@ -202,19 +201,19 @@ static EFuncDataRequired osdPromoteDataRequired(EFuncDataRequired l, EFuncDataRe return r; } -static int32_t osdGetDataRequired(SNodeList* pFuncs) { +static int32_t scanPathOptGetDataRequired(SNodeList* pFuncs) { if (NULL == pFuncs) { return FUNC_DATA_REQUIRED_DATA_LOAD; } EFuncDataRequired dataRequired = FUNC_DATA_REQUIRED_FILTEROUT; SNode* pFunc = NULL; FOREACH(pFunc, pFuncs) { - dataRequired = osdPromoteDataRequired(dataRequired, fmFuncDataRequired((SFunctionNode*)pFunc, NULL)); + dataRequired = scanPathOptPromoteDataRequired(dataRequired, fmFuncDataRequired((SFunctionNode*)pFunc, NULL)); } return dataRequired; } -static void setScanWindowInfo(SScanLogicNode* pScan) { +static void scanPathOptSetScanWin(SScanLogicNode* pScan) { SLogicNode* pParent = pScan->node.pParent; if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pParent) && pParent->pParent && QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pParent->pParent)) { @@ -233,23 +232,23 @@ static void setScanWindowInfo(SScanLogicNode* pScan) { } } -static int32_t osdOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { +static int32_t scanPathOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { SOsdInfo info = {0}; - int32_t code = osdMatch(pCxt, pLogicSubplan->pNode, &info); + int32_t code = scanPathOptMatch(pCxt, pLogicSubplan->pNode, &info); if (TSDB_CODE_SUCCESS == code && info.pScan) { - setScanWindowInfo((SScanLogicNode*)info.pScan); + scanPathOptSetScanWin((SScanLogicNode*)info.pScan); } if (TSDB_CODE_SUCCESS == code && (NULL != info.pDsoFuncs || NULL != info.pSdrFuncs)) { - info.pScan->dataRequired = osdGetDataRequired(info.pSdrFuncs); + info.pScan->dataRequired = scanPathOptGetDataRequired(info.pSdrFuncs); info.pScan->pDynamicScanFuncs = info.pDsoFuncs; - OPTIMIZE_FLAG_SET_MASK(info.pScan->node.optimizedFlag, OPTIMIZE_FLAG_OSD); + OPTIMIZE_FLAG_SET_MASK(info.pScan->node.optimizedFlag, OPTIMIZE_FLAG_SCAN_PATH); pCxt->optimized = true; } nodesDestroyList(info.pSdrFuncs); return code; } -static int32_t cpdMergeCond(SNode** pDst, SNode** pSrc) { +static int32_t pushDownCondOptMergeCond(SNode** pDst, SNode** pSrc) { SLogicConditionNode* pLogicCond = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); if (NULL == pLogicCond) { return TSDB_CODE_OUT_OF_MEMORY; @@ -270,7 +269,7 @@ static int32_t cpdMergeCond(SNode** pDst, SNode** pSrc) { return code; } -static int32_t cpdCondAppend(SNode** pCond, SNode** pAdditionalCond) { +static int32_t pushDownCondOptAppendCond(SNode** pCond, SNode** pAdditionalCond) { if (NULL == *pCond) { TSWAP(*pCond, *pAdditionalCond); return TSDB_CODE_SUCCESS; @@ -283,16 +282,16 @@ static int32_t cpdCondAppend(SNode** pCond, SNode** pAdditionalCond) { *pAdditionalCond = NULL; } } else { - code = cpdMergeCond(pCond, pAdditionalCond); + code = pushDownCondOptMergeCond(pCond, pAdditionalCond); } return code; } -static int32_t cpdCalcTimeRange(SOptimizeContext* pCxt, SScanLogicNode* pScan, SNode** pPrimaryKeyCond, - SNode** pOtherCond) { +static int32_t pushDownCondOptCalcTimeRange(SOptimizeContext* pCxt, SScanLogicNode* pScan, SNode** pPrimaryKeyCond, + SNode** pOtherCond) { int32_t code = TSDB_CODE_SUCCESS; if (pCxt->pPlanCxt->topicQuery || pCxt->pPlanCxt->streamQuery) { - code = cpdCondAppend(pOtherCond, pPrimaryKeyCond); + code = pushDownCondOptAppendCond(pOtherCond, pPrimaryKeyCond); } else { bool isStrict = false; code = filterGetTimeRange(*pPrimaryKeyCond, &pScan->scanRange, &isStrict); @@ -300,7 +299,7 @@ static int32_t cpdCalcTimeRange(SOptimizeContext* pCxt, SScanLogicNode* pScan, S if (isStrict) { nodesDestroyNode(*pPrimaryKeyCond); } else { - code = cpdCondAppend(pOtherCond, pPrimaryKeyCond); + code = pushDownCondOptAppendCond(pOtherCond, pPrimaryKeyCond); } *pPrimaryKeyCond = NULL; } @@ -308,8 +307,9 @@ static int32_t cpdCalcTimeRange(SOptimizeContext* pCxt, SScanLogicNode* pScan, S return code; } -static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan) { - if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD) || +static int32_t pushDownCondOptDealScan(SOptimizeContext* pCxt, SScanLogicNode* pScan) { + if (NULL == pScan->node.pConditions || + OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE) || TSDB_SYSTEM_TABLE == pScan->tableType) { return TSDB_CODE_SUCCESS; } @@ -319,14 +319,14 @@ static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* int32_t code = nodesPartitionCond(&pScan->node.pConditions, &pPrimaryKeyCond, &pScan->pTagIndexCond, &pScan->pTagCond, &pOtherCond); if (TSDB_CODE_SUCCESS == code && NULL != pPrimaryKeyCond) { - code = cpdCalcTimeRange(pCxt, pScan, &pPrimaryKeyCond, &pOtherCond); + code = pushDownCondOptCalcTimeRange(pCxt, pScan, &pPrimaryKeyCond, &pOtherCond); } if (TSDB_CODE_SUCCESS == code) { pScan->node.pConditions = pOtherCond; } if (TSDB_CODE_SUCCESS == code) { - OPTIMIZE_FLAG_SET_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD); + OPTIMIZE_FLAG_SET_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE); pCxt->optimized = true; } else { nodesDestroyNode(pPrimaryKeyCond); @@ -336,7 +336,7 @@ static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* return code; } -static bool cpdBelongThisTable(SNode* pCondCol, SNodeList* pTableCols) { +static bool pushDownCondOptBelongThisTable(SNode* pCondCol, SNodeList* pTableCols) { SNode* pTableCol = NULL; FOREACH(pTableCol, pTableCols) { if (nodesEqualNode(pCondCol, pTableCol)) { @@ -346,12 +346,12 @@ static bool cpdBelongThisTable(SNode* pCondCol, SNodeList* pTableCols) { return false; } -static EDealRes cpdIsMultiTableCondImpl(SNode* pNode, void* pContext) { +static EDealRes pushDownCondOptIsCrossTableCond(SNode* pNode, void* pContext) { SCpdIsMultiTableCondCxt* pCxt = pContext; if (QUERY_NODE_COLUMN == nodeType(pNode)) { - if (cpdBelongThisTable(pNode, pCxt->pLeftCols)) { + if (pushDownCondOptBelongThisTable(pNode, pCxt->pLeftCols)) { pCxt->havaLeftCol = true; - } else if (cpdBelongThisTable(pNode, pCxt->pRightCols)) { + } else if (pushDownCondOptBelongThisTable(pNode, pCxt->pRightCols)) { pCxt->haveRightCol = true; } return pCxt->havaLeftCol && pCxt->haveRightCol ? DEAL_RES_END : DEAL_RES_CONTINUE; @@ -359,10 +359,11 @@ static EDealRes cpdIsMultiTableCondImpl(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static ECondAction cpdCondAction(EJoinType joinType, SNodeList* pLeftCols, SNodeList* pRightCols, SNode* pNode) { +static ECondAction pushDownCondOptGetCondAction(EJoinType joinType, SNodeList* pLeftCols, SNodeList* pRightCols, + SNode* pNode) { SCpdIsMultiTableCondCxt cxt = { .pLeftCols = pLeftCols, .pRightCols = pRightCols, .havaLeftCol = false, .haveRightCol = false}; - nodesWalkExpr(pNode, cpdIsMultiTableCondImpl, &cxt); + nodesWalkExpr(pNode, pushDownCondOptIsCrossTableCond, &cxt); return (JOIN_TYPE_INNER != joinType ? COND_ACTION_STAY : (cxt.havaLeftCol && cxt.haveRightCol @@ -370,8 +371,8 @@ static ECondAction cpdCondAction(EJoinType joinType, SNodeList* pLeftCols, SNode : (cxt.havaLeftCol ? COND_ACTION_PUSH_LEFT_CHILD : COND_ACTION_PUSH_RIGHT_CHILD))); } -static int32_t cpdPartitionLogicCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, - SNode** pRightChildCond) { +static int32_t pushDownCondOptPartLogicCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, + SNode** pRightChildCond) { SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pJoin->node.pConditions; if (LOGIC_COND_TYPE_AND != pLogicCond->condType) { return TSDB_CODE_SUCCESS; @@ -387,7 +388,7 @@ static int32_t cpdPartitionLogicCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNo SNodeList* pRemainConds = NULL; SNode* pCond = NULL; FOREACH(pCond, pLogicCond->pParameterList) { - ECondAction condAction = cpdCondAction(pJoin->joinType, pLeftCols, pRightCols, pCond); + ECondAction condAction = pushDownCondOptGetCondAction(pJoin->joinType, pLeftCols, pRightCols, pCond); if (COND_ACTION_PUSH_JOIN == condAction) { code = nodesListMakeAppend(&pOnConds, nodesCloneNode(pCond)); } else if (COND_ACTION_PUSH_LEFT_CHILD == condAction) { @@ -439,11 +440,12 @@ static int32_t cpdPartitionLogicCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNo return code; } -static int32_t cpdPartitionOpCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, - SNode** pRightChildCond) { +static int32_t pushDownCondOptPartOpCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, + SNode** pRightChildCond) { SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; - ECondAction condAction = cpdCondAction(pJoin->joinType, pLeftCols, pRightCols, pJoin->node.pConditions); + ECondAction condAction = + pushDownCondOptGetCondAction(pJoin->joinType, pLeftCols, pRightCols, pJoin->node.pConditions); if (COND_ACTION_STAY == condAction) { return TSDB_CODE_SUCCESS; } else if (COND_ACTION_PUSH_JOIN == condAction) { @@ -457,35 +459,35 @@ static int32_t cpdPartitionOpCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode* return TSDB_CODE_SUCCESS; } -static int32_t cpdPartitionCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, - SNode** pRightChildCond) { +static int32_t pushDownCondOptPartCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, + SNode** pRightChildCond) { if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->node.pConditions)) { - return cpdPartitionLogicCond(pJoin, pOnCond, pLeftChildCond, pRightChildCond); + return pushDownCondOptPartLogicCond(pJoin, pOnCond, pLeftChildCond, pRightChildCond); } else { - return cpdPartitionOpCond(pJoin, pOnCond, pLeftChildCond, pRightChildCond); + return pushDownCondOptPartOpCond(pJoin, pOnCond, pLeftChildCond, pRightChildCond); } } -static int32_t cpdPushCondToOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode** pCond) { - return cpdCondAppend(&pJoin->pOnConditions, pCond); +static int32_t pushDownCondOptPushCondToOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode** pCond) { + return pushDownCondOptAppendCond(&pJoin->pOnConditions, pCond); } -static int32_t cpdPushCondToScan(SOptimizeContext* pCxt, SScanLogicNode* pScan, SNode** pCond) { - return cpdCondAppend(&pScan->node.pConditions, pCond); +static int32_t pushDownCondOptPushCondToScan(SOptimizeContext* pCxt, SScanLogicNode* pScan, SNode** pCond) { + return pushDownCondOptAppendCond(&pScan->node.pConditions, pCond); } -static int32_t cpdPushCondToChild(SOptimizeContext* pCxt, SLogicNode* pChild, SNode** pCond) { +static int32_t pushDownCondOptPushCondToChild(SOptimizeContext* pCxt, SLogicNode* pChild, SNode** pCond) { switch (nodeType(pChild)) { case QUERY_NODE_LOGIC_PLAN_SCAN: - return cpdPushCondToScan(pCxt, (SScanLogicNode*)pChild, pCond); + return pushDownCondOptPushCondToScan(pCxt, (SScanLogicNode*)pChild, pCond); default: break; } - planError("cpdPushCondToChild failed, invalid logic plan node %s", nodesNodeName(nodeType(pChild))); + planError("pushDownCondOptPushCondToChild failed, invalid logic plan node %s", nodesNodeName(nodeType(pChild))); return TSDB_CODE_PLAN_INTERNAL_ERROR; } -static bool cpdIsPrimaryKey(SNode* pNode, SNodeList* pTableCols) { +static bool pushDownCondOptIsPriKey(SNode* pNode, SNodeList* pTableCols) { if (QUERY_NODE_COLUMN != nodeType(pNode)) { return false; } @@ -493,10 +495,10 @@ static bool cpdIsPrimaryKey(SNode* pNode, SNodeList* pTableCols) { if (PRIMARYKEY_TIMESTAMP_COL_ID != pCol->colId) { return false; } - return cpdBelongThisTable(pNode, pTableCols); + return pushDownCondOptBelongThisTable(pNode, pTableCols); } -static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { +static bool pushDownCondOptIsPriKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { if (QUERY_NODE_OPERATOR != nodeType(pCond)) { return false; } @@ -508,15 +510,15 @@ static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; - if (cpdIsPrimaryKey(pOper->pLeft, pLeftCols)) { - return cpdIsPrimaryKey(pOper->pRight, pRightCols); - } else if (cpdIsPrimaryKey(pOper->pLeft, pRightCols)) { - return cpdIsPrimaryKey(pOper->pRight, pLeftCols); + if (pushDownCondOptIsPriKey(pOper->pLeft, pLeftCols)) { + return pushDownCondOptIsPriKey(pOper->pRight, pRightCols); + } else if (pushDownCondOptIsPriKey(pOper->pLeft, pRightCols)) { + return pushDownCondOptIsPriKey(pOper->pRight, pLeftCols); } return false; } -static bool cpdContainPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { +static bool pushDownCondOptContainPriKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { if (QUERY_NODE_LOGIC_CONDITION == nodeType(pCond)) { SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pCond; if (LOGIC_COND_TYPE_AND != pLogicCond->condType) { @@ -525,54 +527,56 @@ static bool cpdContainPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { bool hasPrimaryKeyEqualCond = false; SNode* pCond = NULL; FOREACH(pCond, pLogicCond->pParameterList) { - if (cpdContainPrimaryKeyEqualCond(pJoin, pCond)) { + if (pushDownCondOptContainPriKeyEqualCond(pJoin, pCond)) { hasPrimaryKeyEqualCond = true; break; } } return hasPrimaryKeyEqualCond; } else { - return cpdIsPrimaryKeyEqualCond(pJoin, pCond); + return pushDownCondOptIsPriKeyEqualCond(pJoin, pCond); } } -static int32_t cpdCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { +static int32_t pushDownCondOptCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (NULL == pJoin->pOnConditions) { return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN); } - if (!cpdContainPrimaryKeyEqualCond(pJoin, pJoin->pOnConditions)) { + if (!pushDownCondOptContainPriKeyEqualCond(pJoin, pJoin->pOnConditions)) { return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); } return TSDB_CODE_SUCCESS; } -static int32_t cpdPushJoinCondition(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { - if (OPTIMIZE_FLAG_TEST_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_CPD)) { +static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { + if (OPTIMIZE_FLAG_TEST_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) { return TSDB_CODE_SUCCESS; } if (NULL == pJoin->node.pConditions) { - return cpdCheckJoinOnCond(pCxt, pJoin); + return pushDownCondOptCheckJoinOnCond(pCxt, pJoin); } SNode* pOnCond = NULL; SNode* pLeftChildCond = NULL; SNode* pRightChildCond = NULL; - int32_t code = cpdPartitionCond(pJoin, &pOnCond, &pLeftChildCond, &pRightChildCond); + int32_t code = pushDownCondOptPartCond(pJoin, &pOnCond, &pLeftChildCond, &pRightChildCond); if (TSDB_CODE_SUCCESS == code && NULL != pOnCond) { - code = cpdPushCondToOnCond(pCxt, pJoin, &pOnCond); + code = pushDownCondOptPushCondToOnCond(pCxt, pJoin, &pOnCond); } if (TSDB_CODE_SUCCESS == code && NULL != pLeftChildCond) { - code = cpdPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0), &pLeftChildCond); + code = + pushDownCondOptPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0), &pLeftChildCond); } if (TSDB_CODE_SUCCESS == code && NULL != pRightChildCond) { - code = cpdPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1), &pRightChildCond); + code = + pushDownCondOptPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1), &pRightChildCond); } if (TSDB_CODE_SUCCESS == code) { - OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_CPD); + OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE); pCxt->optimized = true; - code = cpdCheckJoinOnCond(pCxt, pJoin); + code = pushDownCondOptCheckJoinOnCond(pCxt, pJoin); } else { nodesDestroyNode(pOnCond); nodesDestroyNode(pLeftChildCond); @@ -582,22 +586,22 @@ static int32_t cpdPushJoinCondition(SOptimizeContext* pCxt, SJoinLogicNode* pJoi return code; } -static int32_t cpdPushAggCondition(SOptimizeContext* pCxt, SAggLogicNode* pAgg) { +static int32_t pushDownCondOptDealAgg(SOptimizeContext* pCxt, SAggLogicNode* pAgg) { // todo return TSDB_CODE_SUCCESS; } -static int32_t cpdPushCondition(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { +static int32_t pushDownCondOptimizeImpl(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pLogicNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: - code = cpdOptimizeScanCondition(pCxt, (SScanLogicNode*)pLogicNode); + code = pushDownCondOptDealScan(pCxt, (SScanLogicNode*)pLogicNode); break; case QUERY_NODE_LOGIC_PLAN_JOIN: - code = cpdPushJoinCondition(pCxt, (SJoinLogicNode*)pLogicNode); + code = pushDownCondOptDealJoin(pCxt, (SJoinLogicNode*)pLogicNode); break; case QUERY_NODE_LOGIC_PLAN_AGG: - code = cpdPushAggCondition(pCxt, (SAggLogicNode*)pLogicNode); + code = pushDownCondOptDealAgg(pCxt, (SAggLogicNode*)pLogicNode); break; default: break; @@ -605,7 +609,7 @@ static int32_t cpdPushCondition(SOptimizeContext* pCxt, SLogicNode* pLogicNode) if (TSDB_CODE_SUCCESS == code) { SNode* pChild = NULL; FOREACH(pChild, pLogicNode->pChildren) { - code = cpdPushCondition(pCxt, (SLogicNode*)pChild); + code = pushDownCondOptimizeImpl(pCxt, (SLogicNode*)pChild); if (TSDB_CODE_SUCCESS != code) { break; } @@ -614,11 +618,11 @@ static int32_t cpdPushCondition(SOptimizeContext* pCxt, SLogicNode* pLogicNode) return code; } -static int32_t cpdOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { - return cpdPushCondition(pCxt, pLogicSubplan->pNode); +static int32_t pushDownCondOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + return pushDownCondOptimizeImpl(pCxt, pLogicSubplan->pNode); } -static bool opkIsPrimaryKeyOrderBy(SNodeList* pSortKeys) { +static bool sortPriKeyOptIsPriKeyOrderBy(SNodeList* pSortKeys) { if (1 != LIST_LENGTH(pSortKeys)) { return false; } @@ -626,17 +630,18 @@ static bool opkIsPrimaryKeyOrderBy(SNodeList* pSortKeys) { return (QUERY_NODE_COLUMN == nodeType(pNode) ? (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) : false); } -static bool opkSortMayBeOptimized(SLogicNode* pNode) { +static bool sortPriKeyOptMayBeOptimized(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SORT != nodeType(pNode)) { return false; } - if (OPTIMIZE_FLAG_TEST_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_OPK)) { - return false; + SSortLogicNode* pSort = (SSortLogicNode*)pNode; + if (pSort->groupSort || !sortPriKeyOptIsPriKeyOrderBy(pSort->pSortKeys) || 1 != LIST_LENGTH(pSort->node.pChildren)) { + return TSDB_CODE_SUCCESS; } return true; } -static int32_t opkGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeList** pScanNodes) { +static int32_t sortPriKeyOptGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeList** pScanNodes) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pNode)) { @@ -646,9 +651,11 @@ static int32_t opkGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeL } break; case QUERY_NODE_LOGIC_PLAN_JOIN: - code = opkGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes); + code = + sortPriKeyOptGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes); if (TSDB_CODE_SUCCESS == code) { - code = opkGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 1), pNotOptimize, pScanNodes); + code = + sortPriKeyOptGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 1), pNotOptimize, pScanNodes); } return code; case QUERY_NODE_LOGIC_PLAN_AGG: @@ -663,77 +670,58 @@ static int32_t opkGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeL return TSDB_CODE_SUCCESS; } - return opkGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes); + return sortPriKeyOptGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes); } -static int32_t opkGetScanNodes(SLogicNode* pNode, SNodeList** pScanNodes) { +static int32_t sortPriKeyOptGetScanNodes(SLogicNode* pNode, SNodeList** pScanNodes) { bool notOptimize = false; - int32_t code = opkGetScanNodesImpl(pNode, ¬Optimize, pScanNodes); + int32_t code = sortPriKeyOptGetScanNodesImpl(pNode, ¬Optimize, pScanNodes); if (TSDB_CODE_SUCCESS != code || notOptimize) { nodesClearList(*pScanNodes); } return code; } -static EOrder opkGetPrimaryKeyOrder(SSortLogicNode* pSort) { +static EOrder sortPriKeyOptGetPriKeyOrder(SSortLogicNode* pSort) { return ((SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0))->order; } -static SNode* opkRewriteDownNode(SSortLogicNode* pSort) { - SNode* pDownNode = nodesListGetNode(pSort->node.pChildren, 0); - // todo - NODES_CLEAR_LIST(pSort->node.pChildren); - return pDownNode; -} - -static int32_t opkDoOptimized(SOptimizeContext* pCxt, SSortLogicNode* pSort, SNodeList* pScanNodes) { - EOrder order = opkGetPrimaryKeyOrder(pSort); +static int32_t sortPriKeyOptApply(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SSortLogicNode* pSort, + SNodeList* pScanNodes) { + EOrder order = sortPriKeyOptGetPriKeyOrder(pSort); if (ORDER_DESC == order) { SNode* pScan = NULL; FOREACH(pScan, pScanNodes) { TSWAP(((SScanLogicNode*)pScan)->scanSeq[0], ((SScanLogicNode*)pScan)->scanSeq[1]); } } - if (NULL == pSort->node.pParent) { - // todo - return TSDB_CODE_SUCCESS; + int32_t code = + replaceLogicNode(pLogicSubplan, (SLogicNode*)pSort, (SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0)); + if (TSDB_CODE_SUCCESS == code) { + NODES_CLEAR_LIST(pSort->node.pChildren); + nodesDestroyNode((SNode*)pSort); } - - SNode* pDownNode = opkRewriteDownNode(pSort); - SNode* pNode; - FOREACH(pNode, pSort->node.pParent->pChildren) { - if (nodesEqualNode(pNode, (SNode*)pSort)) { - REPLACE_NODE(pDownNode); - ((SLogicNode*)pDownNode)->pParent = pSort->node.pParent; - break; - } - } - nodesDestroyNode((SNode*)pSort); - return TSDB_CODE_SUCCESS; + return code; } -static int32_t opkOptimizeImpl(SOptimizeContext* pCxt, SSortLogicNode* pSort) { - OPTIMIZE_FLAG_SET_MASK(pSort->node.optimizedFlag, OPTIMIZE_FLAG_OPK); - if (!opkIsPrimaryKeyOrderBy(pSort->pSortKeys) || 1 != LIST_LENGTH(pSort->node.pChildren)) { - return TSDB_CODE_SUCCESS; - } +static int32_t sortPrimaryKeyOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SSortLogicNode* pSort) { SNodeList* pScanNodes = NULL; - int32_t code = opkGetScanNodes((SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0), &pScanNodes); + int32_t code = sortPriKeyOptGetScanNodes((SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0), &pScanNodes); if (TSDB_CODE_SUCCESS == code && NULL != pScanNodes) { - code = opkDoOptimized(pCxt, pSort, pScanNodes); + code = sortPriKeyOptApply(pCxt, pLogicSubplan, pSort, pScanNodes); } nodesClearList(pScanNodes); return code; } -static int32_t opkOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { - SSortLogicNode* pSort = (SSortLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, opkSortMayBeOptimized); +static int32_t sortPrimaryKeyOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SSortLogicNode* pSort = (SSortLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, sortPriKeyOptMayBeOptimized); if (NULL == pSort) { return TSDB_CODE_SUCCESS; } - return opkOptimizeImpl(pCxt, pSort); + return sortPrimaryKeyOptimizeImpl(pCxt, pLogicSubplan, pSort); } -static bool smaOptMayBeOptimized(SLogicNode* pNode) { +static bool smaIndexOptMayBeOptimized(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode) || NULL == pNode->pParent || QUERY_NODE_LOGIC_PLAN_WINDOW != nodeType(pNode->pParent) || WINDOW_TYPE_INTERVAL != ((SWindowLogicNode*)pNode->pParent)->winType) { @@ -748,7 +736,8 @@ static bool smaOptMayBeOptimized(SLogicNode* pNode) { return true; } -static int32_t smaOptCreateMerge(SLogicNode* pChild, SNodeList* pMergeKeys, SNodeList* pTargets, SLogicNode** pOutput) { +static int32_t smaIndexOptCreateMerge(SLogicNode* pChild, SNodeList* pMergeKeys, SNodeList* pTargets, + SLogicNode** pOutput) { SMergeLogicNode* pMerge = (SMergeLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_MERGE); if (NULL == pMerge) { return TSDB_CODE_OUT_OF_MEMORY; @@ -767,8 +756,8 @@ static int32_t smaOptCreateMerge(SLogicNode* pChild, SNodeList* pMergeKeys, SNod return TSDB_CODE_SUCCESS; } -static int32_t smaOptRecombinationNode(SLogicSubplan* pLogicSubplan, SLogicNode* pInterval, SLogicNode* pMerge, - SLogicNode* pSmaScan) { +static int32_t smaIndexOptRecombinationNode(SLogicSubplan* pLogicSubplan, SLogicNode* pInterval, SLogicNode* pMerge, + SLogicNode* pSmaScan) { int32_t code = nodesListMakeAppend(&pMerge->pChildren, (SNode*)pInterval); if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeAppend(&pMerge->pChildren, (SNode*)pSmaScan); @@ -781,8 +770,8 @@ static int32_t smaOptRecombinationNode(SLogicSubplan* pLogicSubplan, SLogicNode* return code; } -static int32_t smaOptCreateSmaScan(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList* pCols, - SLogicNode** pOutput) { +static int32_t smaIndexOptCreateSmaScan(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList* pCols, + SLogicNode** pOutput) { SScanLogicNode* pSmaScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN); if (NULL == pSmaScan) { return TSDB_CODE_OUT_OF_MEMORY; @@ -811,7 +800,7 @@ static int32_t smaOptCreateSmaScan(SScanLogicNode* pScan, STableIndexInfo* pInde return TSDB_CODE_SUCCESS; } -static bool smaOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pWindow, STableIndexInfo* pIndex) { +static bool smaIndexOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pWindow, STableIndexInfo* pIndex) { if (pWindow->interval != pIndex->interval || pWindow->intervalUnit != pIndex->intervalUnit || pWindow->offset != pIndex->offset || pWindow->sliding != pIndex->sliding || pWindow->slidingUnit != pIndex->slidingUnit) { @@ -831,7 +820,7 @@ static bool smaOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pWindow return true; } -static SNode* smaOptCreateSmaCol(SNode* pFunc, uint64_t tableId, int32_t colId) { +static SNode* smaIndexOptCreateSmaCol(SNode* pFunc, uint64_t tableId, int32_t colId) { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { return NULL; @@ -846,7 +835,7 @@ static SNode* smaOptCreateSmaCol(SNode* pFunc, uint64_t tableId, int32_t colId) return (SNode*)pCol; } -static int32_t smaOptFindSmaFunc(SNode* pQueryFunc, SNodeList* pSmaFuncs) { +static int32_t smaIndexOptFindSmaFunc(SNode* pQueryFunc, SNodeList* pSmaFuncs) { int32_t index = 0; SNode* pSmaFunc = NULL; FOREACH(pSmaFunc, pSmaFuncs) { @@ -858,8 +847,8 @@ static int32_t smaOptFindSmaFunc(SNode* pQueryFunc, SNodeList* pSmaFuncs) { return -1; } -static int32_t smaOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, SNodeList** pOutput, - int32_t* pWStrartIndex) { +static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, SNodeList** pOutput, + int32_t* pWStrartIndex) { SNodeList* pCols = NULL; SNode* pFunc = NULL; int32_t code = TSDB_CODE_SUCCESS; @@ -870,11 +859,11 @@ static int32_t smaOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeLis if (FUNCTION_TYPE_WSTARTTS == ((SFunctionNode*)pFunc)->funcType) { *pWStrartIndex = index; } - smaFuncIndex = smaOptFindSmaFunc(pFunc, pSmaFuncs); + smaFuncIndex = smaIndexOptFindSmaFunc(pFunc, pSmaFuncs); if (smaFuncIndex < 0) { break; } else { - code = nodesListMakeStrictAppend(&pCols, smaOptCreateSmaCol(pFunc, tableId, smaFuncIndex + 2)); + code = nodesListMakeStrictAppend(&pCols, smaIndexOptCreateSmaCol(pFunc, tableId, smaFuncIndex + 2)); if (TSDB_CODE_SUCCESS != code) { break; } @@ -891,22 +880,22 @@ static int32_t smaOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeLis return code; } -static int32_t smaOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList** pCols, - int32_t* pWStrartIndex) { +static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList** pCols, + int32_t* pWStrartIndex) { SWindowLogicNode* pWindow = (SWindowLogicNode*)pScan->node.pParent; - if (!smaOptEqualInterval(pScan, pWindow, pIndex)) { + if (!smaIndexOptEqualInterval(pScan, pWindow, pIndex)) { return TSDB_CODE_SUCCESS; } SNodeList* pSmaFuncs = NULL; int32_t code = nodesStringToList(pIndex->expr, &pSmaFuncs); if (TSDB_CODE_SUCCESS == code) { - code = smaOptCreateSmaCols(pWindow->pFuncs, pIndex->dstTbUid, pSmaFuncs, pCols, pWStrartIndex); + code = smaIndexOptCreateSmaCols(pWindow->pFuncs, pIndex->dstTbUid, pSmaFuncs, pCols, pWStrartIndex); } nodesDestroyList(pSmaFuncs); return code; } -static SNode* smaOptCreateWStartTs() { +static SNode* smaIndexOptCreateWStartTs() { SFunctionNode* pWStart = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); if (NULL == pWStart) { return NULL; @@ -920,7 +909,7 @@ static SNode* smaOptCreateWStartTs() { return (SNode*)pWStart; } -static int32_t smaOptCreateMergeKey(SNode* pCol, SNodeList** pMergeKeys) { +static int32_t smaIndexOptCreateMergeKey(SNode* pCol, SNodeList** pMergeKeys) { SOrderByExprNode* pMergeKey = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); if (NULL == pMergeKey) { return TSDB_CODE_OUT_OF_MEMORY; @@ -935,9 +924,9 @@ static int32_t smaOptCreateMergeKey(SNode* pCol, SNodeList** pMergeKeys) { return nodesListMakeStrictAppend(pMergeKeys, (SNode*)pMergeKey); } -static int32_t smaOptRewriteInterval(SWindowLogicNode* pInterval, int32_t wstrartIndex, SNodeList** pMergeKeys) { +static int32_t smaIndexOptRewriteInterval(SWindowLogicNode* pInterval, int32_t wstrartIndex, SNodeList** pMergeKeys) { if (wstrartIndex < 0) { - SNode* pWStart = smaOptCreateWStartTs(); + SNode* pWStart = smaIndexOptCreateWStartTs(); if (NULL == pWStart) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -948,11 +937,11 @@ static int32_t smaOptRewriteInterval(SWindowLogicNode* pInterval, int32_t wstrar } wstrartIndex = LIST_LENGTH(pInterval->node.pTargets) - 1; } - return smaOptCreateMergeKey(nodesListGetNode(pInterval->node.pTargets, wstrartIndex), pMergeKeys); + return smaIndexOptCreateMergeKey(nodesListGetNode(pInterval->node.pTargets, wstrartIndex), pMergeKeys); } -static int32_t smaOptApplyIndexExt(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, - SNodeList* pSmaCols, int32_t wstrartIndex) { +static int32_t smaIndexOptApplyIndexExt(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, + SNodeList* pSmaCols, int32_t wstrartIndex) { SWindowLogicNode* pInterval = (SWindowLogicNode*)pScan->node.pParent; SNodeList* pMergeTargets = nodesCloneList(pInterval->node.pTargets); if (NULL == pMergeTargets) { @@ -961,42 +950,42 @@ static int32_t smaOptApplyIndexExt(SLogicSubplan* pLogicSubplan, SScanLogicNode* SLogicNode* pSmaScan = NULL; SLogicNode* pMerge = NULL; SNodeList* pMergeKeys = NULL; - int32_t code = smaOptRewriteInterval(pInterval, wstrartIndex, &pMergeKeys); + int32_t code = smaIndexOptRewriteInterval(pInterval, wstrartIndex, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { - code = smaOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); + code = smaIndexOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); } if (TSDB_CODE_SUCCESS == code) { - code = smaOptCreateMerge(pScan->node.pParent, pMergeKeys, pMergeTargets, &pMerge); + code = smaIndexOptCreateMerge(pScan->node.pParent, pMergeKeys, pMergeTargets, &pMerge); } if (TSDB_CODE_SUCCESS == code) { - code = smaOptRecombinationNode(pLogicSubplan, pScan->node.pParent, pMerge, pSmaScan); + code = smaIndexOptRecombinationNode(pLogicSubplan, pScan->node.pParent, pMerge, pSmaScan); } return code; } -static int32_t smaOptApplyIndex(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, - SNodeList* pSmaCols, int32_t wstrartIndex) { +static int32_t smaIndexOptApplyIndex(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, + SNodeList* pSmaCols, int32_t wstrartIndex) { SLogicNode* pSmaScan = NULL; - int32_t code = smaOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); + int32_t code = smaIndexOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); if (TSDB_CODE_SUCCESS == code) { code = replaceLogicNode(pLogicSubplan, pScan->node.pParent, pSmaScan); } return code; } -static void smaOptDestroySmaIndex(void* p) { taosMemoryFree(((STableIndexInfo*)p)->expr); } +static void smaIndexOptDestroySmaIndex(void* p) { taosMemoryFree(((STableIndexInfo*)p)->expr); } -static int32_t smaOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan) { +static int32_t smaIndexOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan) { int32_t code = TSDB_CODE_SUCCESS; int32_t nindexes = taosArrayGetSize(pScan->pSmaIndexes); for (int32_t i = 0; i < nindexes; ++i) { STableIndexInfo* pIndex = taosArrayGet(pScan->pSmaIndexes, i); SNodeList* pSmaCols = NULL; int32_t wstrartIndex = -1; - code = smaOptCouldApplyIndex(pScan, pIndex, &pSmaCols, &wstrartIndex); + code = smaIndexOptCouldApplyIndex(pScan, pIndex, &pSmaCols, &wstrartIndex); if (TSDB_CODE_SUCCESS == code && NULL != pSmaCols) { - code = smaOptApplyIndex(pLogicSubplan, pScan, pIndex, pSmaCols, wstrartIndex); - taosArrayDestroyEx(pScan->pSmaIndexes, smaOptDestroySmaIndex); + code = smaIndexOptApplyIndex(pLogicSubplan, pScan, pIndex, pSmaCols, wstrartIndex); + taosArrayDestroyEx(pScan->pSmaIndexes, smaIndexOptDestroySmaIndex); pScan->pSmaIndexes = NULL; break; } @@ -1004,12 +993,12 @@ static int32_t smaOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubp return code; } -static int32_t smaOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { - SScanLogicNode* pScan = (SScanLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, smaOptMayBeOptimized); +static int32_t smaIndexOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SScanLogicNode* pScan = (SScanLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, smaIndexOptMayBeOptimized); if (NULL == pScan) { return TSDB_CODE_SUCCESS; } - return smaOptimizeImpl(pCxt, pLogicSubplan, pScan); + return smaIndexOptimizeImpl(pCxt, pLogicSubplan, pScan); } static EDealRes partTagsOptHasColImpl(SNode* pNode, void* pContext) { @@ -1084,7 +1073,7 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub int32_t code = TSDB_CODE_SUCCESS; SScanLogicNode* pScan = (SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0); if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { - TSWAP(((SPartitionLogicNode*)pNode)->pPartitionKeys, pScan->pPartTags); + TSWAP(((SPartitionLogicNode*)pNode)->pPartitionKeys, pScan->pGroupTags); int32_t code = replaceLogicNode(pLogicSubplan, pNode, (SLogicNode*)pScan); if (TSDB_CODE_SUCCESS == code) { NODES_CLEAR_LIST(pNode->pChildren); @@ -1094,7 +1083,7 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub SNode* pGroupKey = NULL; FOREACH(pGroupKey, ((SAggLogicNode*)pNode)->pGroupKeys) { code = nodesListMakeStrictAppend( - &pScan->pPartTags, nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pGroupKey)->pParameterList, 0))); + &pScan->pGroupTags, nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pGroupKey)->pParameterList, 0))); if (TSDB_CODE_SUCCESS != code) { break; } @@ -1102,13 +1091,11 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub NODES_DESTORY_LIST(((SAggLogicNode*)pNode)->pGroupKeys); } if (TSDB_CODE_SUCCESS == code) { - code = partTagsOptRebuildTbanme(pScan->pPartTags); + code = partTagsOptRebuildTbanme(pScan->pGroupTags); } return code; } -//===================================================================================================================== -// eliminate project optimization static bool eliminateProjOptCheckProjColumnNames(SProjectLogicNode* pProjectNode) { SHashObj* pProjColNameHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); SNode* pProjection; @@ -1142,7 +1129,7 @@ static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) { return false; } - SNode* pProjection; + SNode* pProjection; FOREACH(pProjection, pProjectNode->pProjections) { SExprNode* pExprNode = (SExprNode*)pProjection; if (QUERY_NODE_COLUMN != nodeType(pExprNode)) { @@ -1190,31 +1177,169 @@ static int32_t eliminateProjOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLog return eliminateProjOptimizeImpl(pCxt, pLogicSubplan, pProjectNode); } -//===================================================================================================================== -// eliminate Set Operator optimization +static bool rewriteTailOptMayBeOptimized(SLogicNode* pNode) { + return QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC == nodeType(pNode) && ((SIndefRowsFuncLogicNode*)pNode)->isTailFunc; +} + +static SNode* rewriteTailOptCreateOrderByExpr(SNode* pSortKey) { + SOrderByExprNode* pOrder = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); + if (NULL == pOrder) { + return NULL; + } + pOrder->order = ORDER_DESC; + pOrder->pExpr = nodesCloneNode(pSortKey); + if (NULL == pOrder->pExpr) { + nodesDestroyNode((SNode*)pOrder); + return NULL; + } + return (SNode*)pOrder; +} + +static int32_t rewriteTailOptCreateLimit(SNode* pLimit, SNode* pOffset, SNode** pOutput) { + SLimitNode* pLimitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT); + if (NULL == pLimitNode) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pLimitNode->limit = NULL == pLimit ? -1 : ((SValueNode*)pLimit)->datum.i; + pLimitNode->offset = NULL == pOffset ? -1 : ((SValueNode*)pOffset)->datum.i; + *pOutput = (SNode*)pLimitNode; + return TSDB_CODE_SUCCESS; +} + +static bool rewriteTailOptNeedGroupSort(SIndefRowsFuncLogicNode* pIndef) { + if (1 != LIST_LENGTH(pIndef->node.pChildren)) { + return false; + } + SNode* pChild = nodesListGetNode(pIndef->node.pChildren, 0); + return QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pChild) || + (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && NULL != ((SScanLogicNode*)pChild)->pGroupTags); +} + +static int32_t rewriteTailOptCreateSort(SIndefRowsFuncLogicNode* pIndef, SLogicNode** pOutput) { + SSortLogicNode* pSort = (SSortLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SORT); + if (NULL == pSort) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pSort->groupSort = rewriteTailOptNeedGroupSort(pIndef); + TSWAP(pSort->node.pChildren, pIndef->node.pChildren); + pSort->node.precision = pIndef->node.precision; + + // tail(expr, [limit, offset,] _rowts) + SFunctionNode* pTail = (SFunctionNode*)nodesListGetNode(pIndef->pFuncs, 0); + int32_t rowtsIndex = LIST_LENGTH(pTail->pParameterList) - 1; + + int32_t code = nodesListMakeStrictAppend( + &pSort->pSortKeys, rewriteTailOptCreateOrderByExpr(nodesListGetNode(pTail->pParameterList, rowtsIndex))); + if (TSDB_CODE_SUCCESS == code) { + pSort->node.pTargets = nodesCloneList(((SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0))->pTargets); + if (NULL == pSort->node.pTargets) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pOutput = (SLogicNode*)pSort; + } else { + nodesDestroyNode((SNode*)pSort); + } + + return code; +} + +static SNode* rewriteTailOptCreateProjectExpr(SFunctionNode* pTail) { + SNode* pExpr = nodesCloneNode(nodesListGetNode(pTail->pParameterList, 0)); + if (NULL == pExpr) { + return NULL; + } + strcpy(((SExprNode*)pExpr)->aliasName, pTail->node.aliasName); + return pExpr; +} + +static int32_t rewriteTailOptCreateProject(SIndefRowsFuncLogicNode* pIndef, SLogicNode** pOutput) { + SProjectLogicNode* pProject = (SProjectLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_PROJECT); + if (NULL == pProject) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + TSWAP(pProject->node.pTargets, pIndef->node.pTargets); + pProject->node.precision = pIndef->node.precision; + + // tail(expr, [limit, offset,] _rowts) + SFunctionNode* pTail = (SFunctionNode*)nodesListGetNode(pIndef->pFuncs, 0); + int32_t limitIndex = LIST_LENGTH(pTail->pParameterList) > 2 ? 1 : -1; + int32_t offsetIndex = LIST_LENGTH(pTail->pParameterList) > 3 ? 2 : -1; + + int32_t code = nodesListMakeStrictAppend(&pProject->pProjections, rewriteTailOptCreateProjectExpr(pTail)); + if (TSDB_CODE_SUCCESS == code) { + code = rewriteTailOptCreateLimit(limitIndex < 0 ? NULL : nodesListGetNode(pTail->pParameterList, limitIndex), + offsetIndex < 0 ? NULL : nodesListGetNode(pTail->pParameterList, offsetIndex), + &pProject->node.pLimit); + } + if (TSDB_CODE_SUCCESS == code) { + *pOutput = (SLogicNode*)pProject; + } else { + nodesDestroyNode((SNode*)pProject); + } + return code; +} + +static int32_t rewriteTailOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, + SIndefRowsFuncLogicNode* pIndef) { + SLogicNode* pSort = NULL; + SLogicNode* pProject = NULL; + int32_t code = rewriteTailOptCreateSort(pIndef, &pSort); + if (TSDB_CODE_SUCCESS == code) { + code = rewriteTailOptCreateProject(pIndef, &pProject); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeAppend(&pProject->pChildren, (SNode*)pSort); + pSort->pParent = pProject; + pSort = NULL; + } + if (TSDB_CODE_SUCCESS == code) { + code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pIndef, pProject); + } + if (TSDB_CODE_SUCCESS == code) { + nodesDestroyNode((SNode*)pIndef); + } else { + nodesDestroyNode((SNode*)pSort); + nodesDestroyNode((SNode*)pProject); + } + return code; +} + +static int32_t rewriteTailOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SIndefRowsFuncLogicNode* pIndef = + (SIndefRowsFuncLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, rewriteTailOptMayBeOptimized); + + if (NULL == pIndef) { + return TSDB_CODE_SUCCESS; + } + + return rewriteTailOptimizeImpl(pCxt, pLogicSubplan, pIndef); +} static bool eliminateSetOpMayBeOptimized(SLogicNode* pNode) { SLogicNode* pParent = pNode->pParent; if (NULL == pParent || - QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pParent) && QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pParent) || + QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pParent) && QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pParent) || LIST_LENGTH(pParent->pChildren) < 2) { return false; } - if (nodeType(pNode) != nodeType(pNode->pParent) || - LIST_LENGTH(pNode->pChildren) < 2) { + if (nodeType(pNode) != nodeType(pNode->pParent) || LIST_LENGTH(pNode->pChildren) < 2) { return false; } return true; } -static int32_t eliminateSetOpOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SLogicNode* pSetOpNode) { +static int32_t eliminateSetOpOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, + SLogicNode* pSetOpNode) { SNode* pSibling; FOREACH(pSibling, pSetOpNode->pParent->pChildren) { if (nodesEqualNode(pSibling, (SNode*)pSetOpNode)) { SNode* pChild; - FOREACH(pChild, pSetOpNode->pChildren) { - ((SLogicNode*)pChild)->pParent = pSetOpNode->pParent; - } + FOREACH(pChild, pSetOpNode->pChildren) { ((SLogicNode*)pChild)->pParent = pSetOpNode->pParent; } INSERT_LIST(pSetOpNode->pParent->pChildren, pSetOpNode->pChildren); pSetOpNode->pChildren = NULL; @@ -1237,13 +1362,14 @@ static int32_t eliminateSetOpOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLo // clang-format off static const SOptimizeRule optimizeRuleSet[] = { - {.pName = "OptimizeScanData", .optimizeFunc = osdOptimize}, - {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, - {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}, - {.pName = "SmaIndex", .optimizeFunc = smaOptimize}, + {.pName = "ScanPath", .optimizeFunc = scanPathOptimize}, + {.pName = "PushDownCondition", .optimizeFunc = pushDownCondOptimize}, + {.pName = "SortPrimaryKey", .optimizeFunc = sortPrimaryKeyOptimize}, + {.pName = "SmaIndex", .optimizeFunc = smaIndexOptimize}, {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize}, - {.pName = "EliminateSetOperator", .optimizeFunc = eliminateSetOpOptimize} + {.pName = "EliminateSetOperator", .optimizeFunc = eliminateSetOpOptimize}, + {.pName = "RewriteTail", .optimizeFunc = rewriteTailOptimize} }; // clang-format on diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 4f46abd8da..46747af3a9 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -41,8 +41,12 @@ typedef struct SPhysiPlanContext { static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { SColumnNode* pCol = (SColumnNode*)pNode; - if (NULL != pStmtName && '\0' != pStmtName[0]) { - return sprintf(pKey, "%s.%s", pStmtName, pCol->node.aliasName); + if (NULL != pStmtName) { + if ('\0' != pStmtName[0]) { + return sprintf(pKey, "%s.%s", pStmtName, pCol->node.aliasName); + } else { + return sprintf(pKey, "%s", pCol->node.aliasName); + } } if ('\0' == pCol->tableAlias[0]) { return sprintf(pKey, "%s", pCol->colName); @@ -56,11 +60,13 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) { return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName); } -static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_t slotId, bool output, bool reserve) { +static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const char* pName, const SNode* pNode, int16_t slotId, + bool output, bool reserve) { SSlotDescNode* pSlot = (SSlotDescNode*)nodesMakeNode(QUERY_NODE_SLOT_DESC); if (NULL == pSlot) { return NULL; } + strcpy(pSlot->name, pName); pSlot->slotId = slotId; pSlot->dataType = ((SExprNode*)pNode)->resType; pSlot->reserve = reserve; @@ -99,10 +105,8 @@ static int32_t putSlotToHashImpl(int16_t dataBlockId, int16_t slotId, const char return taosHashPut(pHash, pName, len, &index, sizeof(SSlotIndex)); } -static int32_t putSlotToHash(int16_t dataBlockId, int16_t slotId, SNode* pNode, SHashObj* pHash) { - char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; - int32_t len = getSlotKey(pNode, NULL, name); - return putSlotToHashImpl(dataBlockId, slotId, name, len, pHash); +static int32_t putSlotToHash(const char* pName, int16_t dataBlockId, int16_t slotId, SNode* pNode, SHashObj* pHash) { + return putSlotToHashImpl(dataBlockId, slotId, pName, strlen(pName), pHash); } static int32_t createDataBlockDescHash(SPhysiPlanContext* pCxt, int32_t capacity, int16_t dataBlockId, @@ -131,9 +135,11 @@ static int32_t buildDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SD int16_t slotId = 0; SNode* pNode = NULL; FOREACH(pNode, pList) { - code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pNode, slotId, true, false)); + char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; + getSlotKey(pNode, NULL, name); + code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, name, pNode, slotId, true, false)); if (TSDB_CODE_SUCCESS == code) { - code = putSlotToHash(pDataBlockDesc->dataBlockId, slotId, pNode, pHash); + code = putSlotToHash(name, pDataBlockDesc->dataBlockId, slotId, pNode, pHash); } if (TSDB_CODE_SUCCESS == code) { pDataBlockDesc->totalRowSize += ((SExprNode*)pNode)->resType.bytes; @@ -196,7 +202,8 @@ static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList, int32_t len = getSlotKey(pExpr, pStmtName, name); SSlotIndex* pIndex = taosHashGet(pHash, name, len); if (NULL == pIndex) { - code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pExpr, nextSlotId, output, reserve)); + code = + nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, name, pExpr, nextSlotId, output, reserve)); if (TSDB_CODE_SUCCESS == code) { code = putSlotToHashImpl(pDataBlockDesc->dataBlockId, nextSlotId, name, len, pHash); } @@ -513,12 +520,13 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName); pTableScan->dataRequired = pScanLogicNode->dataRequired; pTableScan->pDynamicScanFuncs = nodesCloneList(pScanLogicNode->pDynamicScanFuncs); - pTableScan->pPartitionTags = nodesCloneList(pScanLogicNode->pPartTags); + pTableScan->pGroupTags = nodesCloneList(pScanLogicNode->pGroupTags); if ((NULL != pScanLogicNode->pDynamicScanFuncs && NULL == pTableScan->pDynamicScanFuncs) || - (NULL != pScanLogicNode->pPartTags && NULL == pTableScan->pPartitionTags)) { + (NULL != pScanLogicNode->pGroupTags && NULL == pTableScan->pGroupTags)) { nodesDestroyNode((SNode*)pTableScan); return TSDB_CODE_OUT_OF_MEMORY; } + pTableScan->groupSort = pScanLogicNode->groupSort; pTableScan->interval = pScanLogicNode->interval; pTableScan->offset = pScanLogicNode->offset; pTableScan->sliding = pScanLogicNode->sliding; @@ -1170,8 +1178,9 @@ static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildr static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SSortLogicNode* pSortLogicNode, SPhysiNode** pPhyNode) { - SSortPhysiNode* pSort = - (SSortPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pSortLogicNode, QUERY_NODE_PHYSICAL_PLAN_SORT); + SSortPhysiNode* pSort = (SSortPhysiNode*)makePhysiNode( + pCxt, (SLogicNode*)pSortLogicNode, + pSortLogicNode->groupSort ? QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT : QUERY_NODE_PHYSICAL_PLAN_SORT); if (NULL == pSort) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1185,7 +1194,7 @@ static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) { code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pSort->pExprs); if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockSlots(pCxt, pSort->pExprs, pChildTupe); + code = pushdownDataBlockSlots(pCxt, pSort->pExprs, pChildTupe); } } @@ -1322,6 +1331,7 @@ static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SMergeLogicNode* pM pMerge->numOfChannels = pMergeLogicNode->numOfChannels; pMerge->srcGroupId = pMergeLogicNode->srcGroupId; + pMerge->groupSort = pMergeLogicNode->groupSort; int32_t code = addDataBlockSlots(pCxt, pMergeLogicNode->pInputs, pMerge->node.pOutputDataBlockDesc); diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index b4a966f206..49ab50f913 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -362,7 +362,7 @@ static int32_t stbSplGetNumOfVgroups(SLogicNode* pNode) { } static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pSplitNode, - SNodeList* pMergeKeys, SLogicNode* pPartChild) { + SNodeList* pMergeKeys, SLogicNode* pPartChild, bool groupSort) { SMergeLogicNode* pMerge = (SMergeLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_MERGE); if (NULL == pMerge) { return TSDB_CODE_OUT_OF_MEMORY; @@ -371,6 +371,7 @@ static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubpla pMerge->srcGroupId = pCxt->groupId; pMerge->node.precision = pPartChild->precision; pMerge->pMergeKeys = pMergeKeys; + pMerge->groupSort = groupSort; int32_t code = TSDB_CODE_SUCCESS; pMerge->pInputs = nodesCloneList(pPartChild->pTargets); @@ -430,7 +431,7 @@ static int32_t stbSplSplitIntervalForBatch(SSplitContext* pCxt, SStableSplitInfo SNodeList* pMergeKeys = NULL; code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pInfo->pSplitNode)->pTspk, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { - code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, pMergeKeys, pPartWindow); + code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, pMergeKeys, pPartWindow, false); } if (TSDB_CODE_SUCCESS != code) { nodesDestroyList(pMergeKeys); @@ -497,12 +498,16 @@ static int32_t stbSplSplitSessionForStream(SSplitContext* pCxt, SStableSplitInfo return code; } -static void splSetTableScanType(SLogicNode* pNode, EScanType scanType) { +static void stbSplSetTableMergeScan(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { - ((SScanLogicNode*)pNode)->scanType = scanType; + SScanLogicNode* pScan = (SScanLogicNode*)pNode; + pScan->scanType = SCAN_TYPE_TABLE_MERGE; + if (NULL != pScan->pGroupTags) { + pScan->groupSort = true; + } } else { if (1 == LIST_LENGTH(pNode->pChildren)) { - splSetTableScanType((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), scanType); + stbSplSetTableMergeScan((SLogicNode*)nodesListGetNode(pNode->pChildren, 0)); } } } @@ -515,7 +520,7 @@ static int32_t stbSplSplitSessionOrStateForBatch(SSplitContext* pCxt, SStableSpl int32_t code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pWindow)->pTspk, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { - code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pChild, pMergeKeys, (SLogicNode*)pChild); + code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pChild, pMergeKeys, (SLogicNode*)pChild, true); } if (TSDB_CODE_SUCCESS == code) { @@ -524,13 +529,10 @@ static int32_t stbSplSplitSessionOrStateForBatch(SSplitContext* pCxt, SStableSpl } if (TSDB_CODE_SUCCESS == code) { - splSetTableScanType(pChild, SCAN_TYPE_TABLE_MERGE); - ++(pCxt->groupId); - } - - if (TSDB_CODE_SUCCESS == code) { + stbSplSetTableMergeScan(pChild); pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE; SPLIT_FLAG_SET_MASK(pInfo->pSubplan->splitFlag, SPLIT_FLAG_STABLE_SPLIT); + ++(pCxt->groupId); } else { nodesDestroyList(pMergeKeys); } @@ -560,7 +562,7 @@ static int32_t stbSplSplitState(SSplitContext* pCxt, SStableSplitInfo* pInfo) { static SNodeList* stbSplGetPartKeys(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { - return ((SScanLogicNode*)pNode)->pPartTags; + return ((SScanLogicNode*)pNode)->pGroupTags; } else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { return ((SPartitionLogicNode*)pNode)->pPartitionKeys; } else { @@ -775,6 +777,7 @@ static int32_t stbSplCreatePartSortNode(SSortLogicNode* pSort, SLogicNode** pOut pPartSort->node.pChildren = pChildren; splSetParent((SLogicNode*)pPartSort); pPartSort->pSortKeys = pSortKeys; + pPartSort->groupSort = pSort->groupSort; code = stbSplCreateMergeKeys(pPartSort->pSortKeys, pPartSort->node.pTargets, &pMergeKeys); } @@ -789,12 +792,29 @@ static int32_t stbSplCreatePartSortNode(SSortLogicNode* pSort, SLogicNode** pOut return code; } +static void stbSplSetScanPartSort(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + SScanLogicNode* pScan = (SScanLogicNode*)pNode; + if (NULL != pScan->pGroupTags) { + pScan->groupSort = true; + } + } else { + if (1 == LIST_LENGTH(pNode->pChildren)) { + stbSplSetScanPartSort((SLogicNode*)nodesListGetNode(pNode->pChildren, 0)); + } + } +} + static int32_t stbSplSplitSortNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { SLogicNode* pPartSort = NULL; SNodeList* pMergeKeys = NULL; + bool groupSort = ((SSortLogicNode*)pInfo->pSplitNode)->groupSort; int32_t code = stbSplCreatePartSortNode((SSortLogicNode*)pInfo->pSplitNode, &pPartSort, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { - code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, pMergeKeys, pPartSort); + code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, pMergeKeys, pPartSort, groupSort); + } + if (TSDB_CODE_SUCCESS == code && groupSort) { + stbSplSetScanPartSort(pPartSort); } if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, @@ -829,7 +849,7 @@ static int32_t stbSplSplitScanNodeForJoin(SSplitContext* pCxt, SLogicSubplan* pS SNodeList* pMergeKeys = NULL; int32_t code = stbSplCreateMergeKeysByPrimaryKey(stbSplFindPrimaryKeyFromScan(pScan), &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { - code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, (SLogicNode*)pScan); + code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, (SLogicNode*)pScan, false); } if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&pSubplan->pChildren, diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp index ea40567dae..8c19b52a09 100644 --- a/source/libs/planner/test/planBasicTest.cpp +++ b/source/libs/planner/test/planBasicTest.cpp @@ -67,6 +67,14 @@ TEST_F(PlanBasicTest, tailFunc) { run("SELECT TAIL(c1, 10) FROM t1"); run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10"); + + run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 PARTITION BY c1"); + + run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 ORDER BY 1"); + + run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 LIMIT 5"); + + run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 PARTITION BY c1 LIMIT 5"); } TEST_F(PlanBasicTest, interpFunc) { diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index 43b6bf5772..46c2f33048 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -76,6 +76,7 @@ static void parseArg(int argc, char* argv[]) { static struct option long_options[] = { {"dump", optional_argument, NULL, 'd'}, {"skipSql", required_argument, NULL, 's'}, + {"limitSql", required_argument, NULL, 'i'}, {"log", required_argument, NULL, 'l'}, {0, 0, 0, 0} }; @@ -88,6 +89,9 @@ static void parseArg(int argc, char* argv[]) { case 's': setSkipSqlNum(optarg); break; + case 'i': + setLimitSqlNum(optarg); + break; case 'l': setLogLevel(optarg); break; diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index c197a02dd1..d54859c296 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -51,6 +51,7 @@ enum DumpModule { DumpModule g_dumpModule = DUMP_MODULE_NOTHING; int32_t g_skipSql = 0; +int32_t g_limitSql = 0; int32_t g_logLevel = 131; void setDumpModule(const char* pModule) { @@ -76,28 +77,33 @@ void setDumpModule(const char* pModule) { } void setSkipSqlNum(const char* pNum) { g_skipSql = stoi(pNum); } - +void setLimitSqlNum(const char* pNum) { g_limitSql = stoi(pNum); } void setLogLevel(const char* pLogLevel) { g_logLevel = stoi(pLogLevel); } int32_t getLogLevel() { return g_logLevel; } class PlannerTestBaseImpl { public: - PlannerTestBaseImpl() : sqlNo_(0) {} + PlannerTestBaseImpl() : sqlNo_(0), sqlNum_(0) {} void useDb(const string& user, const string& db) { caseEnv_.acctId_ = 0; caseEnv_.user_ = user; caseEnv_.db_ = db; - caseEnv_.nsql_ = g_skipSql; + caseEnv_.numOfSkipSql_ = g_skipSql; + caseEnv_.numOfLimitSql_ = g_limitSql; } void run(const string& sql) { ++sqlNo_; - if (caseEnv_.nsql_ > 0) { - --(caseEnv_.nsql_); + if (caseEnv_.numOfSkipSql_ > 0) { + --(caseEnv_.numOfSkipSql_); return; } + if (caseEnv_.numOfLimitSql_ > 0 && caseEnv_.numOfLimitSql_ == sqlNum_) { + return; + } + ++sqlNum_; reset(); try { @@ -134,7 +140,7 @@ class PlannerTestBaseImpl { } void prepare(const string& sql) { - if (caseEnv_.nsql_ > 0) { + if (caseEnv_.numOfSkipSql_ > 0) { return; } @@ -148,7 +154,7 @@ class PlannerTestBaseImpl { } void bindParams(TAOS_MULTI_BIND* pParams, int32_t colIdx) { - if (caseEnv_.nsql_ > 0) { + if (caseEnv_.numOfSkipSql_ > 0) { return; } @@ -161,8 +167,8 @@ class PlannerTestBaseImpl { } void exec() { - if (caseEnv_.nsql_ > 0) { - --(caseEnv_.nsql_); + if (caseEnv_.numOfSkipSql_ > 0) { + --(caseEnv_.numOfSkipSql_); return; } @@ -197,9 +203,10 @@ class PlannerTestBaseImpl { int32_t acctId_; string user_; string db_; - int32_t nsql_; + int32_t numOfSkipSql_; + int32_t numOfLimitSql_; - caseEnv() : nsql_(0) {} + caseEnv() : numOfSkipSql_(0) {} }; struct stmtEnv { @@ -401,6 +408,7 @@ class PlannerTestBaseImpl { stmtEnv stmtEnv_; stmtRes res_; int32_t sqlNo_; + int32_t sqlNum_; }; PlannerTestBase::PlannerTestBase() : impl_(new PlannerTestBaseImpl()) {} diff --git a/source/libs/planner/test/planTestUtil.h b/source/libs/planner/test/planTestUtil.h index 7d2a9e533f..f9942c93a7 100644 --- a/source/libs/planner/test/planTestUtil.h +++ b/source/libs/planner/test/planTestUtil.h @@ -43,6 +43,7 @@ class PlannerTestBase : public testing::Test { extern void setDumpModule(const char* pModule); extern void setSkipSqlNum(const char* pNum); +extern void setLimitSqlNum(const char* pNum); extern void setLogLevel(const char* pLogLevel); extern int32_t getLogLevel(); diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index b130186516..37f18a6388 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -56,6 +56,31 @@ void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj); void syncEntryLog(const SSyncRaftEntry* pObj); void syncEntryLog2(char* s, const SSyncRaftEntry* pObj); +//----------------------------------- +typedef struct SRaftEntryCache { + SHashObj* pEntryHash; + int32_t maxCount; + int32_t currentCount; + TdThreadMutex mutex; + SSyncNode* pSyncNode; +} SRaftEntryCache; + +SRaftEntryCache* raftCacheCreate(SSyncNode* pSyncNode, int32_t maxCount); +void raftCacheDestroy(SRaftEntryCache* pCache); +int32_t raftCachePutEntry(struct SRaftEntryCache* pCache, SSyncRaftEntry* pEntry); +int32_t raftCacheGetEntry(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); +int32_t raftCacheGetEntryP(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); +int32_t raftCacheDelEntry(struct SRaftEntryCache* pCache, SyncIndex index); +int32_t raftCacheGetAndDel(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); +int32_t raftCacheClear(struct SRaftEntryCache* pCache); + +cJSON* raftCache2Json(SRaftEntryCache* pObj); +char* raftCache2Str(SRaftEntryCache* pObj); +void raftCachePrint(SRaftEntryCache* pObj); +void raftCachePrint2(char* s, SRaftEntryCache* pObj); +void raftCacheLog(SRaftEntryCache* pObj); +void raftCacheLog2(char* s, SRaftEntryCache* pObj); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index f00dfe6f50..55e0755a89 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -35,12 +35,13 @@ extern "C" { #define SYNC_SNAPSHOT_RETRY_MS 5000 +//--------------------------------------------------- typedef struct SSyncSnapshotSender { bool start; int32_t seq; int32_t ack; - void * pReader; - void * pCurrentBlock; + void *pReader; + void *pCurrentBlock; int32_t blockLen; SSnapshot snapshot; SSyncCfg lastConfig; @@ -59,33 +60,36 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot void snapshotSenderStop(SSyncSnapshotSender *pSender); int32_t snapshotSend(SSyncSnapshotSender *pSender); int32_t snapshotReSend(SSyncSnapshotSender *pSender); -cJSON * snapshotSender2Json(SSyncSnapshotSender *pSender); -char * snapshotSender2Str(SSyncSnapshotSender *pSender); -char * snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event); +cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); +char *snapshotSender2Str(SSyncSnapshotSender *pSender); +char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event); + +//--------------------------------------------------- typedef struct SSyncSnapshotReceiver { - bool start; - - int32_t ack; - void * pWriter; - SyncTerm term; - SyncTerm privateTerm; - SSnapshot snapshot; - - SSyncNode *pSyncNode; + bool start; + int32_t ack; + void *pWriter; + SyncTerm term; + SyncTerm privateTerm; + SSnapshot snapshot; SRaftId fromId; + SSyncNode *pSyncNode; } SSyncSnapshotReceiver; SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId); void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver); -void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg); -bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); -void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply); -cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); -char * snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); -char * snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event); +void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg); +bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); +void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply); +cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); +char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); +char *snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event); + +//--------------------------------------------------- +// on message int32_t syncNodeOnSnapshotSendCb(SSyncNode *ths, SyncSnapshotSend *pMsg); int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 3996193037..60f6ca7e7b 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1355,11 +1355,16 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { int32_t userStrLen = strlen(str); SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; - if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { + if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); } - SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); - SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); + + SyncIndex logLastIndex = SYNC_INDEX_INVALID; + SyncIndex logBeginIndex = SYNC_INDEX_INVALID; + if (pSyncNode->pLogStore != NULL) { + logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); + logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); + } char* pCfgStr = syncCfg2SimpleStr(&(pSyncNode->pRaftCfg->cfg)); char* printStr = ""; diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index ff334a76bb..225360630c 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -180,3 +180,247 @@ void syncEntryLog2(char* s, const SSyncRaftEntry* pObj) { sTrace("syncEntryLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } + +//----------------------------------- +SRaftEntryCache* raftCacheCreate(SSyncNode* pSyncNode, int32_t maxCount) { + SRaftEntryCache* pCache = taosMemoryMalloc(sizeof(SRaftEntryCache)); + if (pCache == NULL) { + sError("vgId:%d raft cache create error", pSyncNode->vgId); + return NULL; + } + + pCache->pEntryHash = + taosHashInit(sizeof(SyncIndex), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (pCache->pEntryHash == NULL) { + sError("vgId:%d raft cache create hash error", pSyncNode->vgId); + return NULL; + } + + taosThreadMutexInit(&(pCache->mutex), NULL); + pCache->maxCount = maxCount; + pCache->currentCount = 0; + pCache->pSyncNode = pSyncNode; + + return pCache; +} + +void raftCacheDestroy(SRaftEntryCache* pCache) { + if (pCache != NULL) { + taosThreadMutexLock(&(pCache->mutex)); + taosHashCleanup(pCache->pEntryHash); + taosThreadMutexUnlock(&(pCache->mutex)); + taosThreadMutexDestroy(&(pCache->mutex)); + taosMemoryFree(pCache); + } +} + +// success, return 1 +// max count, return 0 +// error, return -1 +int32_t raftCachePutEntry(struct SRaftEntryCache* pCache, SSyncRaftEntry* pEntry) { + taosThreadMutexLock(&(pCache->mutex)); + + if (pCache->currentCount >= pCache->maxCount) { + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; + } + + taosHashPut(pCache->pEntryHash, &(pEntry->index), sizeof(pEntry->index), pEntry, pEntry->bytes); + ++(pCache->currentCount); + + do { + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "raft cache add, type:%s,%d, type2:%s,%d, index:%ld, bytes:%d", + TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType, + pEntry->index, pEntry->bytes); + syncNodeEventLog(pCache->pSyncNode, eventLog); + } while (0); + + taosThreadMutexUnlock(&(pCache->mutex)); + return 1; +} + +// success, return 0 +// error, return -1 +// not exist, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST +int32_t raftCacheGetEntry(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) { + if (ppEntry == NULL) { + return -1; + } + *ppEntry = NULL; + + taosThreadMutexLock(&(pCache->mutex)); + void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index)); + if (pTmp != NULL) { + SSyncRaftEntry* pEntry = pTmp; + *ppEntry = taosMemoryMalloc(pEntry->bytes); + memcpy(*ppEntry, pTmp, pEntry->bytes); + + do { + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "raft cache get, type:%s,%d, type2:%s,%d, index:%ld", + TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType), + (*ppEntry)->originalRpcType, (*ppEntry)->index); + syncNodeEventLog(pCache->pSyncNode, eventLog); + } while (0); + + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; + } + + taosThreadMutexUnlock(&(pCache->mutex)); + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + return -1; +} + +// success, return 0 +// error, return -1 +// not exist, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST +int32_t raftCacheGetEntryP(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) { + if (ppEntry == NULL) { + return -1; + } + *ppEntry = NULL; + + taosThreadMutexLock(&(pCache->mutex)); + void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index)); + if (pTmp != NULL) { + SSyncRaftEntry* pEntry = pTmp; + *ppEntry = pEntry; + + do { + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "raft cache get, type:%s,%d, type2:%s,%d, index:%ld", + TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType), + (*ppEntry)->originalRpcType, (*ppEntry)->index); + syncNodeEventLog(pCache->pSyncNode, eventLog); + } while (0); + + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; + } + + taosThreadMutexUnlock(&(pCache->mutex)); + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + return -1; +} + +int32_t raftCacheDelEntry(struct SRaftEntryCache* pCache, SyncIndex index) { + taosThreadMutexLock(&(pCache->mutex)); + taosHashRemove(pCache->pEntryHash, &index, sizeof(index)); + --(pCache->currentCount); + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; +} + +int32_t raftCacheGetAndDel(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) { + if (ppEntry == NULL) { + return -1; + } + *ppEntry = NULL; + + taosThreadMutexLock(&(pCache->mutex)); + void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index)); + if (pTmp != NULL) { + SSyncRaftEntry* pEntry = pTmp; + *ppEntry = taosMemoryMalloc(pEntry->bytes); + memcpy(*ppEntry, pTmp, pEntry->bytes); + + do { + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "raft cache get-and-del, type:%s,%d, type2:%s,%d, index:%ld", + TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType), + (*ppEntry)->originalRpcType, (*ppEntry)->index); + syncNodeEventLog(pCache->pSyncNode, eventLog); + } while (0); + + taosHashRemove(pCache->pEntryHash, &index, sizeof(index)); + --(pCache->currentCount); + + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; + } + + taosThreadMutexUnlock(&(pCache->mutex)); + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + return -1; +} + +int32_t raftCacheClear(struct SRaftEntryCache* pCache) { + taosThreadMutexLock(&(pCache->mutex)); + taosHashClear(pCache->pEntryHash); + pCache->currentCount = 0; + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; +} + +//----------------------------------- +cJSON* raftCache2Json(SRaftEntryCache* pCache) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pCache != NULL) { + taosThreadMutexLock(&(pCache->mutex)); + + snprintf(u64buf, sizeof(u64buf), "%p", pCache->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + cJSON_AddNumberToObject(pRoot, "currentCount", pCache->currentCount); + cJSON_AddNumberToObject(pRoot, "maxCount", pCache->maxCount); + cJSON* pEntries = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "entries", pEntries); + + SSyncRaftEntry* pIter = (SSyncRaftEntry*)taosHashIterate(pCache->pEntryHash, NULL); + if (pIter != NULL) { + SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter; + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + } + while (pIter) { + pIter = taosHashIterate(pCache->pEntryHash, pIter); + if (pIter != NULL) { + SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter; + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + } + } + + taosThreadMutexUnlock(&(pCache->mutex)); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SRaftEntryCache", pRoot); + return pJson; +} + +char* raftCache2Str(SRaftEntryCache* pCache) { + cJSON* pJson = raftCache2Json(pCache); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +void raftCachePrint(SRaftEntryCache* pCache) { + char* serialized = raftCache2Str(pCache); + printf("raftCachePrint | len:%lu | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftCachePrint2(char* s, SRaftEntryCache* pCache) { + char* serialized = raftCache2Str(pCache); + printf("raftCachePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftCacheLog(SRaftEntryCache* pCache) { + char* serialized = raftCache2Str(pCache); + sTrace("raftCacheLog | len:%lu | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void raftCacheLog2(char* s, SRaftEntryCache* pCache) { + if (gRaftDetailLog) { + char* serialized = raftCache2Str(pCache); + sTraceLong("raftCacheLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} \ No newline at end of file diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index 48c1b70a04..d2cbabe226 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -32,11 +32,13 @@ SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) { } void syncRespMgrDestroy(SSyncRespMgr *pObj) { - taosThreadMutexLock(&(pObj->mutex)); - taosHashCleanup(pObj->pRespHash); - taosThreadMutexUnlock(&(pObj->mutex)); - taosThreadMutexDestroy(&(pObj->mutex)); - taosMemoryFree(pObj); + if (pObj != NULL) { + taosThreadMutexLock(&(pObj->mutex)); + taosHashCleanup(pObj->pRespHash); + taosThreadMutexUnlock(&(pObj->mutex)); + taosThreadMutexDestroy(&(pObj->mutex)); + taosMemoryFree(pObj); + } } int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub) { diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 0feaf532bb..39e972c4c4 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -21,6 +21,7 @@ #include "syncUtil.h" #include "wal.h" +//---------------------------------- static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg); @@ -49,7 +50,7 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI pSender->pSyncNode->pFsm->FpGetSnapshotInfo(pSender->pSyncNode->pFsm, &(pSender->snapshot)); pSender->finish = false; } else { - sError("snapshotSenderCreate cannot create sender"); + sError("vgId:%d cannot create snapshot sender", pSyncNode->vgId); } return pSender; @@ -57,39 +58,59 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { if (pSender != NULL) { + // free current block if (pSender->pCurrentBlock != NULL) { taosMemoryFree(pSender->pCurrentBlock); + pSender->pCurrentBlock = NULL; } + + // close reader + if (pSender->pReader != NULL) { + int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader); + ASSERT(ret == 0); + pSender->pReader = NULL; + } + + // free sender taosMemoryFree(pSender); } } bool snapshotSenderIsStart(SSyncSnapshotSender *pSender) { return pSender->start; } -// begin send snapshot (current term, seq begin) +// begin send snapshot by snapshot, pReader void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void *pReader) { ASSERT(!snapshotSenderIsStart(pSender)); - pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; - pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; - // init snapshot and reader ASSERT(pSender->pReader == NULL); pSender->pReader = pReader; pSender->snapshot = snapshot; + // init current block if (pSender->pCurrentBlock != NULL) { taosMemoryFree(pSender->pCurrentBlock); } pSender->blockLen = 0; + // update term + pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; + ++(pSender->privateTerm); + + // update state + pSender->finish = false; + pSender->start = true; + pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; + pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; + + // init last config if (pSender->snapshot.lastConfigIndex != SYNC_INDEX_INVALID) { int32_t code = 0; SSyncRaftEntry *pEntry = NULL; + bool getLastConfig = false; + code = pSender->pSyncNode->pLogStore->syncLogGetEntry(pSender->pSyncNode->pLogStore, pSender->snapshot.lastConfigIndex, &pEntry); - - bool getLastConfig = false; if (code == 0) { ASSERT(pEntry != NULL); @@ -112,29 +133,29 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void } } + // last config not found in wal, update to -1 if (!getLastConfig) { - char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "snapshot sender update lcindex from %ld to -1", - pSender->snapshot.lastConfigIndex); - pSender->snapshot.lastConfigIndex = -1; - - char *eventLog = snapshotSender2SimpleStr(pSender, logBuf); - syncNodeEventLog(pSender->pSyncNode, eventLog); - taosMemoryFree(eventLog); - + SyncIndex oldLastConfigIndex = pSender->snapshot.lastConfigIndex; + SyncIndex newLastConfigIndex = SYNC_INDEX_INVALID; + pSender->snapshot.lastConfigIndex = SYNC_INDEX_INVALID; memset(&(pSender->lastConfig), 0, sizeof(SSyncCfg)); + + // event log + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "snapshot sender update lcindex from %ld to %ld", oldLastConfigIndex, + newLastConfigIndex); + char *eventLog = snapshotSender2SimpleStr(pSender, logBuf); + syncNodeEventLog(pSender->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); } } else { + // no last config memset(&(pSender->lastConfig), 0, sizeof(SSyncCfg)); } - pSender->sendingMS = SYNC_SNAPSHOT_RETRY_MS; - pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; - ++(pSender->privateTerm); - pSender->finish = false; - pSender->start = true; - // build begin msg SyncSnapshotSend *pMsg = syncSnapshotSendBuild(0, pSender->pSyncNode->vgId); pMsg->srcId = pSender->pSyncNode->myRaftId; @@ -151,40 +172,39 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void SRpcMsg rpcMsg; syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); - - char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender send"); - syncNodeEventLog(pSender->pSyncNode, eventLog); - taosMemoryFree(eventLog); - syncSnapshotSendDestroy(pMsg); + + // event log + do { + char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender send"); + syncNodeEventLog(pSender->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); } void snapshotSenderStop(SSyncSnapshotSender *pSender) { + // close reader if (pSender->pReader != NULL) { int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader); ASSERT(ret == 0); pSender->pReader = NULL; } + // free current block if (pSender->pCurrentBlock != NULL) { taosMemoryFree(pSender->pCurrentBlock); pSender->pCurrentBlock = NULL; pSender->blockLen = 0; } + // update flag pSender->start = false; - - if (gRaftDetailLog) { - char *s = snapshotSender2Str(pSender); - sInfo("snapshotSenderStop %s", s); - taosMemoryFree(s); - } } -// when sender receiver ack, call this function to send msg from seq +// when sender receive ack, call this function to send msg from seq // seq = ack + 1, already updated int32_t snapshotSend(SSyncSnapshotSender *pSender) { - // free memory last time (seq - 1) + // free memory last time (current seq - 1) if (pSender->pCurrentBlock != NULL) { taosMemoryFree(pSender->pCurrentBlock); pSender->pCurrentBlock = NULL; @@ -198,7 +218,7 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { if (pSender->blockLen > 0) { // has read data } else { - // read finish + // read finish, update seq to end pSender->seq = SYNC_SNAPSHOT_SEQ_END; } @@ -219,25 +239,28 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { SRpcMsg rpcMsg; syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); - - if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) { - char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender finish"); - syncNodeEventLog(pSender->pSyncNode, eventLog); - taosMemoryFree(eventLog); - - } else { - char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender sending"); - syncNodeEventLog(pSender->pSyncNode, eventLog); - taosMemoryFree(eventLog); - } - syncSnapshotSendDestroy(pMsg); + + // event log + do { + char *eventLog = NULL; + if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) { + eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender finish"); + } else { + eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender sending"); + } + syncNodeEventLog(pSender->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); + return 0; } // send snapshot data from cache int32_t snapshotReSend(SSyncSnapshotSender *pSender) { - if (pSender->pCurrentBlock != NULL) { + // send current block data + if (pSender->pCurrentBlock != NULL && pSender->blockLen > 0) { + // build msg SyncSnapshotSend *pMsg = syncSnapshotSendBuild(pSender->blockLen, pSender->pSyncNode->vgId); pMsg->srcId = pSender->pSyncNode->myRaftId; pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; @@ -249,16 +272,20 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { pMsg->seq = pSender->seq; memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); + // send msg SRpcMsg rpcMsg; syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); - - char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender resend"); - syncNodeEventLog(pSender->pSyncNode, eventLog); - taosMemoryFree(eventLog); - syncSnapshotSendDestroy(pMsg); + + // event log + do { + char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender resend"); + syncNodeEventLog(pSender->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); } + return 0; } @@ -294,7 +321,6 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { snprintf(u64buf, sizeof(u64buf), "%lu", pSender->snapshot.lastApplyTerm); cJSON_AddStringToObject(pSnapshot, "lastApplyTerm", u64buf); cJSON_AddItemToObject(pRoot, "snapshot", pSnapshot); - snprintf(u64buf, sizeof(u64buf), "%lu", pSender->sendingMS); cJSON_AddStringToObject(pRoot, "sendingMS", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pSender->pSyncNode); @@ -314,17 +340,17 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { char *snapshotSender2Str(SSyncSnapshotSender *pSender) { cJSON *pJson = snapshotSender2Json(pSender); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event) { int32_t len = 256; - char * s = taosMemoryMalloc(len); + char *s = taosMemoryMalloc(len); SRaftId destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; - char host[128]; + char host[64]; uint16_t port; syncUtilU642Addr(destId.addr, host, sizeof(host), &port); @@ -355,12 +381,12 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId from pReceiver->term = pSyncNode->pRaftStore->currentTerm; pReceiver->privateTerm = 0; pReceiver->snapshot.data = NULL; - pReceiver->snapshot.lastApplyIndex = -1; + pReceiver->snapshot.lastApplyIndex = SYNC_INDEX_INVALID; pReceiver->snapshot.lastApplyTerm = 0; - pReceiver->snapshot.lastConfigIndex = -1; + pReceiver->snapshot.lastConfigIndex = SYNC_INDEX_INVALID; } else { - sInfo("snapshotReceiverCreate cannot create receiver"); + sError("vgId:%d cannot create snapshot receiver", pSyncNode->vgId); } return pReceiver; @@ -368,29 +394,55 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId from void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) { if (pReceiver != NULL) { + // close writer + if (pReceiver->pWriter != NULL) { + int32_t ret = + pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false); + ASSERT(ret == 0); + pReceiver->pWriter = NULL; + } + + // free receiver taosMemoryFree(pReceiver); } } bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { return pReceiver->start; } -// begin receive snapshot msg (current term, seq begin) +// static do start +// receive first snapshot data +// privateTerm, pBeginMsg static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg) { + // update state pReceiver->term = pReceiver->pSyncNode->pRaftStore->currentTerm; pReceiver->privateTerm = privateTerm; pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; pReceiver->fromId = pBeginMsg->srcId; + // update snapshot pReceiver->snapshot.lastApplyIndex = pBeginMsg->lastIndex; pReceiver->snapshot.lastApplyTerm = pBeginMsg->lastTerm; pReceiver->snapshot.lastConfigIndex = pBeginMsg->lastConfigIndex; + // write data ASSERT(pReceiver->pWriter == NULL); int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStartWrite(pReceiver->pSyncNode->pFsm, &(pReceiver->pWriter)); ASSERT(ret == 0); } +static void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) { + // force close, abandon incomplete data + if (pReceiver->pWriter != NULL) { + int32_t ret = + pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false); + ASSERT(ret == 0); + pReceiver->pWriter = NULL; + } + + pReceiver->start = false; +} + // if receiver receive msg from seq = SYNC_SNAPSHOT_SEQ_BEGIN, start receiver // if already start, force close, start again void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg) { @@ -413,12 +465,6 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTer snapshotReceiverDoStart(pReceiver, privateTerm, pBeginMsg); pReceiver->start = true; } - - if (gRaftDetailLog) { - char *s = snapshotReceiver2Str(pReceiver); - sInfo("snapshotReceiverStart %s", s); - taosMemoryFree(s); - } } void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply) { @@ -434,12 +480,6 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply) { if (apply) { // ++(pReceiver->privateTerm); } - - if (gRaftDetailLog) { - char *s = snapshotReceiver2Str(pReceiver); - sInfo("snapshotReceiverStop %s", s); - taosMemoryFree(s); - } } cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { @@ -461,7 +501,7 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { cJSON_AddStringToObject(pFromId, "addr", u64buf); { uint64_t u64 = pReceiver->fromId.addr; - cJSON * pTmp = pFromId; + cJSON *pTmp = pFromId; char host[128] = {0}; uint16_t port; syncUtilU642Addr(u64, host, sizeof(host), &port); @@ -494,14 +534,14 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { cJSON *pJson = snapshotReceiver2Json(pReceiver); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } char *snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event) { int32_t len = 256; - char * s = taosMemoryMalloc(len); + char *s = taosMemoryMalloc(len); SRaftId fromId = pReceiver->fromId; char host[128]; diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 27084286da..bc63462a9e 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -17,6 +17,7 @@ add_executable(syncVotesRespondTest "") add_executable(syncIndexMgrTest "") add_executable(syncLogStoreTest "") add_executable(syncEntryTest "") +add_executable(syncEntryCacheTest "") add_executable(syncRequestVoteTest "") add_executable(syncRequestVoteReplyTest "") add_executable(syncAppendEntriesTest "") @@ -129,6 +130,10 @@ target_sources(syncEntryTest PRIVATE "syncEntryTest.cpp" ) +target_sources(syncEntryCacheTest + PRIVATE + "syncEntryCacheTest.cpp" +) target_sources(syncRequestVoteTest PRIVATE "syncRequestVoteTest.cpp" @@ -362,6 +367,11 @@ target_include_directories(syncEntryTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncEntryCacheTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_include_directories(syncRequestVoteTest PUBLIC "${TD_SOURCE_DIR}/include/libs/sync" @@ -610,6 +620,10 @@ target_link_libraries(syncEntryTest sync gtest_main ) +target_link_libraries(syncEntryCacheTest + sync + gtest_main +) target_link_libraries(syncRequestVoteTest sync gtest_main diff --git a/source/libs/sync/test/syncEntryCacheTest.cpp b/source/libs/sync/test/syncEntryCacheTest.cpp new file mode 100644 index 0000000000..787c08e507 --- /dev/null +++ b/source/libs/sync/test/syncEntryCacheTest.cpp @@ -0,0 +1,162 @@ +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SSyncRaftEntry* createEntry(int i) { + int32_t dataLen = 20; + SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); + assert(pEntry != NULL); + pEntry->msgType = 88; + pEntry->originalRpcType = 99; + pEntry->seqNum = 3; + pEntry->isWeak = true; + pEntry->term = 100 + i; + pEntry->index = i; + snprintf(pEntry->data, dataLen, "value%d", i); + + return pEntry; +} + +SSyncNode* createFakeNode() { + SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); + ASSERT(pSyncNode != NULL); + memset(pSyncNode, 0, sizeof(SSyncNode)); + + return pSyncNode; +} + +SRaftEntryCache* createCache(int maxCount) { + SSyncNode* pSyncNode = createFakeNode(); + ASSERT(pSyncNode != NULL); + + SRaftEntryCache* pCache = raftCacheCreate(pSyncNode, maxCount); + ASSERT(pCache != NULL); + + return pCache; +} + +void test1() { + int32_t code = 0; + SRaftEntryCache* pCache = createCache(5); + for (int i = 0; i < 5; ++i) { + SSyncRaftEntry* pEntry = createEntry(i); + code = raftCachePutEntry(pCache, pEntry); + ASSERT(code == 1); + syncEntryDestory(pEntry); + } + raftCacheLog2((char*)"==test1 write 5 entries==", pCache); + + SyncIndex index; + index = 1; + code = raftCacheDelEntry(pCache, index); + ASSERT(code == 0); + index = 3; + code = raftCacheDelEntry(pCache, index); + ASSERT(code == 0); + raftCacheLog2((char*)"==test1 delete 1,3==", pCache); + + code = raftCacheClear(pCache); + ASSERT(code == 0); + raftCacheLog2((char*)"==clear all==", pCache); +} + +void test2() { + int32_t code = 0; + SRaftEntryCache* pCache = createCache(5); + for (int i = 0; i < 5; ++i) { + SSyncRaftEntry* pEntry = createEntry(i); + code = raftCachePutEntry(pCache, pEntry); + ASSERT(code == 1); + syncEntryDestory(pEntry); + } + raftCacheLog2((char*)"==test2 write 5 entries==", pCache); + + SyncIndex index; + index = 1; + SSyncRaftEntry* pEntry; + code = raftCacheGetEntry(pCache, index, &pEntry); + ASSERT(code == 0); + syncEntryDestory(pEntry); + syncEntryLog2((char*)"==test2 get entry 1==", pEntry); + + index = 2; + code = raftCacheGetEntryP(pCache, index, &pEntry); + ASSERT(code == 0); + syncEntryLog2((char*)"==test2 get entry pointer 2==", pEntry); + + // not found + index = 8; + code = raftCacheGetEntry(pCache, index, &pEntry); + ASSERT(code == -1 && terrno == TSDB_CODE_WAL_LOG_NOT_EXIST); + sTrace("==test2 get entry 8 not found=="); + + // not found + index = 9; + code = raftCacheGetEntryP(pCache, index, &pEntry); + ASSERT(code == -1 && terrno == TSDB_CODE_WAL_LOG_NOT_EXIST); + sTrace("==test2 get entry pointer 9 not found=="); +} + +void test3() { + int32_t code = 0; + SRaftEntryCache* pCache = createCache(5); + for (int i = 0; i < 5; ++i) { + SSyncRaftEntry* pEntry = createEntry(i); + code = raftCachePutEntry(pCache, pEntry); + ASSERT(code == 1); + syncEntryDestory(pEntry); + } + for (int i = 6; i < 10; ++i) { + SSyncRaftEntry* pEntry = createEntry(i); + code = raftCachePutEntry(pCache, pEntry); + ASSERT(code == 0); + syncEntryDestory(pEntry); + } + raftCacheLog2((char*)"==test3 write 10 entries, max count is 5==", pCache); +} + +void test4() { + int32_t code = 0; + SRaftEntryCache* pCache = createCache(5); + for (int i = 0; i < 5; ++i) { + SSyncRaftEntry* pEntry = createEntry(i); + code = raftCachePutEntry(pCache, pEntry); + ASSERT(code == 1); + syncEntryDestory(pEntry); + } + raftCacheLog2((char*)"==test4 write 5 entries==", pCache); + + SyncIndex index; + index = 3; + SSyncRaftEntry* pEntry; + code = raftCacheGetAndDel(pCache, index, &pEntry); + ASSERT(code == 0); + syncEntryLog2((char*)"==test4 get-and-del entry 3==", pEntry); + raftCacheLog2((char*)"==test4 after get-and-del entry 3==", pCache); +} + +int main(int argc, char** argv) { + gRaftDetailLog = true; + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE + DEBUG_DEBUG; + + test1(); + test2(); + test3(); + test4(); + + return 0; +} diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index c328629c4b..462debb247 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -52,7 +52,7 @@ typedef struct { char user[TSDB_UNI_LEN]; // meter ID void (*cfp)(void* parent, SRpcMsg*, SEpSet*); - bool (*retry)(int32_t code); + bool (*retry)(int32_t code, tmsg_t msgType); int index; void* parent; diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 4f7b19b539..cc2e95cfb3 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -164,11 +164,6 @@ void rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) { transSetDefaultAddr(thandle, ip, fqdn); } -// void rpcSetMsgTraceId(SRpcMsg* pMsg, STraceId uid) { -// SRpcHandleInfo* pInfo = &pMsg->info; -// pInfo->traceId = uid; -//} - int32_t rpcInit() { // impl later return 0; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 7374d1fffc..ab08ce82a8 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1030,7 +1030,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { */ STransConnCtx* pCtx = pMsg->ctx; int32_t code = pResp->code; - if (pTransInst->retry != NULL && pTransInst->retry(code)) { + if (pTransInst->retry != NULL && pTransInst->retry(code, pResp->msgType - 1)) { pMsg->sent = 0; pCtx->retryCnt += 1; if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index ba07b6c3dd..9c66ac1df8 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -18,6 +18,10 @@ #include "os.h" #if defined(WINDOWS) +BOOL WINAPI CtrlHandler(DWORD fdwCtrlType) { + printf("\n" TAOS_CONSOLE_PROMPT_HEADER); + return TRUE; +} #elif defined(_TD_DARWIN_64) #else #include @@ -124,7 +128,7 @@ int taosSetConsoleEcho(bool on) { void taosSetTerminalMode() { #if defined(WINDOWS) - // assert(0); + SetConsoleCtrlHandler(CtrlHandler, TRUE); #else struct termios newtio; @@ -158,7 +162,6 @@ void taosSetTerminalMode() { int32_t taosGetOldTerminalMode() { #if defined(WINDOWS) - // assert(0); #else /* Make sure stdin is a terminal. */ if (!isatty(STDIN_FILENO)) { @@ -176,7 +179,7 @@ int32_t taosGetOldTerminalMode() { void taosResetTerminalMode() { #if defined(WINDOWS) - // assert(0); + SetConsoleCtrlHandler(CtrlHandler, FALSE); #else if (tcsetattr(0, TCSANOW, &oldtio) != 0) { fprintf(stderr, "Fail to reset the terminal properties!\n"); diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 3f23cd8b5f..f80c2b217f 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -52,7 +52,7 @@ fi docker run \ -v $REP_MOUNT_PARAM \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_TOOLS=true;make -j $THREAD_COUNT" + --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true;make -j $THREAD_COUNT" ret=$? exit $ret diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index b43f87be29..e9db4a5da3 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -637,5 +637,13 @@ class TDCom: column_value_str = ", ".join(str(v) for v in column_value_list) insert_sql = f'insert into {dbname}.{tbname} values ({column_value_str});' tsql.execute(insert_sql) - + def getOneRow(self, location, containElm): + res_list = list() + if 0 <= location < tdSql.queryRows: + for row in tdSql.queryResult: + if row[location] == containElm: + res_list.append(row) + return res_list + else: + tdLog.exit(f"getOneRow out of range: row_index={location} row_count={self.query_row}") tdCom = TDCom() diff --git a/tests/pytest/util/constant.py b/tests/pytest/util/constant.py index e5095c74dc..807398f70f 100644 --- a/tests/pytest/util/constant.py +++ b/tests/pytest/util/constant.py @@ -1,5 +1,76 @@ # -*- coding: utf-8 -*- +# basic type +TAOS_DATA_TYPE = [ + "INT", "BIGINT", "SMALLINT", "TINYINT", "INT UNSIGNED", "BIGINT UNSIGNED", "SMALLINT UNSIGNED", "TINYINT UNSIGNED", + "FLOAT", "DOUBLE", + "BOOL", + "BINARY", "NCHAR", "VARCHAR", + "TIMESTAMP", + # "MEDIUMBLOB", "BLOB", # add in 3.x + # "DECIMAL", "NUMERIC", # add in 3.x + "JSON", # only for tag +] + +TAOS_NUM_TYPE = [ + "INT", "BIGINT", "SMALLINT", "TINYINT", "INT UNSIGNED", "BIGINT UNSIGNED", "SMALLINT UNSIGNED", "TINYINT UNSIGNED", "FLOAT", "DOUBLE", + # "DECIMAL", "NUMERIC", # add in 3.x +] +TAOS_CHAR_TYPE = [ + "BINARY", "NCHAR", "VARCHAR", +] +TAOS_BOOL_TYPE = ["BOOL",] +TAOS_TS_TYPE = ["TIMESTAMP",] +TAOS_BIN_TYPE = [ + "MEDIUMBLOB", "BLOB", # add in 3.x +] + +TAOS_TIME_INIT = ["b", "u", "a", "s", "m", "h", "d", "w", "n", "y"] +TAOS_PRECISION = ["ms", "us", "ns"] +PRECISION_DEFAULT = "ms" +PRECISION = PRECISION_DEFAULT + +TAOS_KEYWORDS = [ + "ABORT", "CREATE", "IGNORE", "NULL", "STAR", + "ACCOUNT", "CTIME", "IMMEDIATE", "OF", "STATE", + "ACCOUNTS", "DATABASE", "IMPORT", "OFFSET", "STATEMENT", + "ADD", "DATABASES", "IN", "OR", "STATE_WINDOW", + "AFTER", "DAYS", "INITIALLY", "ORDER", "STORAGE", + "ALL", "DBS", "INSERT", "PARTITIONS", "STREAM", + "ALTER", "DEFERRED", "INSTEAD", "PASS", "STREAMS", + "AND", "DELIMITERS", "INT", "PLUS", "STRING", + "AS", "DESC", "INTEGER", "PPS", "SYNCDB", + "ASC", "DESCRIBE", "INTERVAL", "PRECISION", "TABLE", + "ATTACH", "DETACH", "INTO", "PREV", "TABLES", + "BEFORE", "DISTINCT", "IS", "PRIVILEGE", "TAG", + "BEGIN", "DIVIDE", "ISNULL", "QTIME", "TAGS", + "BETWEEN", "DNODE", "JOIN", "QUERIES", "TBNAME", + "BIGINT", "DNODES", "KEEP", "QUERY", "TIMES", + "BINARY", "DOT", "KEY", "QUORUM", "TIMESTAMP", + "BITAND", "DOUBLE", "KILL", "RAISE", "TINYINT", + "BITNOT", "DROP", "LE", "REM", "TOPIC", + "BITOR", "EACH", "LIKE", "REPLACE", "TOPICS", + "BLOCKS", "END", "LIMIT", "REPLICA", "TRIGGER", + "BOOL", "EQ", "LINEAR", "RESET", "TSERIES", + "BY", "EXISTS", "LOCAL", "RESTRICT", "UMINUS", + "CACHE", "EXPLAIN", "LP", "ROW", "UNION", + "CACHELAST", "FAIL", "LSHIFT", "RP", "UNSIGNED", + "CASCADE", "FILE", "LT", "RSHIFT", "UPDATE", + "CHANGE", "FILL", "MATCH", "SCORES", "UPLUS", + "CLUSTER", "FLOAT", "MAXROWS", "SELECT", "USE", + "COLON", "FOR", "MINROWS", "SEMI", "USER", + "COLUMN", "FROM", "MINUS", "SESSION", "USERS", + "COMMA", "FSYNC", "MNODES", "SET", "USING", + "COMP", "GE", "MODIFY", "SHOW", "VALUES", + "COMPACT", "GLOB", "MODULES", "SLASH", "VARIABLE", + "CONCAT", "GRANTS", "NCHAR", "SLIDING", "VARIABLES", + "CONFLICT", "GROUP", "NE", "SLIMIT", "VGROUPS", + "CONNECTION", "GT", "NONE", "SMALLINT", "VIEW", + "CONNECTIONS", "HAVING", "NOT", "SOFFSET", "VNODES", + "CONNS", "ID", "NOTNULL", "STABLE", "WAL", + "COPY", "IF", "NOW", "STABLES", "WHERE", +] + # basic data type boundary TINYINT_MAX = 127 TINYINT_MIN = -128 @@ -11,7 +82,7 @@ SMALLINT_MAX = 32767 SMALLINT_MIN = -32768 SMALLINT_UN_MAX = 65535 -MALLINT_UN_MIN = 0 +SMALLINT_UN_MIN = 0 INT_MAX = 2147483647 INT_MIN = -2147483648 @@ -33,8 +104,8 @@ DOUBLE_MIN = -1.7E+308 # schema boundary BINARY_LENGTH_MAX = 16374 -NCAHR_LENGTH_MAX_ = 4093 -DBNAME_LENGTH_MAX_ = 64 +NCAHR_LENGTH_MAX = 4093 +DBNAME_LENGTH_MAX = 64 STBNAME_LENGTH_MAX = 192 STBNAME_LENGTH_MIN = 1 @@ -66,4 +137,32 @@ MNODE_SHM_SIZE_DEFAULT = 6292480 VNODE_SHM_SIZE_MAX = 2147483647 VNODE_SHM_SIZE_MIN = 6292480 -VNODE_SHM_SIZE_DEFAULT = 31458304 \ No newline at end of file +VNODE_SHM_SIZE_DEFAULT = 31458304 + +# time_init +TIME_MS = 1 +TIME_US = TIME_MS/1000 +TIME_NS = TIME_US/1000 + +TIME_S = 1000 * TIME_MS +TIME_M = 60 * TIME_S +TIME_H = 60 * TIME_M +TIME_D = 24 * TIME_H +TIME_W = 7 * TIME_D +TIME_N = 30 * TIME_D +TIME_Y = 365 * TIME_D + + +# session parameters +INTERVAL_MIN = 1 * TIME_MS if PRECISION == PRECISION_DEFAULT else 1 * TIME_US + + +# streams and related agg-function +SMA_INDEX_FUNCTIONS = ["MIN", "MAX"] +ROLLUP_FUNCTIONS = ["AVG", "SUM", "MIN", "MAX", "LAST", "FIRST"] +SMA_WATMARK_MAXDELAY_INIT = ['a', "s", "m"] +WATERMARK_MAX = 900000 +WATERMARK_MIN = 0 + +MAX_DELAY_MAX = 900000 +MAX_DELAY_MIN = 1 \ No newline at end of file diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 580fc8ee47..fe802dd9a3 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -21,6 +21,7 @@ import psutil import shutil import pandas as pd from util.log import * +from util.constant import * def _parse_datetime(timestr): try: @@ -117,8 +118,7 @@ class TDSql: col_name_list = [] col_type_list = [] self.cursor.execute(sql) - self.queryCols = self.cursor.description - for query_col in self.queryCols: + for query_col in self.cursor.description: col_name_list.append(query_col[0]) col_type_list.append(query_col[1]) except Exception as e: @@ -301,6 +301,41 @@ class TDSql: args = (caller.filename, caller.lineno, self.sql, elm, expect_elm) tdLog.exit("%s(%d) failed: sql:%s, elm:%s == expect_elm:%s" % args) + def get_times(self, time_str, precision="ms"): + caller = inspect.getframeinfo(inspect.stack()[1][0]) + if time_str[-1] not in TAOS_TIME_INIT: + tdLog.exit(f"{caller.filename}({caller.lineno}) failed: {time_str} not a standard taos time init") + if precision not in TAOS_PRECISION: + tdLog.exit(f"{caller.filename}({caller.lineno}) failed: {precision} not a standard taos time precision") + + if time_str[-1] == TAOS_TIME_INIT[0]: + times = int(time_str[:-1]) * TIME_NS + if time_str[-1] == TAOS_TIME_INIT[1]: + times = int(time_str[:-1]) * TIME_US + if time_str[-1] == TAOS_TIME_INIT[2]: + times = int(time_str[:-1]) * TIME_MS + if time_str[-1] == TAOS_TIME_INIT[3]: + times = int(time_str[:-1]) * TIME_S + if time_str[-1] == TAOS_TIME_INIT[4]: + times = int(time_str[:-1]) * TIME_M + if time_str[-1] == TAOS_TIME_INIT[5]: + times = int(time_str[:-1]) * TIME_H + if time_str[-1] == TAOS_TIME_INIT[6]: + times = int(time_str[:-1]) * TIME_D + if time_str[-1] == TAOS_TIME_INIT[7]: + times = int(time_str[:-1]) * TIME_W + if time_str[-1] == TAOS_TIME_INIT[8]: + times = int(time_str[:-1]) * TIME_N + if time_str[-1] == TAOS_TIME_INIT[9]: + times = int(time_str[:-1]) * TIME_Y + + if precision == "ms": + return int(times) + elif precision == "us": + return int(times*1000) + elif precision == "ns": + return int(times*1000*1000) + def taosdStatus(self, state): tdLog.sleep(5) pstate = 0 diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 1b61bd1890..d3a6a0d102 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -22,15 +22,19 @@ # ---- dnode ./test.sh -f tsim/dnode/balance_replica1.sim +#./test.sh -f tsim/dnode/balance_replica3.sim ./test.sh -f tsim/dnode/create_dnode.sim ./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim ./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim ./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim -#./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim -#./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim -#./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim -#./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim +./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim +./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim +./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim ./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim # ---- insert ./test.sh -f tsim/insert/basic0.sim diff --git a/tests/script/test-all.bat b/tests/script/test-all.bat index 7a1a4bc7fa..81c1f21c04 100644 --- a/tests/script/test-all.bat +++ b/tests/script/test-all.bat @@ -11,14 +11,17 @@ if not "%2" == "" ( ) for /F "usebackq tokens=*" %%i in (!caseFile!) do ( set line=%%i - if "!line:~,9!" == "./test.sh" ( - set /a a+=1 - echo !a! Processing %%i - call :GetTimeSeconds !time! - set time1=!_timeTemp! - echo Start at !time! - call !line:./test.sh=wtest.bat! > result_!a!.txt 2>error_!a!.txt - if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. ) + call :CheckSkipCase %%i + if !skipCase! == false ( + if "!line:~,9!" == "./test.sh" ( + set /a a+=1 + echo !a! Processing %%i + call :GetTimeSeconds !time! + set time1=!_timeTemp! + echo Start at !time! + call !line:./test.sh=wtest.bat! > result_!a!.txt 2>error_!a!.txt + if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. ) + ) ) ) exit !exitNum! @@ -56,3 +59,10 @@ for %%a in (%tt%) do ( ) set /a _timeTemp=(%hh%*60+%mm%)*60+%ss% goto :eof + +:CheckSkipCase +set skipCase=false +if "%*" == "./test.sh -f tsim/query/scalarFunction.sim" ( set skipCase=true ) +if "%*" == "./test.sh -f tsim/stream/distributeInterval0.sim" ( set skipCase=true ) +if "%*" == "./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim" ( set skipCase=true ) +:goto eof \ No newline at end of file diff --git a/tests/script/tsim/dnode/balance_replica1.sim b/tests/script/tsim/dnode/balance_replica1.sim index 14f3f130fb..998d0654ab 100644 --- a/tests/script/tsim/dnode/balance_replica1.sim +++ b/tests/script/tsim/dnode/balance_replica1.sim @@ -117,7 +117,6 @@ if $rows != 6 then return -1 endi -return system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/balance_replica3.sim b/tests/script/tsim/dnode/balance_replica3.sim new file mode 100644 index 0000000000..b8996e259d --- /dev/null +++ b/tests/script/tsim/dnode/balance_replica3.sim @@ -0,0 +1,362 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 +system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode5 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +sql connect + +print =============== step1 create dnode2 +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 +sql create dnode $hostname port 7500 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 +if $rows != 5 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 +endi + +print =============== step2: create db +sql create database d1 vgroups 4 replica 3 + +print =============== step32 wait vgroup2 +$x = 0 +step32: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 +endi +if $data(2)[6] == leader then + $leaderExist = 1 +endi +if $data(2)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step32 +endi + +print =============== step33 wait vgroup3 +$x = 0 +step33: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(3)[4] == leader then + $leaderExist = 1 +endi +if $data(3)[6] == leader then + $leaderExist = 1 +endi +if $data(3)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step33 +endi + +print =============== step34 wait vgroup4 +$x = 0 +step34: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step34 +endi + +print =============== step35 wait vgroup5 +$x = 0 +step35: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step35 +endi + +print =============== step36: create table +sql use d1 +sql create table d1.st (ts timestamp, i int) tags (j int) +sql create table d1.c1 using st tags(1) +sql create table d1.c2 using st tags(1) +sql create table d1.c3 using st tags(1) +sql create table d1.c4 using st tags(1) +sql create table d1.c5 using st tags(1) +sql create table d1.c6 using st tags(1) +sql show d1.tables +if $rows != 6 then + return -1 +endi + +print =============== step4: start dnode5 +system sh/exec.sh -n dnode5 -s start +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 +if $rows != 5 then + return -1 +endi +if $data(1)[4] != ready then + goto step4 +endi +if $data(2)[4] != ready then + goto step4 +endi +if $data(3)[4] != ready then + goto step4 +endi +if $data(4)[4] != ready then + goto step4 +endi +if $data(5)[4] != ready then + goto step4 +endi + +print =============== step5: balance +sql balance vgroup + +print =============== step62 wait vgroup2 +$x = 0 +step62: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 +endi +if $data(2)[6] == leader then + $leaderExist = 1 +endi +if $data(2)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step62 +endi + +print =============== step63 wait vgroup3 +$x = 0 +step63: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(3)[4] == leader then + $leaderExist = 1 +endi +if $data(3)[6] == leader then + $leaderExist = 1 +endi +if $data(3)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step63 +endi + +print =============== step64 wait vgroup4 +$x = 0 +step64: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step64 +endi + +print =============== step65 wait vgroup5 +$x = 0 +step65: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step65 +endi + + + +print =============== step7: select data + +sql show d1.tables +print rows $rows +if $rows != 6 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim b/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim index 35a2466297..e1b7c4a3a4 100644 --- a/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim +++ b/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim @@ -28,7 +28,7 @@ step1: sql show dnodes print ===> $data00 $data01 $data02 $data03 $data04 $data05 print ===> $data10 $data11 $data12 $data13 $data14 $data15 -if $rows != 3 then +if $rows != 5 then return -1 endi if $data(1)[4] != ready then @@ -49,6 +49,128 @@ endi print =============== step3 create database sql create database d1 vgroups 4 replica 3 + +print =============== step32 wait vgroup2 +$x = 0 +step32: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 +endi +if $data(2)[6] == leader then + $leaderExist = 1 +endi +if $data(2)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step32 +endi + +print =============== step33 wait vgroup3 +$x = 0 +step33: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(3)[4] == leader then + $leaderExist = 1 +endi +if $data(3)[6] == leader then + $leaderExist = 1 +endi +if $data(3)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step33 +endi + +print =============== step34 wait vgroup4 +$x = 0 +step34: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step34 +endi + +print =============== step35 wait vgroup5 +$x = 0 +step35: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step35 +endi + +print =============== step36: create table sql use d1 sql create table d1.st (ts timestamp, i int) tags (j int) sql create table d1.c1 using st tags(1) @@ -57,21 +179,6 @@ if $rows != 1 then return -1 endi -sql show d1.vgroups -print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] -if $rows != 1 then - return -1 -endi -if $data(2)[3] != 2 then - return -1 -endi -if $data(2)[5] != 3 then - return -1 -endi -if $data(2)[7] != 4 then - return -1 -endi - print =============== step4: drop dnode 2 system sh/exec.sh -n dnode5 -s start $x = 0 @@ -85,7 +192,7 @@ step4: sql show dnodes print ===> $data00 $data01 $data02 $data03 $data04 $data05 print ===> $data10 $data11 $data12 $data13 $data14 $data15 -if $rows != 3 then +if $rows != 5 then return -1 endi if $data(1)[4] != ready then @@ -115,21 +222,133 @@ if $rows != 4 then return -1 endi -print show d1.vgroups +print =============== step62 wait vgroup2 +$x = 0 +step62: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi sql show d1.vgroups -print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] -if $rows != 1 then +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then return -1 endi +if $data(2)[4] == leader then + $leaderExist = 1 +endi +if $data(2)[6] == leader then + $leaderExist = 1 +endi +if $data(2)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step62 +endi +print =============== step63 wait vgroup3 +$x = 0 +step63: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(3)[4] == leader then + $leaderExist = 1 +endi +if $data(3)[6] == leader then + $leaderExist = 1 +endi +if $data(3)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step63 +endi -print =============== step6: select data +print =============== step64 wait vgroup4 +$x = 0 +step64: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step64 +endi + +print =============== step35 wait vgroup5 +$x = 0 +step65: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step65 +endi + +print =============== step7: select data +print ===> $data00 $data01 sql show d1.tables if $rows != 1 then return -1 endi -return system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim b/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim index 3ea351f7a7..4855f4dccf 100644 --- a/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim +++ b/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim @@ -49,6 +49,50 @@ endi print =============== step3 create database sql create database d1 vgroups 1 replica 3 +$leaderExist = 0 +$leaderVnode = 0 +$follower1 = 0 +$follower2 = 0 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $rows != 1 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 + $leaderVnode = 2 + $follower1 = 3 + $follower2 = 4 +endi +if $data(2)[6] == leader then + $leaderExist = 1 + $leaderVnode = 3 + $follower1 = 2 + $follower2 = 4 +endi +if $data(2)[8] == leader then + $leaderExist = 1 + $leaderVnode = 4 + $follower1 = 2 + $follower2 = 3 +endi +if $leaderExist != 1 then + goto step3 +endi + +print leader $leaderVnode +print follower1 $follower1 +print follower2 $follower2 + sql use d1 sql create table d1.st (ts timestamp, i int) tags (j int) sql create table d1.c1 using st tags(1) @@ -72,13 +116,6 @@ if $data(2)[7] != 4 then return -1 endi -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT - - -return - - print =============== step4: drop dnode 2 system sh/exec.sh -n dnode5 -s start $x = 0 @@ -92,7 +129,7 @@ step4: sql show dnodes print ===> $data00 $data01 $data02 $data03 $data04 $data05 print ===> $data10 $data11 $data12 $data13 $data14 $data15 -if $rows != 3 then +if $rows != 5 then return -1 endi if $data(1)[4] != ready then @@ -116,8 +153,11 @@ sql drop dnode 2 print show dnodes; sql show dnodes; -print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] -print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 if $rows != 4 then return -1 endi @@ -128,9 +168,9 @@ print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] if $rows != 1 then return -1 endi -#if $data(2)[3] != 3 then -# return -1 -#endi +if $data(2)[3] != 5 then + return -1 +endi print =============== step6: select data sql show d1.tables @@ -138,7 +178,37 @@ if $rows != 1 then return -1 endi -return +$leaderExist = 0 +$leaderVnode = 0 +$follower1 = 0 +$follower2 = 0 + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $rows != 1 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 +endi +if $data(2)[6] == leader then + $leaderExist = 1 +endi +if $data(2)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step6 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica1.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica1.sim new file mode 100644 index 0000000000..d3b5a02a23 --- /dev/null +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica1.sim @@ -0,0 +1,178 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +sql connect +sql create user u1 pass 'taosdata' + +print =============== step1 create dnode2 +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi + +print =============== step2: create db +sql create database d1 vgroups 1 replica 1 + +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 +endi +if $data(1)[4] != ready then + goto step2 +endi +if $data(2)[4] != ready then + goto step2 +endi +if $data(3)[4] != ready then + goto step2 +endi +if $data(4)[4] != ready then + goto step2 +endi + +print =============== step3: create database +sql use d1 +sql create table d1.st (ts timestamp, i int) tags (j int) +sql create table d1.c1 using st tags(1) +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step40: +print redistribute vgroup 2 dnode 3 +sql redistribute vgroup 2 dnode 3 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step41: +print redistribute vgroup 2 dnode 4 +sql redistribute vgroup 2 dnode 4 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step42: +print redistribute vgroup 2 dnode 2 +sql redistribute vgroup 2 dnode 2 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step43: +print redistribute vgroup 2 dnode 3 +sql redistribute vgroup 2 dnode 3 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step44: +print redistribute vgroup 2 dnode 4 +sql redistribute vgroup 2 dnode 4 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step45: +print redistribute vgroup 2 dnode 2 +sql redistribute vgroup 2 dnode 2 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step46: +print redistribute vgroup 2 dnode 3 +sql redistribute vgroup 2 dnode 3 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step47: +print redistribute vgroup 2 dnode 2 +sql redistribute vgroup 2 dnode 2 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim index 9b11e16616..00c6e8a98d 100644 --- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim @@ -65,7 +65,7 @@ sql_error redistribute vgroup 3 dnode 6 dnode 3 dnode 4 # vgroup not exist sql_error redistribute vgroup 3 dnode 5 dnode 3 dnode 4 # un changed -sql_error redistribute vgroup 2 dnode 2 dnode 3 dnode 4 +# sql_error redistribute vgroup 2 dnode 2 dnode 3 dnode 4 # no enought vnodes sql_error redistribute vgroup 2 dnode 1 dnode 3 dnode 4 # offline vnodes @@ -176,8 +176,6 @@ if $rows != 1 then return -1 endi -return - print =============== step32: print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim new file mode 100644 index 0000000000..6614009497 --- /dev/null +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim @@ -0,0 +1,245 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 +system sh/deploy.sh -n dnode6 -i 6 +system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode5 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode6 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +sql connect +sql create user u1 pass 'taosdata' + +print =============== step1 create dnode2 +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 +sql create dnode $hostname port 7500 +sql create dnode $hostname port 7600 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 +if $rows != 6 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 +endi + +print =============== step2: create db +sql create database d1 vgroups 1 replica 3 + +system sh/exec.sh -n dnode5 -s start +system sh/exec.sh -n dnode6 -s start +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 +if $rows != 6 then + return -1 +endi +if $data(1)[4] != ready then + goto step2 +endi +if $data(2)[4] != ready then + goto step2 +endi +if $data(3)[4] != ready then + goto step2 +endi +if $data(4)[4] != ready then + goto step2 +endi +if $data(5)[4] != ready then + goto step2 +endi +if $data(6)[4] != ready then + goto step2 +endi + +print =============== step3: create database +$leaderExist = 0 +$leaderVnode = 0 +$follower1 = 0 +$follower2 = 0 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $rows != 1 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 + $leaderVnode = 2 + $follower1 = 3 + $follower2 = 4 +endi +if $data(2)[6] == leader then + $leaderExist = 1 + $leaderVnode = 3 + $follower1 = 2 + $follower2 = 4 +endi +if $data(2)[8] == leader then + $leaderExist = 1 + $leaderVnode = 4 + $follower1 = 2 + $follower2 = 3 +endi +if $leaderExist != 1 then + goto step3 +endi + +print leader $leaderVnode +print follower1 $follower1 +print follower2 $follower2 + +sql use d1 +sql create table d1.st (ts timestamp, i int) tags (j int) +sql create table d1.c1 using st tags(1) +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step40: +print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode 6 +sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode 6 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step41: +print redistribute vgroup 2 dnode 5 dnode $follower2 dnode $follower1 +sql redistribute vgroup 2 dnode 5 dnode $follower2 dnode $follower1 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step42: +print redistribute vgroup 2 dnode $follower2 dnode 6 dnode $leaderVnode +sql redistribute vgroup 2 dnode $follower2 dnode 6 dnode $leaderVnode +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step43: +print redistribute vgroup 2 dnode $follower1 dnode 5 dnode 6 +sql redistribute vgroup 2 dnode $follower1 dnode 5 dnode 6 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step44: +print redistribute vgroup 2 dnode $follower2 dnode 5 dnode $leaderVnode +sql redistribute vgroup 2 dnode $follower2 dnode 5 dnode $leaderVnode +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step45: +print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 6 +sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 6 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step46: +print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 +sql redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step47: +print redistribute vgroup 2 dnode $follower1 dnode 6 dnode $leaderVnode +sql redistribute vgroup 2 dnode $follower1 dnode 6 dnode $leaderVnode +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim new file mode 100644 index 0000000000..c78d90e352 --- /dev/null +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim @@ -0,0 +1,263 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 +system sh/deploy.sh -n dnode6 -i 6 +system sh/deploy.sh -n dnode7 -i 7 +system sh/deploy.sh -n dnode8 -i 8 +system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode5 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode6 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode7 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode8 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +sql connect +sql create user u1 pass 'taosdata' + +print =============== step1 create dnode2 +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 +sql create dnode $hostname port 7500 +sql create dnode $hostname port 7600 +sql create dnode $hostname port 7700 +sql create dnode $hostname port 7800 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 +if $rows != 8 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 +endi + +print =============== step2: create db +sql create database d1 vgroups 1 replica 3 + +system sh/exec.sh -n dnode5 -s start +system sh/exec.sh -n dnode6 -s start +system sh/exec.sh -n dnode7 -s start +system sh/exec.sh -n dnode8 -s start +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 +if $rows != 8 then + return -1 +endi +if $data(1)[4] != ready then + goto step2 +endi +if $data(2)[4] != ready then + goto step2 +endi +if $data(3)[4] != ready then + goto step2 +endi +if $data(4)[4] != ready then + goto step2 +endi +if $data(5)[4] != ready then + goto step2 +endi +if $data(6)[4] != ready then + goto step2 +endi +if $data(7)[4] != ready then + goto step2 +endi +if $data(8)[4] != ready then + goto step2 +endi + +print =============== step3: create database +$leaderExist = 0 +$leaderVnode = 0 +$follower1 = 0 +$follower2 = 0 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $rows != 1 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 + $leaderVnode = 2 + $follower1 = 3 + $follower2 = 4 +endi +if $data(2)[6] == leader then + $leaderExist = 1 + $leaderVnode = 3 + $follower1 = 2 + $follower2 = 4 +endi +if $data(2)[8] == leader then + $leaderExist = 1 + $leaderVnode = 4 + $follower1 = 2 + $follower2 = 3 +endi +if $leaderExist != 1 then + goto step3 +endi + +print leader $leaderVnode +print follower1 $follower1 +print follower2 $follower2 + +sql use d1 +sql create table d1.st (ts timestamp, i int) tags (j int) +sql create table d1.c1 using st tags(1) +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step40: +print redistribute vgroup 2 dnode 7 dnode 5 dnode 6 +sql redistribute vgroup 2 dnode 7 dnode 5 dnode 6 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step41: +print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode $follower1 +sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode $follower1 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step42: +print redistribute vgroup 2 dnode 7 dnode 6 dnode 8 +sql redistribute vgroup 2 dnode 7 dnode 6 dnode 8 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step43: +print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 +sql redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + + +return + + +print =============== step44: +print redistribute vgroup 2 dnode 6 dnode 7 dnode $leaderVnode +sql redistribute vgroup 2 dnode 6 dnode 7 dnode $leaderVnode +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step45: +print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 8 +sql redistribute vgroup 2dnode $follower1 dnode $follower2 dnode 8 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step46: +print redistribute vgroup 2 dnode $leaderVnode dnode 6 dnode 5 +sql redistribute vgroup 2 dnode $leaderVnode dnode 6 dnode 5 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step47: +print redistribute vgroup 2 dnode $follower1 dnode 7 dnode 8 +sql redistribute vgroup 2 dnode $follower1 dnode 7 dnode 8 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim index 8936655974..a59e989d80 100644 --- a/tests/script/tsim/stream/distributeInterval0.sim +++ b/tests/script/tsim/stream/distributeInterval0.sim @@ -41,7 +41,7 @@ sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); -sql create stream stream_t1 trigger at_once into streamtST1 as select _wstartts, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); +sql create stream stream_t1 trigger at_once watermark 1d into streamtST1 as select _wstartts, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); sleep 1000 diff --git a/tests/script/tsim/stream/distributeIntervalRetrive0.sim b/tests/script/tsim/stream/distributeIntervalRetrive0.sim new file mode 100644 index 0000000000..cde5c7058f --- /dev/null +++ b/tests/script/tsim/stream/distributeIntervalRetrive0.sim @@ -0,0 +1,291 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 + +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +sql create dnode $hostname2 port 7200 + +system sh/exec.sh -n dnode2 -s start + +print ===== step1 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 2 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi + +print ===== step2 + +sql create database test vgroups 4; +sql use test; +sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); +sql create table ts3 using st tags(3,2,2); +sql create table ts4 using st tags(4,2,2); +sql create stream stream_t1 trigger at_once into streamtST1 as select _wstartts, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); + +sleep 1000 + +sql insert into ts1 values(1648791213001,1,12,3,1.0); +sql insert into ts2 values(1648791213001,1,12,3,1.0); +sql insert into ts1 values(1648791213002,NULL,NULL,NULL,NULL); +sql insert into ts2 values(1648791213002,NULL,NULL,NULL,NULL); + +sql insert into ts1 values(1648791223002,2,2,3,1.1); +sql insert into ts1 values(1648791233003,3,2,3,2.1); +sql insert into ts2 values(1648791243004,4,2,43,73.1); + +sql insert into ts1 values(1648791213002,24,22,23,4.1) (1648791243005,4,20,3,3.1); + +sleep 1000 + +sql insert into ts3 values(1648791213001,12,12,13,14.1) (1648791243005,14,30,30,30.1); + +$loop_count = 0 +loop1: +sleep 1000 +sql select * from streamtST1; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data01 != 5 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != 14 then + print =====data02=$data02 + goto loop1 +endi + +# row 1 +if $data11 != 1 then + print =====data11=$data11 + goto loop1 +endi + +if $data12 != 2 then + print =====data12=$data12 + goto loop1 +endi + +#row2 +if $data21 != 1 then + print =====data21=$data21 + goto loop1 +endi + +if $data22 != 3 then + print =====data22=$data22 + goto loop1 +endi + +#row 3 +if $data31 != 3 then + print =====data31=$data31 + goto loop1 +endi + +if $data32 != 22 then + print =====data32=$data32 + goto loop1 +endi + +print loop1 over + +sql insert into ts1 values(1648791223008,4,2,30,3.1) (1648791213009,4,2,3,3.1) (1648791233010,4,2,3,3.1) (1648791243011,4,2,3,3.1)(1648791243012,34,32,33,3.1); + +$loop_count = 0 +loop2: +sleep 1000 +sql select * from streamtST1; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data01 != 6 then + print =====data01=$data01 + goto loop2 +endi + +if $data02 != 18 then + print =====data02=$data02 + goto loop2 +endi + +# row 1 +if $data11 != 2 then + print =====data11=$data11 + goto loop2 +endi + +if $data12 != 6 then + print =====data12=$data12 + goto loop2 +endi + +#row2 +if $data21 != 2 then + print =====data21=$data21 + goto loop2 +endi + +if $data22 != 7 then + print =====data22=$data22 + goto loop2 +endi + +#row 3 +if $data31 != 5 then + print =====data31=$data31 + goto loop2 +endi + +if $data32 != 60 then + print =====data32=$data32 + goto loop2 +endi + +print loop2 over + +sql insert into ts4 values(1648791223008,4,2,30,3.1) (1648791213009,4,2,3,3.1) (1648791233010,4,2,3,3.1); + +$loop_count = 0 +loop3: +sleep 1000 +sql select * from streamtST1; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data01 != 7 then + print =====data01=$data01 + goto loop3 +endi + +if $data02 != 22 then + print =====data02=$data02 + goto loop3 +endi + +# row 1 +if $data11 != 3 then + print =====data11=$data11 + goto loop3 +endi + +if $data12 != 10 then + print =====data12=$data12 + goto loop3 +endi + +#row2 +if $data21 != 3 then + print =====data21=$data21 + goto loop3 +endi + +if $data22 != 11 then + print =====data22=$data22 + goto loop3 +endi + +#row 3 +if $data31 != 5 then + print =====data31=$data31 + goto loop3 +endi + +if $data32 != 60 then + print =====data32=$data32 + goto loop3 +endi + +print loop3 over + +$loop_count = 0 +loop4: +sleep 1000 +sql select * from streamtST1; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data01 != 7 then + print =====data01=$data01 + goto loop4 +endi + +if $data02 != 22 then + print =====data02=$data02 + goto loop4 +endi + +# row 1 +if $data11 != 3 then + print =====data11=$data11 + goto loop4 +endi + +if $data12 != 10 then + print =====data12=$data12 + goto loop4 +endi + +#row2 +if $data21 != 3 then + print =====data21=$data21 + goto loop4 +endi + +if $data22 != 11 then + print =====data22=$data22 + goto loop4 +endi + +#row 3 +if $data31 != 5 then + print =====data31=$data31 + goto loop4 +endi + +if $data32 != 60 then + print =====data32=$data32 + goto loop4 +endi + +print loop4 over + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/distributesession0.sim b/tests/script/tsim/stream/distributeSession0.sim similarity index 100% rename from tests/script/tsim/stream/distributesession0.sim rename to tests/script/tsim/stream/distributeSession0.sim diff --git a/tests/system-test/1-insert/alter_table.py b/tests/system-test/1-insert/alter_table.py index a4e40d1b0b..0f7a830634 100644 --- a/tests/system-test/1-insert/alter_table.py +++ b/tests/system-test/1-insert/alter_table.py @@ -16,273 +16,272 @@ import string from util.log import * from util.cases import * from util.sql import * +from util import constant +from util.common import * +from util.sqlset import * class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + self.ntbname = 'ntb' + self.stbname = 'stb' + self.binary_length = 20 # the length of binary for column_dict + self.nchar_length = 20 # the length of nchar for column_dict + self.column_dict = { + 'ts' : 'timestamp', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.binary_length})', + 'col13': f'nchar({self.nchar_length})' + } + self.tag_dict = { + 'ts_tag' : 'timestamp', + 't1': 'tinyint', + 't2': 'smallint', + 't3': 'int', + 't4': 'bigint', + 't5': 'tinyint unsigned', + 't6': 'smallint unsigned', + 't7': 'int unsigned', + 't8': 'bigint unsigned', + 't9': 'float', + 't10': 'double', + 't11': 'bool', + 't12': f'binary({self.binary_length})', + 't13': f'nchar({self.nchar_length})' + } + self.tag_list = [ + f'now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据"' + ] + self.tbnum = 1 + self.values_list = [ + f'now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据"' + ] + self.column_add_dict = { + 'col_time' : 'timestamp', + 'col_tinyint' : 'tinyint', + 'col_smallint' : 'smallint', + 'col_int' : 'int', + 'col_bigint' : 'bigint', + 'col_untinyint' : 'tinyint unsigned', + 'col_smallint' : 'smallint unsigned', + 'col_int' : 'int unsigned', + 'col_bigint' : 'bigint unsigned', + 'col_bool' : 'bool', + 'col_float' : 'float', + 'col_double' : 'double', + 'col_binary' : f'binary({constant.BINARY_LENGTH_MAX})', + 'col_nchar' : f'nchar({constant.NCAHR_LENGTH_MAX})' - def get_long_name(self, length, mode="mixed"): - """ - generate long name - mode could be numbers/letters/letters_mixed/mixed - """ - if mode == "numbers": - population = string.digits - elif mode == "letters": - population = string.ascii_letters.lower() - elif mode == "letters_mixed": - population = string.ascii_letters.upper() + string.ascii_letters.lower() - else: - population = string.ascii_letters.lower() + string.digits - return "".join(random.choices(population, k=length)) - - def alter_tb_tag_check(self): - tag_tinyint = random.randint(-127,129) - tag_int = random.randint(-2147483648,2147483647) - tag_smallint = random.randint(-32768,32768) - tag_bigint = random.randint(-2147483648,2147483647) - tag_untinyint = random.randint(0,256) - tag_unsmallint = random.randint(0,65536) - tag_unint = random.randint(0,4294967296) - tag_unbigint = random.randint(0,2147483647) - tag_binary = self.get_long_name(length=10, mode="letters") - tag_nchar = self.get_long_name(length=10, mode="letters") - dbname = self.get_long_name(length=10, mode="letters") - tdSql.execute(f'create database if not exists {dbname}') - stbname = self.get_long_name(length=3, mode="letters") - tbname = self.get_long_name(length=3, mode="letters") - tdSql.execute(f'create stable if not exists {dbname}.{stbname} (col_ts timestamp, c1 int) tags (tag_ts timestamp, t1 tinyint, t2 smallint, t3 int, \ - t4 bigint, t5 tinyint unsigned, t6 smallint unsigned, t7 int unsigned, t8 bigint unsigned, t9 float, t10 double, t11 bool,t12 binary(20),t13 nchar(20))') - tdSql.execute(f'create table if not exists {dbname}.{tbname} using {dbname}.{stbname} tags(now, 1, 2, 3, 4, 5, 6, 7, 8, 9.9, 10.1, True,"abc123","涛思数据")') - tdSql.execute(f'insert into {dbname}.{tbname} values(now, 1)') - tdSql.execute(f'alter table {dbname}.{tbname} set tag tag_ts = 1640966400000') - tdSql.execute(f'alter table {dbname}.{tbname} set tag `t1` = 11') - tdSql.query(f'select * from {dbname}.{stbname}') - tdSql.checkData(0,3,11) - tdSql.execute(f'alter table {dbname}.{tbname} set tag t1 = {tag_tinyint}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t2 = {tag_smallint}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t3 = {tag_int}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t4 = {tag_bigint}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t5 = {tag_untinyint}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t6 = {tag_unsmallint}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t7 = {tag_unint}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t8 = {tag_unbigint}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t11 = false') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t12 = "{tag_binary}"') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t13 = "{tag_nchar}"') - tdSql.query(f'select * from {dbname}.{stbname}') - # bug TD-15899 - tdSql.checkData(0,2,'2022-01-01 00:00:00.000') - tdSql.checkData(0,3,tag_tinyint) - tdSql.checkData(0,4,tag_smallint) - tdSql.checkData(0,5,tag_int) - tdSql.checkData(0,6,tag_bigint) - tdSql.checkData(0,7,tag_untinyint) - tdSql.checkData(0,8,tag_unsmallint) - tdSql.checkData(0,9,tag_unint) - tdSql.checkData(0,10,tag_unbigint) + } + def alter_check_ntb(self): - tdSql.checkData(0,13,False) - tdSql.checkData(0,14,tag_binary) - tdSql.checkData(0,15,tag_nchar) + tdSql.prepare() + tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) + for i in self.values_list: + tdSql.execute(f'insert into {self.ntbname} values({i})') + for key,values in self.column_add_dict.items(): + tdSql.execute(f'alter table {self.ntbname} add column {key} {values}') + tdSql.query(f'describe {self.ntbname}') + tdSql.checkRows(len(self.column_dict)+1) + tdSql.query(f'select {key} from {self.ntbname}') + tdSql.checkRows(len(self.values_list)) + tdSql.execute(f'alter table {self.ntbname} drop column {key}') + tdSql.query(f'describe {self.ntbname}') + tdSql.checkRows(len(self.column_dict)) + tdSql.error(f'select {key} from {self.ntbname} ') + for key,values in self.column_dict.items(): + if 'binary' in values.lower(): + v = f'binary({self.binary_length+1})' + v_error = f'binary({self.binary_length-1})' + tdSql.error(f'alter table {self.ntbname} modify column {key} {v_error}') + tdSql.execute(f'alter table {self.ntbname} modify column {key} {v}') + tdSql.query(f'describe {self.ntbname}') + result = tdCom.getOneRow(1,'VARCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + elif 'nchar' in values.lower(): + v = f'nchar({self.binary_length+1})' + v_error = f'nchar({self.binary_length-1})' + tdSql.error(f'alter table {self.ntbname} modify column {key} {v_error}') + tdSql.execute(f'alter table {self.ntbname} modify column {key} {v}') + tdSql.query(f'describe {self.ntbname}') + result = tdCom.getOneRow(1,'NCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + else: + for v in self.column_dict.values(): + tdSql.error(f'alter table {self.ntbname} modify column {key} {v}') + for key,values in self.column_dict.items(): + rename_str = f'{tdCom.getLongName(constant.COL_NAME_LENGTH_MAX,"letters")}' + tdSql.execute(f'alter table {self.ntbname} rename column {key} {rename_str}') + tdSql.query(f'select {rename_str} from {self.ntbname}') + tdSql.checkRows(1) + + def alter_check_tb(self): + tag_tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) + tag_smallint = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) + tag_int = random.randint(constant.INT_MIN,constant.INT_MAX) + tag_bigint = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) + tag_untinyint = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) + tag_unsmallint = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) + tag_unint = random.randint(constant.INT_UN_MIN,constant.INT_MAX) + tag_unbigint = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) + tag_bool = random.randint(0,100)%2 + tag_float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) + tag_double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) + tag_binary = tdCom.getLongName(self.binary_length) + tag_nchar = tdCom.getLongName(self.binary_length) + modify_column_dict = { + 'ts1' : 'timestamp', + 'c1': 'tinyint', + 'c2': 'smallint', + 'c3': 'int', + 'c4': 'bigint', + 'c5': 'tinyint unsigned', + 'c6': 'smallint unsigned', + 'c7': 'int unsigned', + 'c8': 'bigint unsigned', + 'c9': 'float', + 'c10': 'double', + 'c11': 'bool', + 'c12': f'binary({self.binary_length})', + 'c13': f'nchar({self.nchar_length})' + } + tdSql.prepare() + tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) + for i in range(self.tbnum): + tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({self.tag_list[i]})') + for j in self.values_list: + tdSql.execute(f'insert into {self.stbname}_{i} values({j})') + for i in range(self.tbnum): + for k,v in modify_column_dict.items(): + tdSql.error(f'alter table {self.stbname}_{i} add column {k} {v}') + for k in self.column_dict.keys(): + tdSql.error(f'alter table {self.stbname}_{i} drop column {k}') + for k,v in self.column_dict.items(): + if 'binary' in v.lower(): + values = [f'binary({self.binary_length+1})', f'binary({self.binary_length-1})'] + for j in values: + tdSql.error(f'alter table {self.stbname}_{i} modify {k} {j}') + elif 'nchar' in v.lower(): + values = [f'nchar({self.nchar_length+1})', f'binary({self.nchar_length-1})'] + for j in values: + tdSql.error(f'alter table {self.stbname}_{i} modify {k} {j}') + else: + for values in self.column_dict.values(): + tdSql.error(f'alter table {self.stbname}_{i} modify column {k} {values}') + for k,v in self.tag_dict.items(): + if v.lower() == 'tinyint': + self.tag_check(i,k,tag_tinyint) + elif v.lower() == 'smallint': + self.tag_check(i,k,tag_smallint) + elif v.lower() == 'int': + self.tag_check(i,k,tag_int) + elif v.lower() == 'bigint': + self.tag_check(i,k,tag_bigint) + elif v.lower() == 'tinyint unsigned': + self.tag_check(i,k,tag_untinyint) + elif v.lower() == 'smallint unsigned': + self.tag_check(i,k,tag_unsmallint) + elif v.lower() == 'int unsigned': + self.tag_check(i,k,tag_unint) + elif v.lower() == 'bigint unsigned': + self.tag_check(i,k,tag_unbigint) + elif v.lower() == 'bool': + self.tag_check(i,k,tag_bool) + elif v.lower() == 'float': + tdSql.execute(f'alter table {self.stbname}_{i} set tag {k} = {tag_float}') + tdSql.query(f'select {k} from {self.stbname}_{i}') + if abs(tdSql.queryResult[0][0] - tag_float)/tag_float<=0.0001: + tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) + else: + tdLog.exit(f'select {k} from {self.stbname}_{i},data check failure') + elif v.lower() == 'double': + tdSql.execute(f'alter table {self.stbname}_{i} set tag {k} = {tag_double}') + tdSql.query(f'select {k} from {self.stbname}_{i}') + if abs(tdSql.queryResult[0][0] - tag_double)/tag_double<=0.0001: + tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) + else: + tdLog.exit(f'select {k} from {self.stbname}_{i},data check failure') + elif 'binary' in v.lower(): + tdSql.execute(f'alter table {self.stbname}_{i} set tag {k} = "{tag_binary}"') + tdSql.query(f'select {k} from {self.stbname}_{i}') + tdSql.checkData(0,0,tag_binary) + elif 'nchar' in v.lower(): + tdSql.execute(f'alter table {self.stbname}_{i} set tag {k} = "{tag_nchar}"') + tdSql.query(f'select {k} from {self.stbname}_{i}') + tdSql.checkData(0,0,tag_nchar) + + def tag_check(self,tb_no,tag,values): + tdSql.execute(f'alter table {self.stbname}_{tb_no} set tag {tag} = {values}') + tdSql.query(f'select {tag} from {self.stbname}_{tb_no}') + tdSql.checkData(0,0,values) + def alter_check_stb(self): + tdSql.prepare() + tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) + for i in range(self.tbnum): + tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({self.tag_list[i]})') + for j in self.values_list: + tdSql.execute(f'insert into {self.stbname}_{i} values({j})') + for key,values in self.column_add_dict.items(): + tdSql.execute(f'alter table {self.stbname} add column {key} {values}') + tdSql.query(f'describe {self.stbname}') + tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)+1) + for i in range(self.tbnum): + tdSql.query(f'describe {self.stbname}_{i}') + tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)+1) + tdSql.query(f'select {key} from {self.stbname}_{i}') + tdSql.checkRows(len(self.values_list)) + tdSql.execute(f'alter table {self.stbname} drop column {key}') + tdSql.query(f'describe {self.stbname}') + tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)) + for i in range(self.tbnum): + tdSql.query(f'describe {self.stbname}_{i}') + tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)) + tdSql.error(f'select {key} from {self.stbname} ') + for key,values in self.column_dict.items(): + if 'binary' in values.lower(): + v = f'binary({self.binary_length+1})' + v_error = f'binary({self.binary_length-1})' + tdSql.error(f'alter table {self.stbname} modify column {key} {v_error}') + tdSql.execute(f'alter table {self.stbname} modify column {key} {v}') + tdSql.query(f'describe {self.stbname}') + result = tdCom.getOneRow(1,'VARCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + for i in range(self.tbnum): + tdSql.query(f'describe {self.stbname}_{i}') + result = tdCom.getOneRow(1,'VARCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + elif 'nchar' in values.lower(): + v = f'nchar({self.binary_length+1})' + v_error = f'nchar({self.binary_length-1})' + tdSql.error(f'alter table {self.stbname} modify column {key} {v_error}') + tdSql.execute(f'alter table {self.stbname} modify column {key} {v}') + tdSql.query(f'describe {self.stbname}') + result = tdCom.getOneRow(1,'NCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + for i in range(self.tbnum): + tdSql.query(f'describe {self.stbname}') + result = tdCom.getOneRow(1,'NCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + else: + for v in self.column_dict.values(): + tdSql.error(f'alter table {self.stbname} modify column {key} {v}') - # bug TD-16211 insert length more than setting binary and nchar - # error_tag_binary = self.get_long_name(length=21, mode="letters") - # error_tag_nchar = self.get_long_name(length=21, mode="letters") - # tdSql.error(f'alter table {dbname}.{tbname} set tag t12 = "{error_tag_binary}"') - # tdSql.error(f'alter table {dbname}.{tbname} set tag t13 = "{error_tag_nchar}"') - error_tag_binary = self.get_long_name(length=25, mode="letters") - error_tag_nchar = self.get_long_name(length=25, mode="letters") - tdSql.error(f'alter table {dbname}.{tbname} set tag t12 = "{error_tag_binary}"') - tdSql.error(f'alter table {dbname}.{tbname} set tag t13 = "{error_tag_nchar}"') - # bug TD-16210 modify binary to nchar - tdSql.error(f'alter table {dbname}.{tbname} modify tag t12 nchar(10)') - tdSql.execute(f"drop database {dbname}") - def alter_ntb_column_check(self): - ''' - alter ntb column check - ''' - dbname = self.get_long_name(length=10, mode="letters") - tdSql.execute(f'create database if not exists {dbname}') - tbname = self.get_long_name(length=3, mode="letters") - tdLog.info('------------------normal table column check---------------------') - tdLog.info(f'-----------------create normal table {tbname}-------------------') - tdSql.execute(f'create table if not exists {dbname}.{tbname} (ts timestamp, c1 tinyint, c2 smallint, c3 int, \ - c4 bigint, c5 tinyint unsigned, c6 smallint unsigned, c7 int unsigned, c8 bigint unsigned, c9 float, c10 double, c11 bool,c12 binary(20),c13 nchar(20))') - tdSql.execute(f'insert into {dbname}.{tbname} values (now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")') - # bug TD-15757 - tdSql.execute(f'alter table {dbname}.{tbname} add column c14 int') - tdSql.query(f'select c14 from {dbname}.{tbname}') - tdSql.checkRows(1) - tdSql.execute(f'alter table {dbname}.{tbname} add column `c15` int') - tdSql.query(f'select c15 from {dbname}.{tbname}') - tdSql.checkRows(1) - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkRows(16) - tdSql.execute(f'alter table {dbname}.{tbname} drop column c14') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkRows(15) - tdSql.execute(f'alter table {dbname}.{tbname} drop column `c15`') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkRows(14) - #! TD-16422 - # tdSql.execute(f'alter table {dbname}.{tbname} add column c16 binary(10)') - # tdSql.query(f'describe {dbname}.{tbname}') - # tdSql.checkRows(15) - # tdSql.checkEqual(tdSql.queryResult[14][2],10) - # tdSql.execute(f'alter table {dbname}.{tbname} drop column c16') - - # tdSql.execute(f'alter table {dbname}.{tbname} add column c16 nchar(10)') - # tdSql.query(f'describe {dbname}.{tbname}') - # tdSql.checkRows(15) - # tdSql.checkEqual(tdSql.queryResult[14][2],10) - # tdSql.execute(f'alter table {dbname}.{tbname} drop column c16') - - - tdSql.execute(f'alter table {dbname}.{tbname} modify column c12 binary(30)') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkData(12,2,30) - tdSql.execute(f'alter table {dbname}.{tbname} modify column `c12` binary(35)') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkData(12,2,35) - tdSql.error(f'alter table {dbname}.{tbname} modify column c12 binary(34)') - tdSql.error(f'alter table {dbname}.{tbname} modify column c12 nchar(10)') - tdSql.error(f'alter table {dbname}.{tbname} modify column c12 int') - tdSql.execute(f'alter table {dbname}.{tbname} modify column c13 nchar(30)') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkData(13,2,30) - tdSql.execute(f'alter table {dbname}.{tbname} modify column `c13` nchar(35)') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkData(13,2,35) - tdSql.error(f'alter table {dbname}.{tbname} modify column c13 nchar(34)') - tdSql.error(f'alter table {dbname}.{tbname} modify column c13 binary(10)') - tdSql.execute(f'alter table {dbname}.{tbname} rename column c1 c21') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkData(1,0,'c21') - # !bug TD-16423 - # tdSql.error(f'select c1 from {dbname}.{tbname}') - # tdSql.query(f'select c21 from {dbname}.{tbname}') - # tdSql.checkData(0,1,1) - tdSql.execute(f'alter table {dbname}.{tbname} rename column `c21` c1') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkData(1,0,'c1') - # !bug TD-16423 - # tdSql.error(f'select c1 from {dbname}.{tbname}') - # tdSql.query(f'select c1 from {dbname}.{tbname}') - # tdSql.checkData(0,1,1) - tdSql.error(f'alter table {dbname}.{tbname} modify column c1 bigint') - tdSql.error(f'alter table {dbname}.{tbname} modify column c1 double') - tdSql.error(f'alter table {dbname}.{tbname} modify column c4 int') - tdSql.error(f'alter table {dbname}.{tbname} modify column `c1` double') - tdSql.error(f'alter table {dbname}.{tbname} modify column c9 double') - tdSql.error(f'alter table {dbname}.{tbname} modify column c10 float') - tdSql.error(f'alter table {dbname}.{tbname} modify column c1 bool') - tdSql.error(f'alter table {dbname}.{tbname} modify column c1 binary(10)') - tdSql.execute(f'drop database {dbname}') - def alter_stb_column_check(self): - dbname = self.get_long_name(length=10, mode="letters") - tdSql.execute(f'create database if not exists {dbname}') - stbname = self.get_long_name(length=3, mode="letters") - tbname = self.get_long_name(length=3, mode="letters") - tdSql.execute(f'create database if not exists {dbname}') - tdSql.execute(f'use {dbname}') - tdSql.execute( - f'create table {stbname} (ts timestamp, c1 tinyint, c2 smallint, c3 int, \ - c4 bigint, c5 tinyint unsigned, c6 smallint unsigned, c7 int unsigned, c8 bigint unsigned, c9 float, c10 double, c11 bool,c12 binary(20),c13 nchar(20)) tags(t0 int) ') - tdSql.execute(f'create table {tbname} using {stbname} tags(1)') - tdSql.execute(f'insert into {tbname} values (now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")') - tdSql.execute(f'alter table {stbname} add column c14 int') - tdSql.query(f'select c14 from {stbname}') - tdSql.checkRows(1) - tdSql.execute(f'alter table {stbname} add column `c15` int') - tdSql.query(f'select c15 from {stbname}') - tdSql.checkRows(1) - tdSql.query(f'describe {stbname}') - tdSql.checkRows(17) - tdSql.execute(f'alter table {stbname} drop column c14') - tdSql.query(f'describe {stbname}') - tdSql.checkRows(16) - tdSql.execute(f'alter table {stbname} drop column `c15`') - tdSql.query(f'describe {stbname}') - tdSql.checkRows(15) - tdSql.execute(f'alter table {stbname} modify column c12 binary(30)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(12,2,30) - tdSql.execute(f'alter table {stbname} modify column `c12` binary(35)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(12,2,35) - tdSql.error(f'alter table {stbname} modify column `c12` binary(34)') - tdSql.execute(f'alter table {stbname} modify column c13 nchar(30)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(13,2,30) - tdSql.error(f'alter table {stbname} modify column c13 nchar(29)') - tdSql.error(f'alter table {stbname} rename column c1 c21') - tdSql.error(f'alter table {stbname} modify column c1 int') - tdSql.error(f'alter table {stbname} modify column c4 int') - tdSql.error(f'alter table {stbname} modify column c8 int') - tdSql.error(f'alter table {stbname} modify column c1 unsigned int') - tdSql.error(f'alter table {stbname} modify column c9 double') - tdSql.error(f'alter table {stbname} modify column c10 float') - tdSql.error(f'alter table {stbname} modify column c11 int') - tdSql.execute(f'drop database {dbname}') - def alter_stb_tag_check(self): - dbname = self.get_long_name(length=10, mode="letters") - tdSql.execute(f'create database if not exists {dbname}') - stbname = self.get_long_name(length=3, mode="letters") - tbname = self.get_long_name(length=3, mode="letters") - tdSql.execute(f'create database if not exists {dbname}') - tdSql.execute(f'use {dbname}') - tdSql.execute( - f'create table {stbname} (ts timestamp, c1 int) tags(ts_tag timestamp, t1 tinyint, t2 smallint, t3 int, \ - t4 bigint, t5 tinyint unsigned, t6 smallint unsigned, t7 int unsigned, t8 bigint unsigned, t9 float, t10 double, t11 bool,t12 binary(20),t13 nchar(20)) ') - tdSql.execute(f'create table {tbname} using {stbname} tags(now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")') - tdSql.execute(f'insert into {tbname} values(now,1)') - - tdSql.execute(f'alter table {stbname} add tag t14 int') - tdSql.query(f'select t14 from {stbname}') - tdSql.checkRows(1) - tdSql.execute(f'alter table {stbname} add tag `t15` int') - tdSql.query(f'select t14 from {stbname}') - tdSql.checkRows(1) - tdSql.query(f'describe {stbname}') - tdSql.checkRows(18) - tdSql.execute(f'alter table {stbname} drop tag t14') - tdSql.query(f'describe {stbname}') - tdSql.checkRows(17) - tdSql.execute(f'alter table {stbname} drop tag `t15`') - tdSql.query(f'describe {stbname}') - tdSql.checkRows(16) - tdSql.execute(f'alter table {stbname} modify tag t12 binary(30)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(14,2,30) - tdSql.execute(f'alter table {stbname} modify tag `t12` binary(35)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(14,2,35) - tdSql.error(f'alter table {stbname} modify tag `t12` binary(34)') - tdSql.execute(f'alter table {stbname} modify tag t13 nchar(30)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(15,2,30) - tdSql.error(f'alter table {stbname} modify tag t13 nchar(29)') - tdSql.execute(f'alter table {stbname} rename tag t1 t21') - tdSql.query(f'describe {stbname}') - tdSql.checkData(3,0,'t21') - tdSql.execute(f'alter table {stbname} rename tag `t21` t1') - tdSql.query(f'describe {stbname}') - tdSql.checkData(3,0,'t1') - - for i in ['bigint','unsigned int','float','double','binary(10)','nchar(10)']: - for j in [1,2,3]: - tdSql.error(f'alter table {stbname} modify tag t{j} {i}') - for i in ['int','unsigned int','float','binary(10)','nchar(10)']: - tdSql.error(f'alter table {stbname} modify tag t8 {i}') - tdSql.error(f'alter table {stbname} modify tag t4 int') - tdSql.execute(f'drop database {dbname}') def run(self): - self.alter_tb_tag_check() - self.alter_ntb_column_check() - self.alter_stb_column_check() - self.alter_stb_tag_check() + self.alter_check_ntb() + self.alter_check_tb() + self.alter_check_stb() def stop(self): tdSql.close() diff --git a/tests/system-test/1-insert/create_retentions.py b/tests/system-test/1-insert/create_retentions.py index 5a0684e2ee..4b37eeb9a5 100644 --- a/tests/system-test/1-insert/create_retentions.py +++ b/tests/system-test/1-insert/create_retentions.py @@ -21,9 +21,9 @@ SINT_UN_COL = "c_sint_un" BINT_UN_COL = "c_bint_un" INT_UN_COL = "c_int_un" -BINARY_COL = "c8" -NCHAR_COL = "c9" -TS_COL = "c10" +BINARY_COL = "c_binary" +NCHAR_COL = "c_nchar" +TS_COL = "c_ts" NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] CHAR_COL = [ BINARY_COL, NCHAR_COL, ] @@ -51,12 +51,28 @@ class DataSet: binary_data : List[str] = None nchar_data : List[str] = None + def __post_init__(self): + self.ts_data = [] + self.int_data = [] + self.bint_data = [] + self.sint_data = [] + self.tint_data = [] + self.int_un_data = [] + self.bint_un_data = [] + self.sint_un_data = [] + self.tint_un_data = [] + self.float_data = [] + self.double_data = [] + self.bool_data = [] + self.binary_data = [] + self.nchar_data = [] + class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), True) + tdSql.init(conn.cursor(), False) @property def create_databases_sql_err(self): @@ -87,28 +103,28 @@ class TDTestCase: @property def create_stable_sql_err(self): return [ - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(ceil) watermark 1s maxdelay 1m", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(ceil) watermark 1s max_delay 1m", f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(count) watermark 1min", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay -1s", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay -1s", f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark -1m", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 1m ", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) maxdelay 1m ", + # f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 1m ", + # f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) max_delay 1m ", f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} binary(16)) tags (tag1 int) rollup(avg) watermark 1s", - f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) tags (tag1 int) rollup(avg) maxdelay 1m", - # f"create table ntb_1 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) rollup(avg) watermark 1s maxdelay 1s", + f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) tags (tag1 int) rollup(avg) max_delay 1m", + # f"create table ntb_1 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) rollup(avg) watermark 1s max_delay 1s", # f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) tags (tag1 int) " , # f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) " , # f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) " , # f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) " , - # watermark, maxdelay: [0, 900000], [ms, s, m, ?] - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 1u", + # watermark, max_delay: [0, 900000], [ms, s, m, ?] + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 1u", f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 1b", f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 900001ms", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 16m", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 901s", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 1h", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 0.2h", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 16m", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 901s", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 1h", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 0.2h", f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 0.002d", ] @@ -117,11 +133,11 @@ class TDTestCase: def create_stable_sql_current(self): return [ f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(avg)", - f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 5s maxdelay 1m", - f"create stable stb3 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(max) watermark 5s maxdelay 1m", - f"create stable stb4 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(sum) watermark 5s maxdelay 1m", - # f"create stable stb5 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(last) watermark 5s maxdelay 1m", - # f"create stable stb6 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s maxdelay 1m", + f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 5s max_delay 1m", + f"create stable stb3 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(max) watermark 5s max_delay 1m", + f"create stable stb4 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(sum) watermark 5s max_delay 1m", + # f"create stable stb5 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(last) watermark 5s max_delay 1m", + # f"create stable stb6 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s max_delay 1m", ] def test_create_stb(self): @@ -135,7 +151,7 @@ class TDTestCase: tdSql.checkRows(len(self.create_stable_sql_current)) # tdSql.execute("use db") # because db is a noraml database, not a rollup database, should not be able to create a rollup database - # tdSql.error(f"create stable nor_db_rollup_stb ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) file_factor 5.0") + # tdSql.error(f"create stable nor_db_rollup_stb ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 5s max_delay 1m") def test_create_databases(self): @@ -177,21 +193,6 @@ class TDTestCase: def __data_set(self, rows): data_set = DataSet() - # neg_data_set = DataSet() - data_set.ts_data = [] - data_set.int_data = [] - data_set.bint_data = [] - data_set.sint_data = [] - data_set.tint_data = [] - data_set.int_un_data = [] - data_set.bint_un_data = [] - data_set.sint_un_data = [] - data_set.tint_un_data = [] - data_set.float_data = [] - data_set.double_data = [] - data_set.bool_data = [] - data_set.binary_data = [] - data_set.nchar_data = [] for i in range(rows): data_set.ts_data.append(NOW + 1 * (rows - i)) @@ -226,6 +227,7 @@ class TDTestCase: return data_set def __insert_data(self): + tdLog.printNoPrefix("==========step: start inser data into tables now.....") data = self.__data_set(rows=self.rows) # now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) @@ -264,10 +266,10 @@ class TDTestCase: def run(self): self.rows = 10 - + tdSql.prepare() tdLog.printNoPrefix("==========step0:all check") - # self.all_test() + self.all_test() tdLog.printNoPrefix("==========step1:create table in normal database") tdSql.prepare() diff --git a/tests/system-test/1-insert/time_range_wise.py b/tests/system-test/1-insert/time_range_wise.py index 5387970d56..d4434987a6 100644 --- a/tests/system-test/1-insert/time_range_wise.py +++ b/tests/system-test/1-insert/time_range_wise.py @@ -17,25 +17,34 @@ TINT_COL = "c_tint" FLOAT_COL = "c_float" DOUBLE_COL = "c_double" BOOL_COL = "c_bool" -TINT_UN_COL = "c_tint_un" -SINT_UN_COL = "c_sint_un" -BINT_UN_COL = "c_bint_un" -INT_UN_COL = "c_int_un" +TINT_UN_COL = "c_utint" +SINT_UN_COL = "c_usint" +BINT_UN_COL = "c_ubint" +INT_UN_COL = "c_uint" BINARY_COL = "c_binary" NCHAR_COL = "c_nchar" TS_COL = "c_ts" - - NUM_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] CHAR_COL = [BINARY_COL, NCHAR_COL, ] BOOLEAN_COL = [BOOL_COL, ] TS_TYPE_COL = [TS_COL, ] +INT_TAG = "t_int" + +ALL_COL = [PRIMARY_COL, INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BINARY_COL, NCHAR_COL, BOOL_COL, TS_COL] +TAG_COL = [INT_TAG] + # insert data args： TIME_STEP = 10000 NOW = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) +# init db/table +DBNAME = "db" +STBNAME = "stb1" +CTBNAME = "ct1" +NTBNAME = "nt1" + @dataclass class DataSet: @@ -73,19 +82,25 @@ class DataSet: @dataclass class SMAschema: - creation : str = "CREATE" - index_name : str = "sma_index_1" - index_flag : str = "SMA INDEX" - operator : str = "ON" - tbname : str = None - watermark : str = None - maxdelay : str = None - func : Tuple[str] = None - interval : Tuple[str] = None - sliding : str = None - other : Any = None - drop : str = "DROP" - drop_flag : str = "INDEX" + creation : str = "CREATE" + index_name : str = "sma_index_1" + index_flag : str = "SMA INDEX" + operator : str = "ON" + tbname : str = None + watermark : str = "5s" + max_delay : str = "6m" + func : Tuple[str] = None + interval : Tuple[str] = ("6m", "10s") + sliding : str = "6m" + other : Any = None + drop : str = "DROP" + drop_flag : str = "INDEX" + querySmaOptimize : int = 1 + show : str = "SHOW" + show_msg : str = "INDEXES" + show_oper : str = "FROM" + dbname : str = None + rollup_db : bool = False def __post_init__(self): if isinstance(self.other, dict): @@ -111,8 +126,8 @@ class SMAschema: self.watermark = v del self.other[k] - if k.lower() == "maxdelay" and isinstance(v, str) and not self.maxdelay: - self.maxdelay = v + if k.lower() == "max_delay" and isinstance(v, str) and not self.max_delay: + self.max_delay = v del self.other[k] if k.lower() == "functions" and isinstance(v, tuple) and not self.func: @@ -131,12 +146,36 @@ class SMAschema: self.drop_flag = v del self.other[k] + if k.lower() == "show_msg" and isinstance(v, str) and not self.show_msg: + self.show_msg = v + del self.other[k] + + if k.lower() == "dbname" and isinstance(v, str) and not self.dbname: + self.dbname = v + del self.other[k] + + if k.lower() == "show_oper" and isinstance(v, str) and not self.show_oper: + self.show_oper = v + del self.other[k] + + if k.lower() == "rollup_db" and isinstance(v, bool) and not self.rollup_db: + self.rollup_db = v + del self.other[k] + + + +# from ...pytest.util.sql import * +# from ...pytest.util.constant import * class TDTestCase: + updatecfgDict = {"querySmaOptimize": 1} def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) + self.precision = "ms" + self.sma_count = 0 + self.sma_created_index = [] """ create sma index : @@ -155,13 +194,17 @@ class TDTestCase: if sma.func: sql += f" function({', '.join(sma.func)})" if sma.interval: - sql += f" interval({', '.join(sma.interval)})" + interval, offset = self.__get_interval_offset(sma.interval) + if offset: + sql += f" interval({interval}, {offset})" + else: + sql += f" interval({interval})" if sma.sliding: sql += f" sliding({sma.sliding})" if sma.watermark: sql += f" watermark {sma.watermark}" - if sma.maxdelay: - sql += f" maxdelay {sma.maxdelay}" + if sma.max_delay: + sql += f" max_delay {sma.max_delay}" if isinstance(sma.other, dict): for k,v in sma.other.items(): if isinstance(v,tuple) or isinstance(v, list): @@ -171,53 +214,305 @@ class TDTestCase: if isinstance(sma.other, tuple) or isinstance(sma.other, list): sql += " ".join(sma.other) if isinstance(sma.other, int) or isinstance(sma.other, float) or isinstance(sma.other, str): - sql += sma.other + sql += f" {sma.other}" return sql - def sma_create_check(self, sma:SMAschema): + def __get_sma_func_col(self, func): + cols = [] + if isinstance(func, str): + cols.append( func.split("(")[-1].split(")")[0] ) + elif isinstance(func, tuple) or isinstance(func, list): + for func_col in func: + cols.append(func_col.split("(")[-1].split(")")[0]) + else: + cols = [] + return cols + + def __check_sma_func(self, func:tuple): + if not isinstance(func, str) and not isinstance(func, tuple) and not isinstance(func, list): + return False + if isinstance(func, str) : + if "(" not in func or ")" not in func: + return False + if func.split("(")[0].upper() not in SMA_INDEX_FUNCTIONS: + return False + if func.split("(")[1].split(")")[0] not in ALL_COL and func.split("(")[1].split(")")[0] not in TAG_COL : + return False + if isinstance(func, tuple) or isinstance(func, list): + for arg in func: + if not isinstance(arg, str): + return False + if "(" not in arg or ")" not in arg: + return False + if arg.split("(")[0].upper() not in SMA_INDEX_FUNCTIONS: + return False + if arg.split("(")[1].split(")")[0] not in ALL_COL and arg.split("(")[1].split(")")[0] not in TAG_COL : + return False + return True + + def __check_sma_watermark(self, arg): + if not arg: + return False + if not isinstance(arg, str): + return False + if arg[-1] not in SMA_WATMARK_MAXDELAY_INIT: + return False + if len(arg) == 1: + return False + if not arg[:-1].isdecimal(): + return False + if tdSql.get_times(arg) > WATERMARK_MAX: + return False + if tdSql.get_times(arg) < WATERMARK_MIN: + return False + + return True + + def __check_sma_max_delay(self, arg): + if not self.__check_sma_watermark(arg): + return False + if tdSql.get_times(arg) < MAX_DELAY_MIN: + return False + + return True + + def __check_sma_sliding(self, arg): + if not isinstance(arg, str): + return False + if arg[-1] not in TAOS_TIME_INIT: + return False + if len(arg) == 1: + return False + if not arg[:-1].isdecimal(): + return False + + return True + + def __get_interval_offset(self, args): + if isinstance(args, str): + interval, offset = args, None + elif isinstance(args,tuple) or isinstance(args, list): + if len(args) == 1: + interval, offset = args[0], None + elif len(args) == 2: + interval, offset = args + else: + interval, offset = False, False + else: + interval, offset = False, False + + return interval, offset + + def __check_sma_interval(self, args): + if not isinstance(args, tuple) and not isinstance(args,str): + return False + interval, offset = self.__get_interval_offset(args) + if not interval: + return False + if not self.__check_sma_sliding(interval): + return False + if tdSql.get_times(interval) < INTERVAL_MIN: + return False + if offset: + if not self.__check_sma_sliding(offset): + return False + if tdSql.get_times(interval) <= tdSql.get_times(offset) : + return False + + return True + + def __sma_create_check(self, sma:SMAschema): + if self.updatecfgDict["querySmaOptimize"] == 0: + return False + # # TODO: if database is a rollup-db, can not create sma index + # tdSql.query("select database()") + # if sma.rollup_db : + # return False tdSql.query("show stables") + if not sma.tbname: + return False stb_in_list = False for row in tdSql.queryResult: if sma.tbname == row[0]: stb_in_list = True - break if not stb_in_list: - tdSql.error(self.__create_sma_index(sma)) - if not sma.creation: - tdSql.error(self.__create_sma_index(sma)) - if not sma.index_flag: - tdSql.error(self.__create_sma_index(sma)) - if not sma.index_name: - tdSql.error(self.__create_sma_index(sma)) - if not sma.operator: - tdSql.error(self.__create_sma_index(sma)) - if not sma.tbname: - tdSql.error(self.__create_sma_index(sma)) - if not sma.func: - tdSql.error(self.__create_sma_index(sma)) - if not sma.interval: - tdSql.error(self.__create_sma_index(sma)) - if not sma.sliding: - tdSql.error(self.__create_sma_index(sma)) + return False + if not sma.creation or not isinstance(sma.creation, str) or sma.creation.upper() != "CREATE": + return False + if not sma.index_flag or not isinstance(sma.index_flag, str) or sma.index_flag.upper() != "SMA INDEX" : + return False + if not sma.index_name or not isinstance(sma.index_name, str) or sma.index_name.upper() in TAOS_KEYWORDS: + return False + if not sma.operator or not isinstance(sma.operator, str) or sma.operator.upper() != "ON": + return False + + if not sma.func or not self.__check_sma_func(sma.func): + return False + tdSql.query(f"desc {sma.tbname}") + _col_list = [] + for col_row in tdSql.queryResult: + _col_list.append(col_row[0]) + _sma_func_cols = self.__get_sma_func_col(sma.func) + for _sma_func_col in _sma_func_cols: + if _sma_func_col not in _col_list: + return False + + if sma.sliding and not self.__check_sma_sliding(sma.sliding): + return False + interval, _ = self.__get_interval_offset(sma.interval) + if not sma.interval or not self.__check_sma_interval(sma.interval) : + return False + if sma.sliding and tdSql.get_times(interval) < tdSql.get_times(sma.sliding): + return False + if sma.watermark and not self.__check_sma_watermark(sma.watermark): + return False + if sma.max_delay and not self.__check_sma_max_delay(sma.max_delay): + return False if sma.other: + return False + + return True + + def sma_create_check(self, sma:SMAschema): + if self.__sma_create_check(sma): + tdSql.query(self.__create_sma_index(sma)) + self.sma_count += 1 + self.sma_created_index.append(sma.index_name) + tdSql.query("show streams") + tdSql.checkRows(self.sma_count) + + else: tdSql.error(self.__create_sma_index(sma)) + def __drop_sma_index(self, sma:SMAschema): + sql = f"{sma.drop} {sma.drop_flag} {sma.index_name}" + return sql + + def __sma_drop_check(self, sma:SMAschema): + if not sma.drop: + return False + if not sma.drop_flag: + return False + if not sma.index_name: + return False + + return True + + def sma_drop_check(self, sma:SMAschema): + if self.__sma_drop_check(sma): + tdSql.query(self.__drop_sma_index(sma)) + print(self.__drop_sma_index(sma)) + self.sma_count -= 1 + self.sma_created_index = list(filter(lambda x: x != sma.index_name, self.sma_created_index)) + tdSql.query("show streams") + tdSql.checkRows(self.sma_count) + + else: + tdSql.error(self.__drop_sma_index(sma)) + + def __show_sma_index(self, sma:SMAschema): + sql = f"{sma.show} {sma.show_msg} {sma.show_oper} {sma.tbname}" + return sql + + def __sma_show_check(self, sma:SMAschema): + if not sma.show: + return False + if not sma.show_msg: + return False + if not sma.show_oper: + return False + if not sma.tbname: + return False + + return True + + def sma_show_check(self, sma:SMAschema): + if self.__sma_show_check(sma): + tdSql.query(self.__show_sma_index(sma)) + tdSql.checkRows(self.sma_count) + else: + tdSql.error(self.__show_sma_index(sma)) + + @property + def __create_sma_sql(self): + err_sqls = [] + cur_sqls = [] + # err_set + # # case 1: required fields check + err_sqls.append( SMAschema(creation="", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(index_name="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(index_flag="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(operator="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(tbname="", func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(func=("",),tbname=STBNAME ) ) + err_sqls.append( SMAschema(interval=(""),tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + + # # case 2: err fields + err_sqls.append( SMAschema(creation="show",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(creation="alter",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(creation="select",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + + err_sqls.append( SMAschema(index_flag="SMA INDEXES", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(index_flag="SMA INDEX ,", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(index_name="tbname", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + + + # current_set + + cur_sqls.append( SMAschema(max_delay="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + cur_sqls.append( SMAschema(watermark="",index_name="sma_index_2",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + cur_sqls.append( SMAschema(sliding="",index_name='sma_index_3',tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + + + return err_sqls, cur_sqls + + def test_create_sma(self): + err_sqls , cur_sqls = self.__create_sma_sql + for err_sql in err_sqls: + self.sma_create_check(err_sql) + for cur_sql in cur_sqls: + self.sma_create_check(cur_sql) + + @property + def __drop_sma_sql(self): + err_sqls = [] + cur_sqls = [] + # err_set + ## case 1: required fields check + err_sqls.append( SMAschema(drop="") ) + err_sqls.append( SMAschema(drop_flag="") ) + err_sqls.append( SMAschema(index_name="") ) + + for index in self.sma_created_index: + cur_sqls.append(SMAschema(index_name=index)) + + return err_sqls, cur_sqls + + def test_drop_sma(self): + err_sqls , cur_sqls = self.__drop_sma_sql + for err_sql in err_sqls: + self.sma_drop_check(err_sql) + # for cur_sql in cur_sqls: + # self.sma_drop_check(cur_sql) def all_test(self): + self.test_create_sma() + self.test_drop_sma() + pass def __create_tb(self): tdLog.printNoPrefix("==========step: create table") - create_stb_sql = f'''create table stb1( + create_stb_sql = f'''create table {STBNAME}( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp, {TINT_UN_COL} tinyint unsigned, {SINT_UN_COL} smallint unsigned, {INT_UN_COL} int unsigned, {BINT_UN_COL} bigint unsigned - ) tags (tag1 int) + ) tags ({INT_TAG} int) ''' - create_ntb_sql = f'''create table t1( + create_ntb_sql = f'''create table {NTBNAME}( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp, @@ -253,6 +548,7 @@ class TDTestCase: return data_set def __insert_data(self): + tdLog.printNoPrefix("==========step: start inser data into tables now.....") data = self.__data_set(rows=self.rows) # now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) @@ -278,7 +574,7 @@ class TDTestCase: tdSql.execute( f"insert into ct4 values ( {NOW - i * int(TIME_STEP * 0.8) }, {row_data} )") tdSql.execute( - f"insert into t1 values ( {NOW - i * int(TIME_STEP * 1.2)}, {row_data} )") + f"insert into {NTBNAME} values ( {NOW - i * int(TIME_STEP * 1.2)}, {row_data} )") tdSql.execute( f"insert into ct2 values ( {NOW + int(TIME_STEP * 0.6)}, {null_data} )") @@ -295,28 +591,31 @@ class TDTestCase: f"insert into ct4 values ( {NOW - self.rows * int(TIME_STEP * 0.39)}, {null_data} )") tdSql.execute( - f"insert into t1 values ( {NOW + int(TIME_STEP * 1.2)}, {null_data} )") + f"insert into {NTBNAME} values ( {NOW + int(TIME_STEP * 1.2)}, {null_data} )") tdSql.execute( - f"insert into t1 values ( {NOW - (self.rows + 1) * int(TIME_STEP * 1.2)}, {null_data} )") + f"insert into {NTBNAME} values ( {NOW - (self.rows + 1) * int(TIME_STEP * 1.2)}, {null_data} )") tdSql.execute( - f"insert into t1 values ( {NOW - self.rows * int(TIME_STEP * 0.59)}, {null_data} )") + f"insert into {NTBNAME} values ( {NOW - self.rows * int(TIME_STEP * 0.59)}, {null_data} )") def run(self): - sma1 = SMAschema(func=("min(c1)","max(c2)")) - sql1 = self.__create_sma_index(sma1) - print("================") - print(sql1) - # a = DataSet() - # return self.rows = 10 tdLog.printNoPrefix("==========step0:all check") - # self.all_test() tdLog.printNoPrefix("==========step1:create table in normal database") tdSql.prepare() self.__create_tb() - self.__insert_data() + # self.__insert_data() + self.all_test() + + # drop databases, create same name db、stb and sma index + # tdSql.prepare() + # self.__create_tb() + # self.__insert_data() + # self.all_test() + + + return tdLog.printNoPrefix("==========step2:create table in rollup database") diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index 244cccb041..961a6446b5 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -23,7 +23,6 @@ class TDTestCase: self.time_step = 1000 def insert_datas_and_check_abs(self ,tbnums , rownums , time_step ): - tdLog.info(" prepare datas for auto check abs function ") tdSql.execute(" create database test ") @@ -36,7 +35,7 @@ class TDTestCase: ts = self.ts for row in range(rownums): - ts += time_step*row + ts = self.ts + time_step*row c1 = random.randint(0,10000) c2 = random.randint(0,100000) c3 = random.randint(0,125) @@ -538,25 +537,41 @@ class TDTestCase: # tdSql.query(" select sum(c1) from stb1 where t1+10 >1; ") #　taosd crash tdSql.query("select c1 ,t1 from stb1 where t1 =0 ") tdSql.checkRows(13) - # tdSql.query("select t1 from stb1 where t1 >0 ") - # tdSql.checkRows(3) + tdSql.query("select t1 from stb1 where t1 >0 ") + tdSql.checkRows(3) + tdSql.query("select t1 from stb1 where t1 =3 ") + tdSql.checkRows(1) # tdSql.query("select sum(t1) from (select c1 ,t1 from stb1)") # tdSql.checkData(0,0,61) # tdSql.query("select distinct(c1) ,t1 from stb1") # tdSql.checkRows(20) - # tdSql.query("select max(t2) , t1 ,c1, t2 from stb1") - # tdSql.checkData(0,3,33333) + tdSql.query("select max(t2) , t1 ,c1, t2 from stb1") + tdSql.checkData(0,3,33333) # tag filter with abs function - # tdSql.query("select t1 from stb1 where abs(t1)=1") - # tdSql.checkRows(1) + tdSql.query("select t1 from stb1 where abs(t1)=1") + tdSql.checkRows(1) tdSql.query("select t1 from stb1 where abs(c1+t1)=1") tdSql.checkRows(1) - # tdSql.query("select t1 from stb1 where abs(t1+c1)=1") - # tdSql.checkRows(1) + tdSql.checkData(0,0,0) + tdSql.query( "select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1") + def support_super_table_test(self): + tdSql.execute(" use testdb ") + self.check_result_auto( " select c1 from stb1 order by ts " , "select abs(c1) from stb1 order by ts" ) + self.check_result_auto( " select c1 from stb1 order by tbname " , "select abs(c1) from stb1 order by tbname" ) + self.check_result_auto( " select c1 from stb1 where c1 > 0 order by tbname " , "select abs(c1) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select c1 from stb1 where c1 > 0 order by tbname " , "select abs(c1) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto( " select t1,c1 from stb1 order by ts " , "select t1, abs(c1) from stb1 order by ts" ) + self.check_result_auto( " select t2,c1 from stb1 order by tbname " , "select t2 ,abs(c1) from stb1 order by tbname" ) + self.check_result_auto( " select t3,c1 from stb1 where c1 > 0 order by tbname " , "select t3 ,abs(c1) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select t4,c1 from stb1 where c1 > 0 order by tbname " , "select t4 , abs(c1) from stb1 where c1 > 0 order by tbname" ) + pass + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -593,6 +608,10 @@ class TDTestCase: self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step) + tdLog.printNoPrefix("==========step8: check abs result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/arccos.py b/tests/system-test/2-query/arccos.py index e15f7675f2..edb9e25c11 100644 --- a/tests/system-test/2-query/arccos.py +++ b/tests/system-test/2-query/arccos.py @@ -479,6 +479,20 @@ class TDTestCase: tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) self.check_result_auto_acos("select num1,num2 from tb3;" , "select acos(num1),acos(num2) from tb3") + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_acos( " select c5 from stb1 order by ts " , "select acos(c5) from stb1 order by ts" ) + self.check_result_auto_acos( " select c5 from stb1 order by tbname " , "select acos(c5) from stb1 order by tbname" ) + self.check_result_auto_acos( " select c5 from stb1 where c1 > 0 order by tbname " , "select acos(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_acos( " select c5 from stb1 where c1 > 0 order by tbname " , "select acos(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_acos( " select t1,c5 from stb1 order by ts " , "select acos(t1), acos(c5) from stb1 order by ts" ) + self.check_result_auto_acos( " select t1,c5 from stb1 order by tbname " , "select acos(t1) ,acos(c5) from stb1 order by tbname" ) + self.check_result_auto_acos( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select acos(t1) ,acos(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_acos( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select acos(t1) , acos(c5) from stb1 where c1 > 0 order by tbname" ) + pass + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -512,6 +526,14 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step7: acos filter query ============") + + self.abs_func_filter() + + tdLog.printNoPrefix("==========step8: check acos result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/arcsin.py b/tests/system-test/2-query/arcsin.py index ee134a56a8..faed5ef3c4 100644 --- a/tests/system-test/2-query/arcsin.py +++ b/tests/system-test/2-query/arcsin.py @@ -479,6 +479,20 @@ class TDTestCase: tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) self.check_result_auto_asin("select num1,num2 from tb3;" , "select asin(num1),asin(num2) from tb3") + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_asin( " select c5 from stb1 order by ts " , "select asin(c5) from stb1 order by ts" ) + self.check_result_auto_asin( " select c5 from stb1 order by tbname " , "select asin(c5) from stb1 order by tbname" ) + self.check_result_auto_asin( " select c5 from stb1 where c1 > 0 order by tbname " , "select asin(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_asin( " select c5 from stb1 where c1 > 0 order by tbname " , "select asin(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_asin( " select t1,c5 from stb1 order by ts " , "select asin(t1), asin(c5) from stb1 order by ts" ) + self.check_result_auto_asin( " select t1,c5 from stb1 order by tbname " , "select asin(t1) ,asin(c5) from stb1 order by tbname" ) + self.check_result_auto_asin( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select asin(t1) ,asin(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_asin( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select asin(t1) , asin(c5) from stb1 where c1 > 0 order by tbname" ) + pass + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -512,6 +526,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step8: check asin result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/arctan.py b/tests/system-test/2-query/arctan.py index 1890c4605c..80d28b5ee5 100644 --- a/tests/system-test/2-query/arctan.py +++ b/tests/system-test/2-query/arctan.py @@ -476,6 +476,20 @@ class TDTestCase: tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) self.check_result_auto_atan("select num1,num2 from tb3;" , "select atan(num1),atan(num2) from tb3") + + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_atan( " select c5 from stb1 order by ts " , "select atan(c5) from stb1 order by ts" ) + self.check_result_auto_atan( " select c5 from stb1 order by tbname " , "select atan(c5) from stb1 order by tbname" ) + self.check_result_auto_atan( " select c5 from stb1 where c1 > 0 order by tbname " , "select atan(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_atan( " select c5 from stb1 where c1 > 0 order by tbname " , "select atan(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_atan( " select t1,c5 from stb1 order by ts " , "select atan(t1), atan(c5) from stb1 order by ts" ) + self.check_result_auto_atan( " select t1,c5 from stb1 order by tbname " , "select atan(t1) ,atan(c5) from stb1 order by tbname" ) + self.check_result_auto_atan( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select atan(t1) ,atan(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_atan( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select atan(t1) , atan(c5) from stb1 where c1 > 0 order by tbname" ) + pass def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -509,6 +523,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step8: check arctan result of stable query ============") + + self.support_super_table_test() + def stop(self): diff --git a/tests/system-test/2-query/ceil.py b/tests/system-test/2-query/ceil.py index 4196a5a8ce..b269b54a17 100644 --- a/tests/system-test/2-query/ceil.py +++ b/tests/system-test/2-query/ceil.py @@ -427,6 +427,18 @@ class TDTestCase: self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select ceil(c1+1) ,ceil(c2) , ceil(c3*1) , ceil(c4/2), ceil(c5)/2, ceil(c6) from sub1_bound ") + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto( " select c5 from stb1 order by ts " , "select ceil(c5) from stb1 order by ts" ) + self.check_result_auto( " select c5 from stb1 order by tbname " , "select ceil(c5) from stb1 order by tbname" ) + self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select ceil(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select ceil(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto( " select t1,c5 from stb1 order by ts " , "select ceil(t1), ceil(c5) from stb1 order by ts" ) + self.check_result_auto( " select t1,c5 from stb1 order by tbname " , "select ceil(t1) ,ceil(c5) from stb1 order by tbname" ) + self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select ceil(t1) ,ceil(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select ceil(t1) , ceil(c5) from stb1 where c1 > 0 order by tbname" ) + pass def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -455,6 +467,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step7: check ceil result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/cos.py b/tests/system-test/2-query/cos.py index 0cb9f66937..1165d8d681 100644 --- a/tests/system-test/2-query/cos.py +++ b/tests/system-test/2-query/cos.py @@ -476,6 +476,19 @@ class TDTestCase: tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) self.check_result_auto_cos("select num1,num2 from tb3;" , "select cos(num1),cos(num2) from tb3") + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_cos( " select c5 from stb1 order by ts " , "select cos(c5) from stb1 order by ts" ) + self.check_result_auto_cos( " select c5 from stb1 order by tbname " , "select cos(c5) from stb1 order by tbname" ) + self.check_result_auto_cos( " select c5 from stb1 where c1 > 0 order by tbname " , "select cos(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_cos( " select c5 from stb1 where c1 > 0 order by tbname " , "select cos(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_cos( " select t1,c5 from stb1 order by ts " , "select cos(t1), cos(c5) from stb1 order by ts" ) + self.check_result_auto_cos( " select t1,c5 from stb1 order by tbname " , "select cos(t1) ,cos(c5) from stb1 order by tbname" ) + self.check_result_auto_cos( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select cos(t1) ,cos(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_cos( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select cos(t1) , cos(c5) from stb1 where c1 > 0 order by tbname" ) + pass def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -509,6 +522,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step8: check cos result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/floor.py b/tests/system-test/2-query/floor.py index c955c24e05..7362191958 100644 --- a/tests/system-test/2-query/floor.py +++ b/tests/system-test/2-query/floor.py @@ -427,6 +427,19 @@ class TDTestCase: self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select floor(c1+1) ,floor(c2) , floor(c3*1) , floor(c4/2), floor(c5)/2, floor(c6) from sub1_bound ") + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto( " select c5 from stb1 order by ts " , "select floor(c5) from stb1 order by ts" ) + self.check_result_auto( " select c5 from stb1 order by tbname " , "select floor(c5) from stb1 order by tbname" ) + self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select floor(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select floor(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto( " select t1,c5 from stb1 order by ts " , "select floor(t1), floor(c5) from stb1 order by ts" ) + self.check_result_auto( " select t1,c5 from stb1 order by tbname " , "select floor(t1) ,floor(c5) from stb1 order by tbname" ) + self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select floor(t1) ,floor(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select floor(t1) , floor(c5) from stb1 where c1 > 0 order by tbname" ) + pass + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -454,6 +467,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step7: check floor result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 5ff11c84dd..df6390f59c 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), True) + tdSql.init(conn.cursor(), False) def __query_condition(self,tbname): query_condition = [] diff --git a/tests/system-test/2-query/join2.py b/tests/system-test/2-query/join2.py index 40da41eee7..5533cb840e 100644 --- a/tests/system-test/2-query/join2.py +++ b/tests/system-test/2-query/join2.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), True) + tdSql.init(conn.cursor(), False) def __query_condition(self,tbname): query_condition = [] diff --git a/tests/system-test/2-query/log.py b/tests/system-test/2-query/log.py index 907ba329ee..f9d6e91199 100644 --- a/tests/system-test/2-query/log.py +++ b/tests/system-test/2-query/log.py @@ -670,7 +670,19 @@ class TDTestCase: tdSql.checkData(0,2,math.log(32767.000000000,2)) tdSql.checkData(0,3,math.log(63.500000000,2)) tdSql.checkData(0,4,63.999401166) - + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_log2( " select c5 from stb1 order by ts " , "select log(c5,2) from stb1 order by ts" ) + self.check_result_auto_log2( " select c5 from stb1 order by tbname " , "select log(c5,2) from stb1 order by tbname" ) + self.check_result_auto_log2( " select c5 from stb1 where c1 > 0 order by tbname " , "select log(c5,2) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_log2( " select c5 from stb1 where c1 > 0 order by tbname " , "select log(c5,2) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_log2( " select t1,c5 from stb1 order by ts " , "select log(t1,2), log(c5,2) from stb1 order by ts" ) + self.check_result_auto_log2( " select t1,c5 from stb1 order by tbname " , "select log(t1,2) ,log(c5,2) from stb1 order by tbname" ) + self.check_result_auto_log2( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select log(t1,2) ,log(c5,2) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_log2( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select log(t1,2) , log(c5,2) from stb1 where c1 > 0 order by tbname" ) + pass def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -707,7 +719,9 @@ class TDTestCase: self.abs_func_filter() - + tdLog.printNoPrefix("==========step9: check log result of stable query ============") + + self.support_super_table_test() def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/pow.py b/tests/system-test/2-query/pow.py index 8b0137b411..c67162961b 100644 --- a/tests/system-test/2-query/pow.py +++ b/tests/system-test/2-query/pow.py @@ -606,7 +606,23 @@ class TDTestCase: tdSql.checkData(0,3,math.pow(63.500000000,2)) tdSql.checkData(0,5,None) + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_pow2( " select c5 from stb1 order by ts " , "select pow(c5,2) from stb1 order by ts" ) + self.check_result_auto_pow2( " select c5 from stb1 order by tbname " , "select pow(c5,2) from stb1 order by tbname" ) + self.check_result_auto_pow2( " select c5 from stb1 where c1 > 0 order by tbname " , "select pow(c5,2) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_pow2( " select c5 from stb1 where c1 > 0 order by tbname " , "select pow(c5,2) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_pow2( " select t1,c5 from stb1 order by ts " , "select pow(t1,2), pow(c5,2) from stb1 order by ts" ) + self.check_result_auto_pow2( " select t1,c5 from stb1 order by tbname " , "select pow(t1,2) ,pow(c5,2) from stb1 order by tbname" ) + self.check_result_auto_pow2( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select pow(t1,2) ,pow(c5,2) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_pow2( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select pow(t1,2) , pow(c5,2) from stb1 where c1 > 0 order by tbname" ) + pass + + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -642,7 +658,9 @@ class TDTestCase: self.abs_func_filter() - + tdLog.printNoPrefix("==========step9: check pow result of stable query ============") + + self.support_super_table_test() def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/round.py b/tests/system-test/2-query/round.py index 223e56bce6..cc272abf42 100644 --- a/tests/system-test/2-query/round.py +++ b/tests/system-test/2-query/round.py @@ -432,6 +432,20 @@ class TDTestCase: self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select round(c1+1) ,round(c2) , round(c3*1) , round(c4/2), round(c5)/2, round(c6) from sub1_bound ") + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto( " select c5 from stb1 order by ts " , "select round(c5) from stb1 order by ts" ) + self.check_result_auto( " select c5 from stb1 order by tbname " , "select round(c5) from stb1 order by tbname" ) + self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select round(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select round(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto( " select t1,c5 from stb1 order by ts " , "select round(t1), round(c5) from stb1 order by ts" ) + self.check_result_auto( " select t1,c5 from stb1 order by tbname " , "select round(t1) ,round(c5) from stb1 order by tbname" ) + self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select round(t1) ,round(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select round(t1) , round(c5) from stb1 where c1 > 0 order by tbname" ) + pass + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -459,6 +473,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step7: check round result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/sin.py b/tests/system-test/2-query/sin.py index 62d42a991e..2c4d90d3e7 100644 --- a/tests/system-test/2-query/sin.py +++ b/tests/system-test/2-query/sin.py @@ -476,6 +476,19 @@ class TDTestCase: tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) self.check_result_auto_sin("select num1,num2 from tb3;" , "select sin(num1),sin(num2) from tb3") + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_sin( " select c5 from stb1 order by ts " , "select sin(c5) from stb1 order by ts" ) + self.check_result_auto_sin( " select c5 from stb1 order by tbname " , "select sin(c5) from stb1 order by tbname" ) + self.check_result_auto_sin( " select c5 from stb1 where c1 > 0 order by tbname " , "select sin(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_sin( " select c5 from stb1 where c1 > 0 order by tbname " , "select sin(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_sin( " select t1,c5 from stb1 order by ts " , "select sin(t1), sin(c5) from stb1 order by ts" ) + self.check_result_auto_sin( " select t1,c5 from stb1 order by tbname " , "select sin(t1) ,sin(c5) from stb1 order by tbname" ) + self.check_result_auto_sin( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select sin(t1) ,sin(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_sin( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select sin(t1) , sin(c5) from stb1 where c1 > 0 order by tbname" ) + pass def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -509,6 +522,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step8: check sin result of stable query ============") + + self.support_super_table_test() + def stop(self): diff --git a/tests/system-test/2-query/sqrt.py b/tests/system-test/2-query/sqrt.py index 28e869e044..772056fd93 100644 --- a/tests/system-test/2-query/sqrt.py +++ b/tests/system-test/2-query/sqrt.py @@ -505,7 +505,19 @@ class TDTestCase: tdSql.checkData(0,2,math.sqrt(32767.000000000)) tdSql.checkData(0,3,math.sqrt(63.500000000)) - + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_sqrt( " select c5 from stb1 order by ts " , "select sqrt(c5) from stb1 order by ts" ) + self.check_result_auto_sqrt( " select c5 from stb1 order by tbname " , "select sqrt(c5) from stb1 order by tbname" ) + self.check_result_auto_sqrt( " select c5 from stb1 where c1 > 0 order by tbname " , "select sqrt(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_sqrt( " select c5 from stb1 where c1 > 0 order by tbname " , "select sqrt(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_sqrt( " select t1,c5 from stb1 order by ts " , "select sqrt(t1), sqrt(c5) from stb1 order by ts" ) + self.check_result_auto_sqrt( " select t1,c5 from stb1 order by tbname " , "select sqrt(t1) ,sqrt(c5) from stb1 order by tbname" ) + self.check_result_auto_sqrt( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select sqrt(t1) ,sqrt(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_sqrt( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select sqrt(t1) , sqrt(c5) from stb1 where c1 > 0 order by tbname" ) + pass + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -541,6 +553,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step9: check sqrt result of stable query ============") + + self.support_super_table_test() + def stop(self): diff --git a/tests/system-test/2-query/substr.py b/tests/system-test/2-query/substr.py index 79b5ac515b..f833a42b57 100644 --- a/tests/system-test/2-query/substr.py +++ b/tests/system-test/2-query/substr.py @@ -31,7 +31,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(),False) def __substr_condition(self): # sourcery skip: extract-method substr_condition = [] diff --git a/tests/system-test/2-query/tail.py b/tests/system-test/2-query/tail.py index 0e2110f912..a4c85cc5ed 100644 --- a/tests/system-test/2-query/tail.py +++ b/tests/system-test/2-query/tail.py @@ -188,8 +188,8 @@ class TDTestCase: def check_tail_table(self , tbname , col_name , tail_rows , offset): tail_sql = f"select tail({col_name} , {tail_rows} , {offset}) from {tbname}" - equal_sql = f"select {col_name} from (select ts , {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}) order by ts" - #equal_sql = f"select {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}" + #equal_sql = f"select {col_name} from (select ts , {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}) order by ts" + equal_sql = f"select {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}" tdSql.query(tail_sql) tail_result = tdSql.queryResult @@ -404,7 +404,7 @@ class TDTestCase: f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) - tdSql.query("select tail(c2,2) from sub1_bound") + tdSql.query("select tail(c2,2) from sub1_bound order by 1 desc") tdSql.checkRows(2) tdSql.checkData(0,0,9223372036854775803) diff --git a/tests/system-test/2-query/tan.py b/tests/system-test/2-query/tan.py index c8914d18c0..9610ffef24 100644 --- a/tests/system-test/2-query/tan.py +++ b/tests/system-test/2-query/tan.py @@ -476,6 +476,20 @@ class TDTestCase: tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) self.check_result_auto_tan("select num1,num2 from tb3;" , "select tan(num1),tan(num2) from tb3") + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_tan( " select c5 from stb1 order by ts " , "select tan(c5) from stb1 order by ts" ) + self.check_result_auto_tan( " select c5 from stb1 order by tbname " , "select tan(c5) from stb1 order by tbname" ) + self.check_result_auto_tan( " select c5 from stb1 where c1 > 0 order by tbname " , "select tan(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_tan( " select c5 from stb1 where c1 > 0 order by tbname " , "select tan(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_tan( " select t1,c5 from stb1 order by ts " , "select tan(t1), tan(c5) from stb1 order by ts" ) + self.check_result_auto_tan( " select t1,c5 from stb1 order by tbname " , "select tan(t1) ,tan(c5) from stb1 order by tbname" ) + self.check_result_auto_tan( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select tan(t1) ,tan(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_tan( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select tan(t1) , tan(c5) from stb1 where c1 > 0 order by tbname" ) + pass + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -509,6 +523,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step8: check tan result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() diff --git a/tests/system-test/7-tmq/stbTagFilter.py b/tests/system-test/7-tmq/stbTagFilter.py new file mode 100644 index 0000000000..2a2cb40c09 --- /dev/null +++ b/tests/system-test/7-tmq/stbTagFilter.py @@ -0,0 +1,113 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db2', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + + topicNameList = ['topic1'] + expectRowsList = [] + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdLog.info("create stb") + tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'], ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.asyncInsertData(paraDict) + + tdLog.info("create topics from stb with filter") + # queryString = "select ts, sin(c1), pow(c2,3) from %s.%s where t2 == 'beijing' or t2 == 'changsha'" %(paraDict['dbName'], paraDict['stbName']) + queryString = "select * from %s.%s where t2 == 'beijing' or t2 == 'changsha'" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # start tmq consume processor + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:2000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + # tmqCom.getStartCommitNotifyFromTmqsim() + tmqCom.getStartConsumeNotifyFromTmqsim() + tdLog.info("create some new ctb") + paraDict['ctbStartIdx'] = paraDict['ctbStartIdx'] + paraDict['ctbNum'] + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'], ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data into new ctb") + pThread = tmqCom.asyncInsertData(paraDict) + + pThread.join() + tdLog.info("wait insert end") + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectRowsList[0] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.exit("0 tmq consume rows error!") + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + tdSql.prepare() + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqCheckData1.py b/tests/system-test/7-tmq/tmqCheckData1.py new file mode 100644 index 0000000000..6cf849d1b9 --- /dev/null +++ b/tests/system-test/7-tmq/tmqCheckData1.py @@ -0,0 +1,180 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def checkFileContent(self, consumerId, queryString): + buildPath = tdCom.getBuildPath() + cfgPath = tdCom.getClientCfgPath() + dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) + cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) + tdLog.info(cmdStr) + os.system(cmdStr) + + consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) + tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) + + consumeFile = open(consumeRowsFile, mode='r') + queryFile = open(dstFile, mode='r') + + # skip first line for it is schema + queryFile.readline() + + while True: + dst = queryFile.readline() + src = consumeFile.readline() + + if dst: + if dst != src: + tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) + else: + break + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db1', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 1, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + + topicNameList = ['topic1', 'topic2', 'topic3'] + expectRowsList = [] + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdLog.info("create stb") + tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdLog.info("create ctb") + tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) + tdLog.info("insert data") + tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) + + tdLog.info("create topics from stb with filter") + queryString = "select ts,c1,c2 from %s.%s" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as stable %s.%s" %(topicNameList[0], paraDict["dbName"],paraDict["stbName"]) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectRowsList[0] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.exit("0 tmq consume rows error!") + + self.checkFileContent(consumerId, queryString) + + # reinit consume info, and start tmq_sim, then check consume result + tmqCom.initConsumerTable() + queryString = "select ts, c1, c2 from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as database %s" %(topicNameList[1], paraDict['dbName']) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + consumerId = 1 + topicList = topicNameList[1] + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + if expectRowsList[1] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) + tdLog.exit("1 tmq consume rows error!") + + self.checkFileContent(consumerId, queryString) + + # reinit consume info, and start tmq_sim, then check consume result + tmqCom.initConsumerTable() + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[2], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + consumerId = 2 + topicList = topicNameList[2] + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + if expectRowsList[2] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0])) + tdLog.exit("2 tmq consume rows error!") + + self.checkFileContent(consumerId, queryString) + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + tdSql.prepare() + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 788ae3474c..9254f57c40 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -159,7 +159,7 @@ class TMQCom: tdLog.debug("complete to create %s.%s" %(dbName, stbName)) return - def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1): + def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1,ctbStartIdx=0): tsql.execute("use %s" %dbName) pre_create = "create table" sql = pre_create @@ -168,8 +168,10 @@ class TMQCom: tagValue = 'beijing' if (i % 2 == 0): tagValue = 'shanghai' + elif (i % 3 == 0): + tagValue = 'changsha' - sql += " %s%d using %s tags(%d, '%s')"%(ctbPrefix,i,stbName,i+1, tagValue) + sql += " %s%d using %s tags(%d, '%s')"%(ctbPrefix,i+ctbStartIdx,stbName,i+ctbStartIdx+1, tagValue) if (i > 0) and (i%100 == 0): tsql.execute(sql) sql = pre_create @@ -235,7 +237,7 @@ class TMQCom: tdLog.debug("insert data ............ [OK]") return - def insert_data_2(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs): + def insert_data_2(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs,ctbStartIdx=0): tdLog.debug("start to insert data ............") tsql.execute("use %s" %dbName) pre_insert = "insert into " @@ -245,7 +247,7 @@ class TMQCom: startTs = int(round(t * 1000)) #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) for i in range(ctbNum): - sql += " %s%d values "%(ctbPrefix,i) + sql += " %s%d values "%(ctbPrefix,i+ctbStartIdx) for j in range(rowsPerTbl): if (j % 2 == 0): sql += "(%d, %d, %d, 'tmqrow_%d', now) "%(startTs + j, j, j, j) @@ -254,7 +256,7 @@ class TMQCom: if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): tsql.execute(sql) if j < rowsPerTbl - 1: - sql = "insert into %s%d values " %(ctbPrefix,i) + sql = "insert into %s%d values " %(ctbPrefix,i+ctbStartIdx) else: sql = "insert into " #end sql @@ -354,7 +356,10 @@ class TMQCom: def threadFunctionForInsert(self, **paraDict): # create new connector for new tdSql instance in my thread newTdSql = tdCom.newTdSql() - self.insert_data_2(newTdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) + if 'ctbStartIdx' in paraDict.keys(): + self.insert_data_2(newTdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"],paraDict["ctbStartIdx"]) + else: + self.insert_data_2(newTdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) return def asyncInsertData(self, paraDict): diff --git a/tests/system-test/fulltest.bat b/tests/system-test/fulltest.bat deleted file mode 100644 index 59ddd3fb7d..0000000000 --- a/tests/system-test/fulltest.bat +++ /dev/null @@ -1,102 +0,0 @@ - -python3 .\test.py -f 0-others\taosShell.py -python3 .\test.py -f 0-others\taosShellError.py -python3 .\test.py -f 0-others\taosShellNetChk.py -python3 .\test.py -f 0-others\telemetry.py -python3 .\test.py -f 0-others\taosdMonitor.py -python3 .\test.py -f 0-others\udfTest.py -python3 .\test.py -f 0-others\udf_create.py -python3 .\test.py -f 0-others\udf_restart_taosd.py -python3 .\test.py -f 0-others\cachelast.py - -python3 .\test.py -f 0-others\user_control.py -python3 .\test.py -f 0-others\fsync.py - -python3 .\test.py -f 1-insert\influxdb_line_taosc_insert.py -python3 .\test.py -f 1-insert\opentsdb_telnet_line_taosc_insert.py -python3 .\test.py -f 1-insert\opentsdb_json_taosc_insert.py -@REM #python3 .\test.py -f 1-insert\test_stmt_muti_insert_query.py -python3 .\test.py -f 1-insert\alter_stable.py -python3 .\test.py -f 1-insert\alter_table.py -python3 .\test.py -f 2-query\between.py -python3 .\test.py -f 2-query\distinct.py -python3 .\test.py -f 2-query\varchar.py -python3 .\test.py -f 2-query\ltrim.py -python3 .\test.py -f 2-query\rtrim.py -python3 .\test.py -f 2-query\length.py -python3 .\test.py -f 2-query\char_length.py -python3 .\test.py -f 2-query\upper.py -python3 .\test.py -f 2-query\lower.py -python3 .\test.py -f 2-query\join.py -python3 .\test.py -f 2-query\join2.py -python3 .\test.py -f 2-query\cast.py -python3 .\test.py -f 2-query\union.py -python3 .\test.py -f 2-query\union1.py -python3 .\test.py -f 2-query\concat.py -python3 .\test.py -f 2-query\concat2.py -python3 .\test.py -f 2-query\concat_ws.py -python3 .\test.py -f 2-query\concat_ws2.py -python3 .\test.py -f 2-query\check_tsdb.py -python3 .\test.py -f 2-query\spread.py -python3 .\test.py -f 2-query\hyperloglog.py - -python3 .\test.py -f 2-query\timezone.py -python3 .\test.py -f 2-query\Now.py -python3 .\test.py -f 2-query\Today.py -python3 .\test.py -f 2-query\max.py -python3 .\test.py -f 2-query\min.py -python3 .\test.py -f 2-query\count.py -python3 .\test.py -f 2-query\last.py -python3 .\test.py -f 2-query\first.py -python3 .\test.py -f 2-query\To_iso8601.py -python3 .\test.py -f 2-query\To_unixtimestamp.py -python3 .\test.py -f 2-query\timetruncate.py -python3 .\test.py -f 2-query\diff.py -python3 .\test.py -f 2-query\Timediff.py - -python3 .\test.py -f 2-query\top.py -python3 .\test.py -f 2-query\bottom.py -python3 .\test.py -f 2-query\percentile.py -python3 .\test.py -f 2-query\apercentile.py -python3 .\test.py -f 2-query\abs.py -python3 .\test.py -f 2-query\ceil.py -python3 .\test.py -f 2-query\floor.py -python3 .\test.py -f 2-query\round.py -python3 .\test.py -f 2-query\log.py -python3 .\test.py -f 2-query\pow.py -python3 .\test.py -f 2-query\sqrt.py -python3 .\test.py -f 2-query\sin.py -python3 .\test.py -f 2-query\cos.py -python3 .\test.py -f 2-query\tan.py -python3 .\test.py -f 2-query\arcsin.py -python3 .\test.py -f 2-query\arccos.py -python3 .\test.py -f 2-query\arctan.py -python3 .\test.py -f 2-query\query_cols_tags_and_or.py -@REM # python3 .\test.py -f 2-query\nestedQuery.py -@REM # TD-15983 subquery output duplicate name column. -@REM # Please Xiangyang Guo modify the following script -@REM # python3 .\test.py -f 2-query\nestedQuery_str.py - -python3 .\test.py -f 2-query\avg.py -python3 .\test.py -f 2-query\elapsed.py -python3 .\test.py -f 2-query\csum.py -python3 .\test.py -f 2-query\mavg.py -python3 .\test.py -f 2-query\diff.py -python3 .\test.py -f 2-query\sample.py -python3 .\test.py -f 2-query\function_diff.py -python3 .\test.py -f 2-query\unique.py -python3 .\test.py -f 2-query\stateduration.py -python3 .\test.py -f 2-query\function_stateduration.py -python3 .\test.py -f 2-query\statecount.py - -python3 .\test.py -f 7-tmq\basic5.py -python3 .\test.py -f 7-tmq\subscribeDb.py -python3 .\test.py -f 7-tmq\subscribeDb0.py -python3 .\test.py -f 7-tmq\subscribeDb1.py -python3 .\test.py -f 7-tmq\subscribeStb.py -python3 .\test.py -f 7-tmq\subscribeStb0.py -python3 .\test.py -f 7-tmq\subscribeStb1.py -python3 .\test.py -f 7-tmq\subscribeStb2.py -python3 .\test.py -f 7-tmq\subscribeStb3.py -python3 .\test.py -f 7-tmq\subscribeStb4.py -python3 .\test.py -f 7-tmq\db.py \ No newline at end of file diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 571bb166e1..2e2e390d12 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -141,6 +141,7 @@ python3 ./test.py -f 7-tmq/tmqError.py python3 ./test.py -f 7-tmq/schema.py python3 ./test.py -f 7-tmq/stbFilter.py python3 ./test.py -f 7-tmq/tmqCheckData.py +python3 ./test.py -f 7-tmq/tmqCheckData1.py python3 ./test.py -f 7-tmq/tmqUdf.py #python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5 python3 ./test.py -f 7-tmq/tmqConsumerGroup.py diff --git a/tests/system-test/test-all.bat b/tests/system-test/test-all.bat index adc9e0ce28..d4032cbc0d 100644 --- a/tests/system-test/test-all.bat +++ b/tests/system-test/test-all.bat @@ -6,19 +6,23 @@ if "%1" == "full" ( echo Windows Taosd Full Test set /a exitNum=0 del /Q /F failed.txt - set caseFile="fulltest.bat" + set caseFile="fulltest.sh" if not "%2" == "" ( set caseFile="%2" ) for /F "usebackq tokens=*" %%i in (!caseFile!) do ( - for /f "tokens=1* delims= " %%a in ("%%i") do if not "%%a" == "@REM" ( - set /a a+=1 - echo !a! Processing %%i - call :GetTimeSeconds !time! - set time1=!_timeTemp! - echo Start at !time! - call %%i ARG1 > result_!a!.txt 2>error_!a!.txt - if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. ) + call :CheckSkipCase %%i + if !skipCase! == false ( + set line=%%i + if "!line:~,7!" == "python3" ( + set /a a+=1 + echo !a! Processing %%i + call :GetTimeSeconds !time! + set time1=!_timeTemp! + echo Start at !time! + call %%i ARG1 > result_!a!.txt 2>error_!a!.txt + if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. ) + ) ) ) exit !exitNum! @@ -83,3 +87,8 @@ for %%a in (%tt%) do ( ) set /a _timeTemp=(%hh%*60+%mm%)*60+%ss% goto :eof + +:CheckSkipCase +set skipCase=false +if "%*" == "python3 ./test.py -f 1-insert/insertWithMoreVgroup.py" ( set skipCase=true ) +:goto eof \ No newline at end of file diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index cd6613b17a..2037f67089 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -339,8 +339,8 @@ int32_t shellParseArgs(int32_t argc, char *argv[]) { shell.info.clientVersion = "Welcome to the TDengine shell from %s, Client Version:%s\n" "Copyright (c) 2022 by TAOS Data, Inc. All rights reserved.\n\n"; - shell.info.promptHeader = "taos> "; - shell.info.promptContinue = " -> "; + shell.info.promptHeader = TAOS_CONSOLE_PROMPT_HEADER; + shell.info.promptContinue = TAOS_CONSOLE_PROMPT_CONTINUE; shell.info.promptSize = 6; snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), "version: %s", version); diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index f236c1eb88..9cb3541017 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -411,8 +411,9 @@ char taosGetConsoleChar() { static char mbStr[5]; static unsigned long bufLen = 0; static uint16_t bufIndex = 0, mbStrIndex = 0, mbStrLen = 0; - if (bufLen == 0) { + while (bufLen == 0) { ReadConsoleW(console, buf, SHELL_INPUT_MAX_COMMAND_SIZE, &bufLen, NULL); + if (bufLen > 0 && buf[0] == 0) bufLen = 0; bufIndex = 0; } if (mbStrLen == 0){ @@ -466,6 +467,8 @@ int32_t shellReadCommand(char *command) { } else if (c < '\033') { // Ctrl keys. TODO: Implement ctrl combinations switch (c) { + case 0: + break; case 1: // ctrl A shellPositionCursorHome(&cmd); break;