diff --git a/Jenkinsfile2 b/Jenkinsfile2 index f582461fb2..f618912295 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -338,6 +338,14 @@ pipeline { changeRequest() } steps { + script { + def linux_node_ip = sh ( + script: 'ip addr|grep 192|grep -v virbr|awk "{print \\\$2}"|sed "s/\\/.*//"', + returnStdout: true + ).trim() + echo "${linux_node_ip}" + echo "${WKDIR}/restore.sh -p ${BRANCH_NAME} -n ${BUILD_ID} -c {container name}" + } catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { timeout(time: 120, unit: 'MINUTES'){ pre_test() diff --git a/docs/en/21-tdinternal/01-arch.md b/docs/en/21-tdinternal/01-arch.md index d7d472eb98..44651c0496 100644 --- a/docs/en/21-tdinternal/01-arch.md +++ b/docs/en/21-tdinternal/01-arch.md @@ -22,7 +22,7 @@ A complete TDengine system runs on one or more physical nodes. Logically, it inc **Virtual node (vnode)**: To better support data sharding, load balancing and prevent data from overheating or skewing, data nodes are virtualized into multiple virtual nodes (vnode, V2, V3, V4, etc. in the figure). Each vnode is a relatively independent work unit, which is the basic unit of time-series data storage and has independent running threads, memory space and persistent storage path. A vnode contains a certain number of tables (data collection points). When a new table is created, the system checks whether a new vnode needs to be created. The number of vnodes that can be created on a data node depends on the capacity of the hardware of the physical node where the data node is located. A vnode belongs to only one DB, but a DB can have multiple vnodes. In addition to the stored time-series data, a vnode also stores the schema and tag values of the included tables. A virtual node is uniquely identified in the system by the EP of the data node and the VGroup ID to which it belongs and is created and managed by the management node. -**Management node (mnode)**: A virtual logical unit responsible for monitoring and maintaining the running status of all data nodes and load balancing among nodes (M in the figure). At the same time, the management node is also responsible for the storage and management of metadata (including users, databases, tables, static tags, etc.), so it is also called Meta Node. Multiple (up to 5) mnodes can be configured in a TDengine cluster, and they are automatically constructed into a virtual management node group (M0, M1, M2 in the figure). The leader/follower mechanism is adopted for the mnode group and the data synchronization is carried out in a strongly consistent way. Any data update operation can only be executed on the leader. The creation of mnode cluster is completed automatically by the system without manual intervention. There is at most one mnode on each dnode, which is uniquely identified by the EP of the data node to which it belongs. Each dnode automatically obtains the EP of the dnode where all mnodes in the whole cluster are located, through internal messaging interaction. +**Management node (mnode)**: A virtual logical unit responsible for monitoring and maintaining the running status of all data nodes and load balancing among nodes (M in the figure). At the same time, the management node is also responsible for the storage and management of metadata (including users, databases, tables, static tags, etc.), so it is also called Meta Node. Multiple (up to 3) mnodes can be configured in a TDengine cluster, and they are automatically constructed into a virtual management node group (M0, M1, M2 in the figure). The leader/follower mechanism is adopted for the mnode group and the data synchronization is carried out in a strongly consistent way. Any data update operation can only be executed on the leader. The creation of mnode cluster is completed automatically by the system without manual intervention. There is at most one mnode on each dnode, which is uniquely identified by the EP of the data node to which it belongs. Each dnode automatically obtains the EP of the dnode where all mnodes in the whole cluster are located, through internal messaging interaction. **Virtual node group (VGroup)**: Vnodes on different data nodes can form a virtual node group to ensure the high availability of the system. The virtual node group is managed in a leader/follower mechanism. Write operations can only be performed on the leader vnode, and then replicated to follower vnodes, thus ensuring that one single replica of data is copied on multiple physical nodes. The number of virtual nodes in a vgroup equals the number of data replicas. If the number of replicas of a DB is N, the system must have at least N data nodes. The number of replicas can be specified by the parameter `“replica”` when creating a DB, and the default is 1. Using the multi-replication feature of TDengine, the same high data reliability can be achieved without the need for expensive storage devices such as disk arrays. Virtual node groups are created and managed by the management node, and the management node assigns a system unique ID, aka VGroup ID. If two virtual nodes have the same vnode group ID, it means that they belong to the same group and the data is backed up to each other. The number of virtual nodes in a virtual node group can be dynamically changed, allowing only one, that is, no data replication. VGroup ID is never changed. Even if a virtual node group is deleted, its ID will not be reused. diff --git a/docs/zh/21-tdinternal/01-arch.md b/docs/zh/21-tdinternal/01-arch.md index 433cb4808b..507ccea629 100644 --- a/docs/zh/21-tdinternal/01-arch.md +++ b/docs/zh/21-tdinternal/01-arch.md @@ -23,7 +23,7 @@ TDengine 分布式架构的逻辑结构图如下： **虚拟节点（vnode）：** 为更好的支持数据分片、负载均衡，防止数据过热或倾斜，数据节点被虚拟化成多个虚拟节点（vnode，图中 V2，V3，V4 等）。每个 vnode 都是一个相对独立的工作单元，是时序数据存储的基本单元，具有独立的运行线程、内存空间与持久化存储的路径。一个 vnode 包含一定数量的表（数据采集点）。当创建一张新表时，系统会检查是否需要创建新的 vnode。一个数据节点上能创建的 vnode 的数量取决于该数据节点所在物理节点的硬件资源。一个 vnode 只属于一个 DB，但一个 DB 可以有多个 vnode。一个 vnode 除存储的时序数据外，也保存有所包含的表的 schema、标签值等。一个虚拟节点由所属的数据节点的 EP，以及所属的 VGroup ID 在系统内唯一标识，由管理节点创建并管理。 -**管理节点（mnode）：** 一个虚拟的逻辑单元，负责所有数据节点运行状态的监控和维护，以及节点之间的负载均衡（图中 M）。同时，管理节点也负责元数据（包括用户、数据库、表、静态标签等）的存储和管理，因此也称为 Meta Node。TDengine 集群中可配置多个（开源版最多不超过 3 个）mnode，它们自动构建成为一个虚拟管理节点组（图中 M0，M1，M2）。mnode 间采用 master/slave 的机制进行管理，而且采取强一致方式进行数据同步，任何数据更新操作只能在 Master 上进行。mnode 集群的创建由系统自动完成，无需人工干预。每个 dnode 上至多有一个 mnode，由所属的数据节点的 EP 来唯一标识。每个 dnode 通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的 EP。 +**管理节点（mnode）：** 一个虚拟的逻辑单元，负责所有数据节点运行状态的监控和维护，以及节点之间的负载均衡（图中 M）。同时，管理节点也负责元数据（包括用户、数据库、表、静态标签等）的存储和管理，因此也称为 Meta Node。TDengine 集群中可配置多个（最多不超过 3 个）mnode，它们自动构建成为一个虚拟管理节点组（图中 M0，M1，M2）。mnode 间采用 master/slave 的机制进行管理，而且采取强一致方式进行数据同步，任何数据更新操作只能在 Master 上进行。mnode 集群的创建由系统自动完成，无需人工干预。每个 dnode 上至多有一个 mnode，由所属的数据节点的 EP 来唯一标识。每个 dnode 通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的 EP。 **虚拟节点组（VGroup）：** 不同数据节点上的 vnode 可以组成一个虚拟节点组（vgroup）来保证系统的高可靠。虚拟节点组内采取 master/slave 的方式进行管理。写操作只能在 master vnode 上进行，系统采用异步复制的方式将数据同步到 slave vnode，这样确保了一份数据在多个物理节点上有拷贝。一个 vgroup 里虚拟节点个数就是数据的副本数。如果一个 DB 的副本数为 N，系统必须有至少 N 数据节点。副本数在创建 DB 时通过参数 replica 可以指定，缺省为 1。使用 TDengine 的多副本特性，可以不再需要昂贵的磁盘阵列等存储设备，就可以获得同样的数据高可靠性。虚拟节点组由管理节点创建、管理，并且由管理节点分配一个系统唯一的 ID，VGroup ID。如果两个虚拟节点的 VGroup ID 相同，说明他们属于同一个组，数据互为备份。虚拟节点组里虚拟节点的个数是可以动态改变的，容许只有一个，也就是没有数据复制。VGroup ID 是永远不变的，即使一个虚拟节点组被删除，它的 ID 也不会被收回重复利用。 diff --git a/include/client/taos.h b/include/client/taos.h index 651faf5dcc..79f567fc9a 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -128,6 +128,7 @@ typedef struct setConfRet { DLL_EXPORT void taos_cleanup(void); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); +DLL_EXPORT setConfRet taos_set_config(const char *config); DLL_EXPORT int taos_init(void); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 2e2c7d1700..6653fdd9cd 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -236,7 +236,7 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* int8_t needCompress); const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData); -void blockDebugShowData(const SArray* dataBlocks, const char* flag); +void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag); // for debug char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf); diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index f9ede63f7f..8f7f22a6a0 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -78,7 +78,6 @@ int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag); int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag); int32_t tTagToValArray(const STag *pTag, SArray **ppArray); void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remove -void debugCheckTags(STag *pTag); // TODO: remove // STRUCT ================= struct STColumn { diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a5688af18a..c5b0b89311 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1886,7 +1886,7 @@ typedef struct SVCreateStbReq { int8_t rollup; SSchemaWrapper schemaRow; SSchemaWrapper schemaTag; - SRSmaParam pRSmaParam; + SRSmaParam rsmaParam; } SVCreateStbReq; int tEncodeSVCreateStbReq(SEncoder* pCoder, const SVCreateStbReq* pReq); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index e23b84c1f8..0bd917a9c6 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -24,6 +24,8 @@ extern "C" { #include "querynodes.h" #include "tname.h" +#define SLOT_NAME_LEN TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN + typedef struct SLogicNode { ENodeType type; SNodeList* pTargets; // SColumnNode @@ -74,8 +76,8 @@ typedef struct SScanLogicNode { int16_t tsColId; double filesFactor; SArray* pSmaIndexes; - SNodeList* pPartTags; - bool partSort; + SNodeList* pGroupTags; + bool groupSort; } SScanLogicNode; typedef struct SJoinLogicNode { @@ -100,6 +102,7 @@ typedef struct SProjectLogicNode { typedef struct SIndefRowsFuncLogicNode { SLogicNode node; SNodeList* pFuncs; + bool isTailFunc; } SIndefRowsFuncLogicNode; typedef struct SInterpFuncLogicNode { @@ -138,6 +141,7 @@ typedef struct SMergeLogicNode { SNodeList* pInputs; int32_t numOfChannels; int32_t srcGroupId; + bool groupSort; } SMergeLogicNode; typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType; @@ -184,6 +188,7 @@ typedef struct SFillLogicNode { typedef struct SSortLogicNode { SLogicNode node; SNodeList* pSortKeys; + bool groupSort; } SSortLogicNode; typedef struct SPartitionLogicNode { @@ -230,6 +235,7 @@ typedef struct SSlotDescNode { bool reserve; bool output; bool tag; + char name[SLOT_NAME_LEN]; } SSlotDescNode; typedef struct SDataBlockDescNode { @@ -279,7 +285,8 @@ typedef struct STableScanPhysiNode { double ratio; int32_t dataRequired; SNodeList* pDynamicScanFuncs; - SNodeList* pPartitionTags; + SNodeList* pGroupTags; + bool groupSort; int64_t interval; int64_t offset; int64_t sliding; @@ -353,6 +360,7 @@ typedef struct SMergePhysiNode { SNodeList* pTargets; int32_t numOfChannels; int32_t srcGroupId; + bool groupSort; } SMergePhysiNode; typedef struct SWinodwPhysiNode { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index a6e466e73e..39c569acef 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -259,6 +259,7 @@ typedef struct SSelectStmt { bool hasTailFunc; bool hasInterpFunc; bool hasLastRowFunc; + bool groupSort; } SSelectStmt; typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType; diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 52d9b6f810..5fa7eed40c 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -26,9 +26,9 @@ extern "C" { extern bool gRaftDetailLog; -#define SYNC_INDEX_BEGIN 0 +#define SYNC_INDEX_BEGIN 0 #define SYNC_INDEX_INVALID -1 -#define SYNC_TERM_INVALID 0xFFFFFFFFFFFFFFFF +#define SYNC_TERM_INVALID 0xFFFFFFFFFFFFFFFF typedef uint64_t SyncNodeId; typedef int32_t SyncGroupId; diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index e51b20a404..5d892352d6 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -524,7 +524,7 @@ void syncReconfigFinishLog2(char* s, const SyncReconfigFinish* pMsg); int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg); -int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg); +int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex); int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg); int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg); int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg); @@ -541,7 +541,7 @@ int32_t syncNodeOnSnapshotRspCb(SSyncNode* ths, SyncSnapshotRsp* pMsg); // ----------------------------------------- typedef int32_t (*FpOnPingCb)(SSyncNode* ths, SyncPing* pMsg); typedef int32_t (*FpOnPingReplyCb)(SSyncNode* ths, SyncPingReply* pMsg); -typedef int32_t (*FpOnClientRequestCb)(SSyncNode* ths, SyncClientRequest* pMsg); +typedef int32_t (*FpOnClientRequestCb)(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex); typedef int32_t (*FpOnRequestVoteCb)(SSyncNode* ths, SyncRequestVote* pMsg); typedef int32_t (*FpOnRequestVoteReplyCb)(SSyncNode* ths, SyncRequestVoteReply* pMsg); typedef int32_t (*FpOnAppendEntriesCb)(SSyncNode* ths, SyncAppendEntries* pMsg); diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index c2c1a3534d..2b8c6a895e 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -69,7 +69,7 @@ typedef struct SRpcMsg { } SRpcMsg; typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *rf); -typedef bool (*RpcRfp)(int32_t code); +typedef bool (*RpcRfp)(int32_t code, tmsg_t msgType); typedef struct SRpcInit { char localFqdn[TSDB_FQDN_LEN]; diff --git a/include/os/osSystem.h b/include/os/osSystem.h index 6770be6e46..581e688ccb 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -29,6 +29,9 @@ extern "C" { #define tcgetattr TCGETATTR_FUNC_TAOS_FORBID #endif +#define TAOS_CONSOLE_PROMPT_HEADER "taos> " +#define TAOS_CONSOLE_PROMPT_CONTINUE " -> " + typedef struct TdCmd *TdCmdPtr; TdCmdPtr taosOpenCmd(const char* cmd); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 56b675596f..22b491994e 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -88,9 +88,12 @@ void closeTransporter(SAppInstInfo *pAppInfo) { rpcClose(pAppInfo->pTransporter); } -static bool clientRpcRfp(int32_t code) { +static bool clientRpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { + if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) { + return false; + } return true; } else { return false; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index be3efe0f98..e568112dfa 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -81,6 +81,19 @@ void taos_cleanup(void) { taosCloseLog(); } +static setConfRet taos_set_config_imp(const char *config){ + setConfRet ret = {SET_CONF_RET_SUCC, {0}}; + // TODO: need re-implementation + return ret; +} + +setConfRet taos_set_config(const char *config){ +// TODO pthread_mutex_lock(&setConfMutex); + setConfRet ret = taos_set_config_imp(config); +// pthread_mutex_unlock(&setConfMutex); + return ret; +} + TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { tscDebug("try to connect to %s:%u, user:%s db:%s", ip, port, user, db); if (user == NULL) { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 9f89d72172..1ec298ee15 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1605,7 +1605,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { return buf; } -void blockDebugShowData(const SArray* dataBlocks, const char* flag) { +void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) { char pBuf[128] = {0}; int32_t sz = taosArrayGetSize(dataBlocks); for (int32_t i = 0; i < sz; i++) { @@ -1613,7 +1613,7 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) { size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); int32_t rows = pDataBlock->info.rows; - printf("%s |block type %d |child id %d|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId); + printf("%s |block type %d |child id %d|group id %zX\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId); for (int32_t j = 0; j < rows; j++) { printf("%s |", flag); for (int32_t k = 0; k < numOfCols; k++) { diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 8460a27a0e..7c1b31b6e4 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -862,21 +862,6 @@ void debugPrintSTag(STag *pTag, const char *tag, int32_t ln) { printf("\n"); } -void debugCheckTags(STag *pTag) { - switch (pTag->flags) { - case 0x0: - case 0x20: - case 0x40: - case 0x60: - break; - default: - ASSERT(0); - } - - ASSERT(pTag->nTag <= 128 && pTag->nTag >= 0); - ASSERT(pTag->ver <= 512 && pTag->ver >= 0); // temp condition for pTag->ver -} - static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { int32_t n = 0; @@ -999,7 +984,6 @@ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag) { debugPrintSTag(*ppTag, __func__, __LINE__); #endif - debugCheckTags(*ppTag); // TODO: remove this line after debug return code; _err: diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 9db5019512..e9b5c67d76 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3696,7 +3696,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeI8(&decoder, &pReq->isTsma) < 0) return -1; if (pReq->isTsma) { - if (tDecodeBinaryAlloc(&decoder, &pReq->pTsma, NULL) < 0) return -1; + if (tDecodeBinary(&decoder, (uint8_t **)&pReq->pTsma, NULL) < 0) return -1; } tEndDecode(&decoder); @@ -3707,9 +3707,6 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * int32_t tFreeSCreateVnodeReq(SCreateVnodeReq *pReq) { taosArrayDestroy(pReq->pRetensions); pReq->pRetensions = NULL; - if (pReq->isTsma) { - taosMemoryFreeClear(pReq->pTsma); - } return 0; } @@ -4747,9 +4744,8 @@ int32_t tDecodeSRSmaParam(SDecoder *pCoder, SRSmaParam *pRSmaParam) { if (tDecodeI64v(pCoder, &pRSmaParam->watermark[i]) < 0) return -1; if (tDecodeI32v(pCoder, &pRSmaParam->qmsgLen[i]) < 0) return -1; if (pRSmaParam->qmsgLen[i] > 0) { - uint64_t len; - if (tDecodeBinaryAlloc(pCoder, (void **)&pRSmaParam->qmsg[i], &len) < 0) - return -1; // qmsgLen contains len of '\0' + tDecoderMalloc(pCoder, pRSmaParam->qmsgLen[i]); + if (tDecodeBinary(pCoder, (uint8_t **)&pRSmaParam->qmsg[i], NULL) < 0) return -1; // qmsgLen contains len of '\0' } else { pRSmaParam->qmsg[i] = NULL; } @@ -4767,7 +4763,7 @@ int tEncodeSVCreateStbReq(SEncoder *pCoder, const SVCreateStbReq *pReq) { if (tEncodeSSchemaWrapper(pCoder, &pReq->schemaRow) < 0) return -1; if (tEncodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1; if (pReq->rollup) { - if (tEncodeSRSmaParam(pCoder, &pReq->pRSmaParam) < 0) return -1; + if (tEncodeSRSmaParam(pCoder, &pReq->rsmaParam) < 0) return -1; } tEndEncode(pCoder); @@ -4783,7 +4779,7 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) { if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaRow) < 0) return -1; if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1; if (pReq->rollup) { - if (tDecodeSRSmaParam(pCoder, &pReq->pRSmaParam) < 0) return -1; + if (tDecodeSRSmaParam(pCoder, &pReq->rsmaParam) < 0) return -1; } tEndDecode(pCoder); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index a4745abd5b..7e31cc3144 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -248,9 +248,12 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { } } -static bool rpcRfp(int32_t code) { +static bool rpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { + if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) { + return false; + } return true; } else { return false; diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index bc2d5c82b1..1497bba11c 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -39,8 +39,10 @@ typedef struct { int32_t id; int32_t errCode; int32_t acceptableCode; - ETrnStage stage; + int32_t retryCode; ETrnAct actionType; + ETrnStage stage; + int8_t reserved; int8_t rawWritten; int8_t msgSent; int8_t msgReceived; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index e03f82ba18..136afb714e 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -442,7 +442,7 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { syncPingReplyDestroy(pSyncMsg); } else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pMsg); - code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); + code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg, NULL); syncClientRequestDestroy(pSyncMsg); } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg); @@ -491,7 +491,7 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { syncPingReplyDestroy(pSyncMsg); } else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pMsg); - code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); + code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg, NULL); syncClientRequestDestroy(pSyncMsg); } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg); diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 029a1e6f8c..164bcc7d60 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -234,7 +234,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { if (retrieveReq.user[0] != 0) { memcpy(pReq->info.conn.user, retrieveReq.user, TSDB_USER_LEN); } else { - memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, TSDB_USER_LEN); + memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, strlen(TSDB_DEFAULT_USER) + 1); } if (mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db) != 0) { return -1; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 92f85ecd04..dd01a0fa16 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -427,17 +427,17 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.schemaTag.pSchema = pStb->pTags; if (req.rollup) { - req.pRSmaParam.maxdelay[0] = pStb->maxdelay[0]; - req.pRSmaParam.maxdelay[1] = pStb->maxdelay[1]; + req.rsmaParam.maxdelay[0] = pStb->maxdelay[0]; + req.rsmaParam.maxdelay[1] = pStb->maxdelay[1]; if (pStb->ast1Len > 0) { - if (mndConvertRsmaTask(&req.pRSmaParam.qmsg[0], &req.pRSmaParam.qmsgLen[0], pStb->pAst1, pStb->uid, - STREAM_TRIGGER_WINDOW_CLOSE, req.pRSmaParam.watermark[0]) < 0) { + if (mndConvertRsmaTask(&req.rsmaParam.qmsg[0], &req.rsmaParam.qmsgLen[0], pStb->pAst1, pStb->uid, + STREAM_TRIGGER_WINDOW_CLOSE, req.rsmaParam.watermark[0]) < 0) { goto _err; } } if (pStb->ast2Len > 0) { - if (mndConvertRsmaTask(&req.pRSmaParam.qmsg[1], &req.pRSmaParam.qmsgLen[1], pStb->pAst2, pStb->uid, - STREAM_TRIGGER_WINDOW_CLOSE, req.pRSmaParam.watermark[1]) < 0) { + if (mndConvertRsmaTask(&req.rsmaParam.qmsg[1], &req.rsmaParam.qmsgLen[1], pStb->pAst2, pStb->uid, + STREAM_TRIGGER_WINDOW_CLOSE, req.rsmaParam.watermark[1]) < 0) { goto _err; } } @@ -470,12 +470,12 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt tEncoderClear(&encoder); *pContLen = contLen; - taosMemoryFreeClear(req.pRSmaParam.qmsg[0]); - taosMemoryFreeClear(req.pRSmaParam.qmsg[1]); + taosMemoryFreeClear(req.rsmaParam.qmsg[0]); + taosMemoryFreeClear(req.rsmaParam.qmsg[1]); return pHead; _err: - taosMemoryFreeClear(req.pRSmaParam.qmsg[0]); - taosMemoryFreeClear(req.pRSmaParam.qmsg[1]); + taosMemoryFreeClear(req.rsmaParam.qmsg[0]); + taosMemoryFreeClear(req.rsmaParam.qmsg[1]); return NULL; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index a9de1a05a7..b09ee5f608 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -15,15 +15,15 @@ #define _DEFAULT_SOURCE #include "mndTrans.h" -#include "mndPrivilege.h" #include "mndConsumer.h" #include "mndDb.h" +#include "mndPrivilege.h" #include "mndShow.h" #include "mndSync.h" #include "mndUser.h" -#define TRANS_VER_NUMBER 1 -#define TRANS_ARRAY_SIZE 8 +#define TRANS_VER_NUMBER 1 +#define TRANS_ARRAY_SIZE 8 #define TRANS_RESERVE_SIZE 64 static SSdbRaw *mndTransActionEncode(STrans *pTrans); @@ -55,7 +55,7 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans); static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsMaster(pMnode); } static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans); -static int32_t mndProcessTransReq(SRpcMsg *pReq); +static int32_t mndProcessTransTimer(SRpcMsg *pReq); static int32_t mndProcessTtl(SRpcMsg *pReq); static int32_t mndProcessKillTransReq(SRpcMsg *pReq); @@ -73,7 +73,7 @@ int32_t mndInitTrans(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndTransActionDelete, }; - mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransReq); + mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransTimer); mndSetMsgHandle(pMnode, TDMT_MND_KILL_TRANS, mndProcessKillTransReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TRANS, mndRetrieveTrans); @@ -139,8 +139,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) + SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER) if (pAction->actionType == TRANS_ACTION_RAW) { int32_t len = sdbGetRawTotalSize(pAction->pRaw); SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER) @@ -163,8 +165,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) + SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER) if (pAction->actionType == TRANS_ACTION_RAW) { int32_t len = sdbGetRawTotalSize(pAction->pRaw); SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER) @@ -187,8 +191,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) + SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER) if (pAction->actionType == TRANS_ACTION_RAW) { int32_t len = sdbGetRawTotalSize(pAction->pRaw); SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER) @@ -291,10 +297,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER) SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) action.actionType = actionType; SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) action.stage = stage; + SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER) if (action.actionType == TRANS_ACTION_RAW) { SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER) SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) @@ -324,10 +332,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER) SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) action.actionType = actionType; SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) action.stage = stage; + SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER) if (action.actionType == TRANS_ACTION_RAW) { SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER) SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) @@ -357,10 +367,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER) SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) action.actionType = actionType; SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) action.stage = stage; + SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER) if (action.actionType) { SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER) SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) @@ -463,15 +475,25 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { if (fp) { (*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen); } + pTrans->startFunc = 0; } return 0; } static void mndTransDropData(STrans *pTrans) { - mndTransDropActions(pTrans->redoActions); - mndTransDropActions(pTrans->undoActions); - mndTransDropActions(pTrans->commitActions); + if (pTrans->redoActions != NULL) { + mndTransDropActions(pTrans->redoActions); + pTrans->redoActions = NULL; + } + if (pTrans->undoActions != NULL) { + mndTransDropActions(pTrans->undoActions); + pTrans->undoActions = NULL; + } + if (pTrans->commitActions != NULL) { + mndTransDropActions(pTrans->commitActions); + pTrans->commitActions = NULL; + } if (pTrans->rpcRsp != NULL) { taosMemoryFree(pTrans->rpcRsp); pTrans->rpcRsp = NULL; @@ -492,6 +514,7 @@ static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) { if (fp) { (*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen); } + pTrans->stopFunc = 0; } mndTransDropData(pTrans); @@ -805,7 +828,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { sendRsp = true; } } else { - if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 3) { + if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 6) { if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR; sendRsp = true; } @@ -875,8 +898,8 @@ int32_t mndTransProcessRsp(SRpcMsg *pRsp) { pAction->errCode = pRsp->code; } - mDebug("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x", transId, mndTransStr(pAction->stage), action, - pRsp->code, pAction->acceptableCode); + mDebug("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId, + mndTransStr(pAction->stage), action, pRsp->code, pAction->acceptableCode, pAction->retryCode); mndTransExecute(pMnode, pTrans); _OVER: @@ -884,6 +907,21 @@ _OVER: return 0; } +static void mndTransResetAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { + pAction->rawWritten = 0; + pAction->msgSent = 0; + pAction->msgReceived = 0; + if (pAction->errCode == TSDB_CODE_RPC_REDIRECT || pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR || + pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) { + pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps; + mDebug("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage), + pAction->id, pAction->epSet.inUse); + } else { + mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), pAction->id); + } + pAction->errCode = 0; +} + static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) { int32_t numOfActions = taosArrayGetSize(pArray); @@ -894,18 +932,7 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) continue; if (pAction->rawWritten && (pAction->errCode == 0 || pAction->errCode == pAction->acceptableCode)) continue; - pAction->rawWritten = 0; - pAction->msgSent = 0; - pAction->msgReceived = 0; - if (pAction->errCode == TSDB_CODE_RPC_REDIRECT || pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR || - pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) { - pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps; - mDebug("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage), - action, pAction->epSet.inUse); - } else { - mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), action); - } - pAction->errCode = 0; + mndTransResetAction(pMnode, pTrans, pAction); } } @@ -1112,9 +1139,9 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) if (pAction->msgReceived) { if (pAction->errCode != 0 && pAction->errCode != pAction->acceptableCode) { code = pAction->errCode; - pAction->msgSent = 0; - pAction->msgReceived = 0; - mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), action); + mndTransResetAction(pMnode, pTrans, pAction); + } else { + mDebug("trans:%d, %s:%d execute successfully", pTrans->id, mndTransStr(pAction->stage), action); } } else { code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -1123,6 +1150,8 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) if (pAction->rawWritten) { if (pAction->errCode != 0 && pAction->errCode != pAction->acceptableCode) { code = pAction->errCode; + } else { + mDebug("trans:%d, %s:%d write successfully", pTrans->id, mndTransStr(pAction->stage), action); } } } @@ -1156,9 +1185,16 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) { mDebug("trans:%d, %s:%d is in progress and wait it finish", pTrans->id, mndTransStr(pAction->stage), pAction->id); break; + } else if (code == pAction->retryCode) { + mDebug("trans:%d, %s:%d receive code:0x%x and retry", pTrans->id, mndTransStr(pAction->stage), pAction->id, code); + taosMsleep(300); + action--; + continue; } else { terrno = code; pTrans->code = code; + mDebug("trans:%d, %s:%d receive code:0x%x and wait another schedule, failedTimes:%d", pTrans->id, + mndTransStr(pAction->stage), pAction->id, code, pTrans->failedTimes); break; } } @@ -1343,7 +1379,7 @@ void mndTransExecute(SMnode *pMnode, STrans *pTrans) { mndTransSendRpcRsp(pMnode, pTrans); } -static int32_t mndProcessTransReq(SRpcMsg *pReq) { +static int32_t mndProcessTransTimer(SRpcMsg *pReq) { mndTransPullup(pReq->info.node); return 0; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 0e931e0a9c..d42016e6a0 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -15,10 +15,10 @@ #define _DEFAULT_SOURCE #include "mndVgroup.h" -#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" +#include "mndPrivilege.h" #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" @@ -896,6 +896,8 @@ int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD action.pCont = pHead; action.contLen = contLen; action.msgType = TDMT_VND_ALTER_CONFIRM; + // incorrect redirect result will cause this erro + action.retryCode = TSDB_CODE_VND_INVALID_VGROUP_ID; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pHead); @@ -942,6 +944,8 @@ static int32_t mndAddSetVnodeStandByAction(SMnode *pMnode, STrans *pTrans, SDbOb action.contLen = contLen; action.msgType = TDMT_SYNC_SET_VNODE_STANDBY; action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + // Keep retrying until the target vnode is not the leader + action.retryCode = TSDB_CODE_SYN_IS_LEADER; if (isRedo) { if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -1003,7 +1007,7 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, mInfo("vgId:%d, will add 1 vnodes", pVgroup->vgId); if (mndAddVnodeToVgroup(pMnode, &newVg, pArray) != 0) return -1; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[1], true) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[newVg.replica - 1], true) != 0) return -1; if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg, TDMT_VND_ALTER_REPLICA) != 0) return -1; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; @@ -1017,10 +1021,19 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; - SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) return -1; - sdbSetRawStatus(pRaw, SDB_STATUS_READY); - pRaw = NULL; + { + SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) return -1; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + pRaw = NULL; + } + + { + SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) return -1; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + pRaw = NULL; + } mInfo("vgId:%d, vgroup info after move, replica:%d", newVg.vgId, newVg.replica); for (int32_t i = 0; i < newVg.replica; ++i) { @@ -1168,10 +1181,19 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld3->id) != 0) goto _OVER; } - pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; - sdbSetRawStatus(pRaw, SDB_STATUS_READY); - pRaw = NULL; + { + pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + pRaw = NULL; + } + + { + pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + pRaw = NULL; + } mInfo("vgId:%d, vgroup info after redistribute, replica:%d", newVg.vgId, newVg.replica); for (int32_t i = 0; i < newVg.replica; ++i) { @@ -1229,7 +1251,8 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { } if (req.dnodeId1 == pVgroup->vnodeGid[0].dnodeId) { - terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; + // terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; + code = 0; goto _OVER; } @@ -1351,7 +1374,8 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { } if (pNew1 == NULL && pOld1 == NULL && pNew2 == NULL && pOld2 == NULL && pNew3 == NULL && pOld3 == NULL) { - terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; + // terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED; + code = 0; goto _OVER; } @@ -1424,13 +1448,25 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, S } else { } - SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); - if (pVgRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) { - sdbFreeRaw(pVgRaw); - return -1; + { + SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { + sdbFreeRaw(pVgRaw); + return -1; + } + sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); + } + + { + SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) { + sdbFreeRaw(pVgRaw); + return -1; + } + sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); } - sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); } return 0; @@ -1538,12 +1574,23 @@ static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SD if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pDst->id) != 0) return -1; if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pSrc->id) != 0) return -1; - SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) { - sdbFreeRaw(pRaw); - return -1; + { + SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) { + sdbFreeRaw(pRaw); + return -1; + } + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + } + + { + SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) { + sdbFreeRaw(pRaw); + return -1; + } + sdbSetRawStatus(pRaw, SDB_STATUS_READY); } - sdbSetRawStatus(pRaw, SDB_STATUS_READY); mInfo("vgId:%d, vgroup info after balance, replica:%d", newVg.vgId, newVg.replica); for (int32_t i = 0; i < newVg.replica; ++i) { @@ -1552,7 +1599,8 @@ static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SD return 0; } -static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDnodeObj *pSrc, SDnodeObj *pDst) { +static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDnodeObj *pSrc, SDnodeObj *pDst, + SHashObj *pBalancedVgroups) { void *pIter = NULL; int32_t code = -1; SSdb *pSdb = pMnode->pSdb; @@ -1561,6 +1609,10 @@ static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDno SVgObj *pVgroup = NULL; pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; + if (taosHashGet(pBalancedVgroups, &pVgroup->vgId, sizeof(int32_t)) != NULL) { + sdbRelease(pSdb, pVgroup); + continue; + } bool existInSrc = false; bool existInDst = false; @@ -1577,6 +1629,9 @@ static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDno SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName); code = mndSetBalanceVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, pSrc, pDst); + if (code == 0) { + code = taosHashPut(pBalancedVgroups, &pVgroup->vgId, sizeof(int32_t), &pVgroup->vgId, sizeof(int32_t)); + } mndReleaseDb(pMnode, pDb); sdbRelease(pSdb, pVgroup); sdbCancelFetch(pSdb, pIter); @@ -1590,6 +1645,10 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) { int32_t code = -1; int32_t numOfVgroups = 0; STrans *pTrans = NULL; + SHashObj *pBalancedVgroups = NULL; + + pBalancedVgroups = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (pBalancedVgroups == NULL) goto _OVER; pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); if (pTrans == NULL) goto _OVER; @@ -1613,18 +1672,18 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) { pDst->id, dstScore); if (srcScore > dstScore - 0.000001) { - code = mndBalanceVgroupBetweenDnode(pMnode, pTrans, pSrc, pDst); + code = mndBalanceVgroupBetweenDnode(pMnode, pTrans, pSrc, pDst, pBalancedVgroups); if (code == 0) { pSrc->numOfVnodes--; pDst->numOfVnodes++; numOfVgroups++; continue; } else { - mError("trans:%d, failed to balance vgroup from dnode:%d to dnode:%d", pTrans->id, pSrc->id, pDst->id); - return -1; + mDebug("trans:%d, no vgroup need to balance from dnode:%d to dnode:%d", pTrans->id, pSrc->id, pDst->id); + break; } } else { - mDebug("trans:%d, no vgroup need to balance vgroup any more", pTrans->id); + mDebug("trans:%d, no vgroup need to balance any more", pTrans->id); break; } } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 858376519d..5c2d2cd712 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -125,6 +125,7 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo bool isTsdbCacheLastRow(tsdbReaderT *pReader); int32_t tsdbGetAllTableList(SMeta *pMeta, uint64_t uid, SArray *list); int32_t tsdbGetCtbIdList(SMeta *pMeta, int64_t suid, SArray *list); +int32_t tsdbGetStbIdList(SMeta *pMeta, int64_t suid, SArray *list); void *tsdbGetIdx(SMeta *pMeta); void *tsdbGetIvtIdx(SMeta *pMeta); int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT *pHandle); @@ -198,15 +199,20 @@ typedef struct { uint64_t groupId; } STableKeyInfo; +#define TABLE_ROLLUP_ON ((int8_t)0x1) +#define TABLE_IS_ROLLUP(FLG) (((FLG) & (TABLE_ROLLUP_ON)) != 0) +#define TABLE_SET_ROLLUP(FLG) ((FLG) |= TABLE_ROLLUP_ON) struct SMetaEntry { int64_t version; int8_t type; + int8_t flags; // TODO: need refactor? tb_uid_t uid; char *name; union { struct { SSchemaWrapper schemaRow; SSchemaWrapper schemaTag; + SRSmaParam rsmaParam; } stbEntry; struct { int64_t ctime; diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 63a8e7ec68..66d1689d57 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -69,6 +69,7 @@ struct SMeta { TTB* pUidIdx; TTB* pNameIdx; TTB* pCtbIdx; + TTB* pSuidIdx; // ivt idx and idx void* pTagIvtIdx; TTB* pTagIdx; diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 1489b02492..4b42ab5263 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -60,8 +60,9 @@ struct SRSmaStat { SSma *pSma; void *tmrHandle; tmr_h tmrId; - int8_t tmrStat; int32_t tmrSeconds; + int8_t triggerStat; + int8_t runningStat; SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; }; @@ -72,13 +73,21 @@ struct SSmaStat { }; T_REF_DECLARE() }; -#define SMA_TSMA_STAT(s) (&(s)->tsmaStat) -#define SMA_RSMA_STAT(s) (&(s)->rsmaStat) -#define SMA_RSMA_INFO_HASH(s) ((s)->rsmaStat.rsmaInfoHash) -#define SMA_RSMA_TMR_HANDLE(s) ((s)->rsmaStat.tmrHandle) -#define SMA_RSMA_TMR_STAT(s) ((s)->rsmaStat.tmrStat) -#define RSMA_INFO_HASH(r) ((r)->rsmaInfoHash) +#define SMA_TSMA_STAT(s) (&(s)->tsmaStat) +#define SMA_RSMA_STAT(s) (&(s)->rsmaStat) +#define RSMA_INFO_HASH(r) ((r)->rsmaInfoHash) +#define RSMA_TMR_ID(r) ((r)->tmrId) +#define RSMA_TMR_HANDLE(r) ((r)->tmrHandle) +#define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat) +#define RSMA_RUNNING_STAT(r) (&(r)->runningStat) +enum { + TASK_TRIGGER_STAT_INIT = 0, + TASK_TRIGGER_STAT_ACTIVE = 1, + TASK_TRIGGER_STAT_INACTIVE = 2, + TASK_TRIGGER_STAT_CANCELLED = 3, + TASK_TRIGGER_STAT_FINISHED = 4, +}; void tdDestroySmaEnv(SSmaEnv *pSmaEnv); void *tdFreeSmaEnv(SSmaEnv *pSmaEnv); @@ -167,13 +176,18 @@ static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) { static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType); +void *tdFreeRSmaInfo(SRSmaInfo *pInfo); -void *tdFreeRSmaInfo(SRSmaInfo *pInfo); - +int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); +int32_t tdProcessRSmaRestoreImpl(SSma *pSma); int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg); int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg); int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); +// smaFileUtil ================ + +#define TD_FILE_HEAD_SIZE 512 + typedef struct STFInfo STFInfo; typedef struct STFile STFile; @@ -181,7 +195,7 @@ struct STFInfo { uint32_t magic; uint32_t ftype; uint32_t fver; - uint64_t fsize; + int64_t fsize; }; struct STFile { @@ -199,11 +213,8 @@ struct STFile { #define TD_FILE_OPENED(tf) (TD_FILE_PFILE(tf) != NULL) #define TD_FILE_CLOSED(tf) (!TD_FILE_OPENED(tf)) #define TD_FILE_SET_CLOSED(f) (TD_FILE_PFILE(f) = NULL) -#define TD_FILE_STATE(tf) ((tf)->state) #define TD_FILE_SET_STATE(tf, s) ((tf)->state = (s)) #define TD_FILE_DID(tf) (TD_FILE_F(tf)->did) -#define TD_FILE_IS_OK(tf) (TD_FILE_STATE(tf) == TD_FILE_STATE_OK) -#define TD_FILE_IS_BAD(tf) (TD_FILE_STATE(tf) == TD_FILE_STATE_BAD) int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname); int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType); @@ -212,12 +223,14 @@ int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte); int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence); int64_t tdWriteTFile(STFile *pTFile, void *buf, int64_t nbyte); int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset); +int64_t tdGetTFileSize(STFile *pTFile, int64_t *size); int32_t tdRemoveTFile(STFile *pTFile); int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo); int32_t tdUpdateTFileHeader(STFile *pTFile); void tdUpdateTFileMagic(STFile *pTFile, void *pCksm); void tdCloseTFile(STFile *pTFile); -void tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName); + +void tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 3b30224e17..68ed6dde51 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -76,6 +76,7 @@ void vnodeFree(void* p); // meta typedef struct SMCtbCursor SMCtbCursor; +typedef struct SMStbCursor SMStbCursor; typedef struct STbUidStore STbUidStore; int metaOpen(SVnode* pVnode, SMeta** ppMeta); @@ -97,6 +98,9 @@ int metaGetTbNum(SMeta* pMeta); SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid); void metaCloseCtbCursor(SMCtbCursor* pCtbCur); tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur); +SMStbCursor* metaOpenStbCursor(SMeta* pMeta, tb_uid_t uid); +void metaCloseStbCursor(SMStbCursor* pStbCur); +tb_uid_t metaStbCursorNext(SMStbCursor* pStbCur); STSma* metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid); STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid, bool deepCopy); SArray* metaGetSmaIdsByTable(SMeta* pMeta, tb_uid_t uid); @@ -158,6 +162,8 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool // sma int32_t smaOpen(SVnode* pVnode); int32_t smaClose(SSma* pSma); +int32_t smaCloseEnv(SSma* pSma); +int32_t smaCloseEx(SSma* pSma); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index acf5b0b613..23d7665ba3 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -24,22 +24,25 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { if (tEncodeCStr(pCoder, pME->name) < 0) return -1; if (pME->type == TSDB_SUPER_TABLE) { + if (tEncodeI8(pCoder, pME->flags) < 0) return -1; // TODO: need refactor? if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaRow) < 0) return -1; if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1; + if (TABLE_IS_ROLLUP(pME->flags)) { + if (tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1; + } } else if (pME->type == TSDB_CHILD_TABLE) { if (tEncodeI64(pCoder, pME->ctbEntry.ctime) < 0) return -1; if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1; - if (tEncodeI32(pCoder, pME->ctbEntry.commentLen) < 0) return -1; + if (tEncodeI32v(pCoder, pME->ctbEntry.commentLen) < 0) return -1; if (pME->ctbEntry.commentLen > 0){ if (tEncodeCStr(pCoder, pME->ctbEntry.comment) < 0) return -1; } if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1; - debugCheckTags((STag*)pME->ctbEntry.pTags); // TODO: remove after debug if (tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags) < 0) return -1; } else if (pME->type == TSDB_NORMAL_TABLE) { if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1; if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1; - if (tEncodeI32(pCoder, pME->ntbEntry.commentLen) < 0) return -1; + if (tEncodeI32v(pCoder, pME->ntbEntry.commentLen) < 0) return -1; if (pME->ntbEntry.commentLen > 0){ if (tEncodeCStr(pCoder, pME->ntbEntry.comment) < 0) return -1; } @@ -64,23 +67,26 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { if (tDecodeCStr(pCoder, &pME->name) < 0) return -1; if (pME->type == TSDB_SUPER_TABLE) { + if (tDecodeI8(pCoder, &pME->flags) < 0) return -1; // TODO: need refactor? if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaRow) < 0) return -1; if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag) < 0) return -1; + if (TABLE_IS_ROLLUP(pME->flags)) { + if (tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1; + } } else if (pME->type == TSDB_CHILD_TABLE) { if (tDecodeI64(pCoder, &pME->ctbEntry.ctime) < 0) return -1; if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1; - if (tDecodeI32(pCoder, &pME->ctbEntry.commentLen) < 0) return -1; + if (tDecodeI32v(pCoder, &pME->ctbEntry.commentLen) < 0) return -1; if (pME->ctbEntry.commentLen > 0){ if (tDecodeCStr(pCoder, &pME->ctbEntry.comment) < 0) return -1; } if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1; if (tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags) < 0) return -1; // (TODO) - debugCheckTags((STag*)pME->ctbEntry.pTags); // TODO: remove after debug } else if (pME->type == TSDB_NORMAL_TABLE) { if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1; if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1; - if (tDecodeI32(pCoder, &pME->ntbEntry.commentLen) < 0) return -1; + if (tDecodeI32v(pCoder, &pME->ntbEntry.commentLen) < 0) return -1; if (pME->ntbEntry.commentLen > 0){ if (tDecodeCStr(pCoder, &pME->ntbEntry.comment) < 0) return -1; } diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 95510c25e5..614ccc5b5e 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -92,6 +92,13 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { goto _err; } + // open pSuidIdx + ret = tdbTbOpen("suid.idx", sizeof(tb_uid_t), 0, uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pSuidIdx); + if (ret < 0) { + metaError("vgId:%d, failed to open meta super table index since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + // open pTagIdx // TODO(yihaoDeng), refactor later char indexFullPath[128] = {0}; @@ -141,6 +148,7 @@ _err: if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx); if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); + if (pMeta->pSuidIdx) tdbTbClose(pMeta->pSuidIdx); if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb); @@ -162,6 +170,7 @@ int metaClose(SMeta *pMeta) { if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); #endif if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); + if (pMeta->pSuidIdx) tdbTbClose(pMeta->pSuidIdx); if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 910f4bba51..85106f46c2 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -325,6 +325,72 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { return pCtbIdxKey->uid; } +struct SMStbCursor { + SMeta *pMeta; + TBC *pCur; + tb_uid_t suid; + void *pKey; + void *pVal; + int kLen; + int vLen; +}; + +SMStbCursor *metaOpenStbCursor(SMeta *pMeta, tb_uid_t suid) { + SMStbCursor *pStbCur = NULL; + int ret = 0; + int c = 0; + + pStbCur = (SMStbCursor *)taosMemoryCalloc(1, sizeof(*pStbCur)); + if (pStbCur == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pStbCur->pMeta = pMeta; + pStbCur->suid = suid; + metaRLock(pMeta); + + ret = tdbTbcOpen(pMeta->pSuidIdx, &pStbCur->pCur, NULL); + if (ret < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + metaULock(pMeta); + taosMemoryFree(pStbCur); + return NULL; + } + + // move to the suid + tdbTbcMoveTo(pStbCur->pCur, &suid, sizeof(suid), &c); + if (c > 0) { + tdbTbcMoveToNext(pStbCur->pCur); + } + + return pStbCur; +} + +void metaCloseStbCursor(SMStbCursor *pStbCur) { + if (pStbCur) { + if (pStbCur->pMeta) metaULock(pStbCur->pMeta); + if (pStbCur->pCur) { + tdbTbcClose(pStbCur->pCur); + + tdbFree(pStbCur->pKey); + tdbFree(pStbCur->pVal); + } + + taosMemoryFree(pStbCur); + } +} + +tb_uid_t metaStbCursorNext(SMStbCursor *pStbCur) { + int ret; + + ret = tdbTbcNext(pStbCur->pCur, &pStbCur->pKey, &pStbCur->kLen, &pStbCur->pVal, &pStbCur->vLen); + if (ret < 0) { + return 0; + } + return *(tb_uid_t*)pStbCur->pKey; +} + STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { // SMetaReader mr = {0}; STSchema * pTSchema = NULL; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 7109bf1dfc..a621b4ddb0 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -23,6 +23,7 @@ static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME); +static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry); static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type); @@ -138,6 +139,10 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { me.name = pReq->name; me.stbEntry.schemaRow = pReq->schemaRow; me.stbEntry.schemaTag = pReq->schemaTag; + if (pReq->rollup) { + TABLE_SET_ROLLUP(me.flags); + me.stbEntry.rsmaParam = pReq->rsmaParam; + } if (metaHandleEntry(pMeta, &me) < 0) goto _err; @@ -209,6 +214,7 @@ _drop_super_table: &pMeta->txn); tdbTbDelete(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pMeta->txn); tdbTbDelete(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), &pMeta->txn); + tdbTbDelete(pMeta->pSuidIdx, &pReq->suid, sizeof(tb_uid_t), &pMeta->txn); metaULock(pMeta); @@ -436,11 +442,13 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), &pMeta->txn); if(e.type != TSDB_SUPER_TABLE) metaDeleteTtlIdx(pMeta, &e); + if (e.type == TSDB_CHILD_TABLE) { tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), &pMeta->txn); } else if (e.type == TSDB_NORMAL_TABLE) { // drop schema.db (todo) } else if (e.type == TSDB_SUPER_TABLE) { + tdbTbDelete(pMeta->pSuidIdx, &e.uid, sizeof(tb_uid_t), &pMeta->txn); // drop schema.db (todo) } @@ -911,6 +919,10 @@ static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { return tdbTbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); } +static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME) { + return tdbTbInsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, &pMeta->txn); +} + static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); } @@ -1081,6 +1093,10 @@ static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) { } else { // update schema.db if (metaSaveToSkmDb(pMeta, pME) < 0) goto _err; + + if (pME->type == TSDB_SUPER_TABLE) { + if (metaUpdateSuidIdx(pMeta, pME) < 0) goto _err; + } } if (pME->type != TSDB_SUPER_TABLE) { diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 4e907e93ca..7f115633b9 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -126,22 +126,21 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS } if (smaType == TSDB_SMA_TYPE_ROLLUP) { - SMA_RSMA_STAT(*pSmaStat)->pSma = (SSma*)pSma; + SRSmaStat *pRSmaStat = (SRSmaStat *)(*pSmaStat); + pRSmaStat->pSma = (SSma *)pSma; // init timer - SMA_RSMA_TMR_HANDLE(*pSmaStat) = taosTmrInit(10000, 100, 10000, "RSMA_G"); - if (!SMA_RSMA_TMR_HANDLE(*pSmaStat)) { + RSMA_TMR_HANDLE(pRSmaStat) = taosTmrInit(10000, 100, 10000, "RSMA"); + if (!RSMA_TMR_HANDLE(pRSmaStat)) { taosMemoryFreeClear(*pSmaStat); return TSDB_CODE_FAILED; } - - atomic_store_8(&SMA_RSMA_TMR_STAT(*pSmaStat), TASK_TRIGGER_STATUS__ACTIVE); // init hash - SMA_RSMA_INFO_HASH(*pSmaStat) = taosHashInit( + RSMA_INFO_HASH(pRSmaStat) = taosHashInit( RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); - if (!SMA_RSMA_INFO_HASH(*pSmaStat)) { - if (SMA_RSMA_TMR_HANDLE(*pSmaStat)) { - taosTmrCleanUp(SMA_RSMA_TMR_HANDLE(*pSmaStat)); + if (!RSMA_INFO_HASH(pRSmaStat)) { + if (RSMA_TMR_HANDLE(pRSmaStat)) { + taosTmrCleanUp(RSMA_TMR_HANDLE(pRSmaStat)); } taosMemoryFreeClear(*pSmaStat); return TSDB_CODE_FAILED; @@ -155,12 +154,80 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS return TSDB_CODE_SUCCESS; } -static void *tdFreeTSmaStat(STSmaStat *pStat) { +static void tdDestroyTSmaStat(STSmaStat *pStat) { if (pStat) { + smaDebug("destroy tsma stat"); tDestroyTSma(pStat->pTSma); taosMemoryFreeClear(pStat->pTSma); - taosMemoryFreeClear(pStat); + taosMemoryFreeClear(pStat->pTSchema); } +} + +static void *tdFreeTSmaStat(STSmaStat *pStat) { + tdDestroyTSmaStat(pStat); + taosMemoryFreeClear(pStat); + return NULL; +} + +static void tdDestroyRSmaStat(SRSmaStat *pStat) { + if (pStat) { + smaDebug("vgId:%d destroy rsma stat", SMA_VID(pStat->pSma)); + // step 1: set persistence task cancelled + atomic_store_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_CANCELLED); + + // step 2: stop the persistence timer + taosTmrStopA(&RSMA_TMR_ID(pStat)); + + // step 3: wait the persistence thread to finish + int32_t nLoops = 0; + if (atomic_load_8(RSMA_RUNNING_STAT(pStat)) == 1) { + while (1) { + if (atomic_load_8(RSMA_TRIGGER_STAT(pStat)) == TASK_TRIGGER_STAT_FINISHED) { + break; + } else { + smaDebug("not destroyed since rsma stat in %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pStat))); + } + ++nLoops; + if (nLoops > 1000) { + sched_yield(); + nLoops = 0; + } + } + } + + // step 4: destroy the rsma info and associated fetch tasks + // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. + void *infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), NULL); + while (infoHash) { + SRSmaInfo *pSmaInfo = *(SRSmaInfo **)infoHash; + tdFreeRSmaInfo(pSmaInfo); + infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), infoHash); + } + taosHashCleanup(RSMA_INFO_HASH(pStat)); + + // step 5: wait all triggered fetch tasks finished + nLoops = 0; + while (1) { + if (T_REF_VAL_GET((SSmaStat *)pStat) == 0) { + break; + } + ++nLoops; + if (nLoops > 1000) { + sched_yield(); + nLoops = 0; + } + } + + // step 6: cleanup the timer handle + if (RSMA_TMR_HANDLE(pStat)) { + taosTmrCleanUp(RSMA_TMR_HANDLE(pStat)); + } + } +} + +static void *tdFreeRSmaStat(SRSmaStat *pStat) { + tdDestroyRSmaStat(pStat); + taosMemoryFreeClear(pStat); return NULL; } @@ -179,19 +246,9 @@ void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) { int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { if (pSmaStat) { if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { - tdFreeTSmaStat(&pSmaStat->tsmaStat); + tdDestroyTSmaStat(SMA_TSMA_STAT(pSmaStat)); } else if (smaType == TSDB_SMA_TYPE_ROLLUP) { - if (SMA_RSMA_TMR_HANDLE(pSmaStat)) { - taosTmrCleanUp(SMA_RSMA_TMR_HANDLE(pSmaStat)); - } - // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. - void *infoHash = taosHashIterate(SMA_RSMA_INFO_HASH(pSmaStat), NULL); - while (infoHash) { - SRSmaInfo *pInfoHash = *(SRSmaInfo **)infoHash; - tdFreeRSmaInfo(pInfoHash); - infoHash = taosHashIterate(SMA_RSMA_INFO_HASH(pSmaStat), infoHash); - } - taosHashCleanup(SMA_RSMA_INFO_HASH(pSmaStat)); + tdDestroyRSmaStat(SMA_RSMA_STAT(pSmaStat)); } else { ASSERT(0); } @@ -259,35 +316,4 @@ int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) { tdUnLockSma(pSma); return TSDB_CODE_SUCCESS; -}; - -int32_t smaTimerInit(void **timer, int8_t *initFlag, const char *label) { - int8_t old; - while (1) { - old = atomic_val_compare_exchange_8(initFlag, 0, 2); - if (old != 2) break; - } - - if (old == 0) { - *timer = taosTmrInit(10000, 100, 10000, label); - if (!(*timer)) { - atomic_store_8(initFlag, 0); - return -1; - } - atomic_store_8(initFlag, 1); - } - return 0; -} - -void smaTimerCleanUp(void *timer, int8_t *initFlag) { - int8_t old; - while (1) { - old = atomic_val_compare_exchange_8(initFlag, 1, 2); - if (old != 2) break; - } - - if (old == 1) { - taosTmrCleanUp(timer); - atomic_store_8(initFlag, 0); - } -} \ No newline at end of file +}; \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index 383c264e7b..88ed7426f7 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -18,6 +18,7 @@ static int32_t smaEvalDays(SRetention *r, int8_t precision); static int32_t smaSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type); +static int32_t rsmaRestore(SSma *pSma); #define SMA_SET_KEEP_CFG(l) \ do { \ @@ -100,6 +101,9 @@ int32_t smaOpen(SVnode *pVnode) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + + pVnode->pSma = pSma; + pSma->pVnode = pVnode; taosThreadMutexInit(&pSma->mutex, NULL); pSma->locked = false; @@ -117,37 +121,53 @@ int32_t smaOpen(SVnode *pVnode) { ASSERT(0); } } + + // restore the rsma +#if 0 + if (rsmaRestore(pSma) < 0) { + goto _err; + } +#endif } - pVnode->pSma = pSma; return 0; _err: - taosMemoryFreeClear(pSma); return -1; } -int32_t smaClose(SSma *pSma) { +int32_t smaCloseEnv(SSma *pSma) { + if (pSma) { + SMA_TSMA_ENV(pSma) = tdFreeSmaEnv(SMA_TSMA_ENV(pSma)); + SMA_RSMA_ENV(pSma) = tdFreeSmaEnv(SMA_RSMA_ENV(pSma)); + } + return 0; +} + +int32_t smaCloseEx(SSma *pSma) { if (pSma) { taosThreadMutexDestroy(&pSma->mutex); if SMA_RSMA_TSDB0 (pSma) tsdbClose(&SMA_RSMA_TSDB0(pSma)); if SMA_RSMA_TSDB1 (pSma) tsdbClose(&SMA_RSMA_TSDB1(pSma)); if SMA_RSMA_TSDB2 (pSma) tsdbClose(&SMA_RSMA_TSDB2(pSma)); - // SMA_TSMA_ENV(pSma) = tdFreeSmaEnv(SMA_TSMA_ENV(pSma)); - // SMA_RSMA_ENV(pSma) = tdFreeSmaEnv(SMA_RSMA_ENV(pSma)); taosMemoryFreeClear(pSma); } return 0; } +int32_t smaClose(SSma *pSma) { + smaCloseEnv(pSma); + smaCloseEx(pSma); + return 0; +} + /** * @brief rsma env restore - * - * @param pSma - * @return int32_t + * + * @param pSma + * @return int32_t */ -int32_t smaRestore(SSma *pSma) { - if (!pSma) return 0; - // iterate all stables to restore the rsma env - - return TSDB_CODE_SUCCESS; +static int32_t rsmaRestore(SSma *pSma) { + ASSERT(VND_IS_RSMA(pSma->pVnode)); + + return tdProcessRSmaRestoreImpl(pSma); } \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 3c2f710fa7..ed5b6f4055 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -14,11 +14,15 @@ */ #include "sma.h" -#include "tstream.h" +#define RSMA_QTASKINFO_PERSIST_MS 7200000 +#define RSMA_QTASKINFO_BUFSIZE 32768 typedef enum { TD_QTASK_TMP_FILE = 0, TD_QTASK_CUR_FILE } TD_QTASK_FILE_T; static const char *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"}; +typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem; +typedef struct SRSmaQTaskFIter SRSmaQTaskFIter; + static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids); static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo *pRSmaInfo, SReadHandle *handle, @@ -27,17 +31,24 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType tb_uid_t suid, int8_t level); static void tdRSmaFetchTrigger(void *param, void *tmrId); static void tdRSmaPersistTrigger(void *param, void *tmrId); +static void *tdRSmaPersistExec(void *param); +static void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char *outputName); + +static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskFIter *pIter, STFile *pTFile); +static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskFIter *pIter, bool *isFinish); +static int32_t tdRSmaQTaskInfoIterNext(SRSmaQTaskFIter *pIter, SRSmaQTaskInfoItem *pItem, bool *isEnd); +static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *infoItem); struct SRSmaInfoItem { SRSmaInfo *pRsmaInfo; void *taskInfo; // qTaskInfo_t - void *tmrHandle; tmr_h tmrId; int8_t level; int8_t tmrInitFlag; - int8_t triggerStatus; // TASK_TRIGGER_STATUS__IN_ACTIVE/TASK_TRIGGER_STATUS__ACTIVE + int8_t triggerStat; int32_t maxDelay; }; + struct SRSmaInfo { STSchema *pTSchema; SSma *pSma; @@ -45,11 +56,38 @@ struct SRSmaInfo { SRSmaInfoItem items[TSDB_RETENTION_L2]; }; -static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle) { +struct SRSmaQTaskInfoItem { + int32_t len; + int8_t type; + int64_t suid; + void *qTaskInfo; +}; + +struct SRSmaQTaskFIter { + STFile *pTFile; + int64_t offset; + int64_t fsize; + int32_t nBytes; + int32_t nAlloc; + char *buf; + // ------------ + int32_t nBufPos; +}; + +static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) { + return lenWithHead - sizeof(int32_t) - sizeof(int8_t) - sizeof(int64_t); +} + +static FORCE_INLINE void tdRSmaQTaskInfoIterDestroy(SRSmaQTaskFIter *pIter) { taosMemoryFreeClear(pIter->buf); } + +static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) { // Note: free/kill may in RC qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { + smaDebug("vgId:%d, free qTaskInfo_t %p of level %d", vgId, otaskHandle, level); qDestroyTask(otaskHandle); + } else { + smaDebug("vgId:%d, not free qTaskInfo_t %p of level %d", vgId, otaskHandle, level); } } @@ -58,14 +96,19 @@ void *tdFreeRSmaInfo(SRSmaInfo *pInfo) { for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { SRSmaInfoItem *pItem = &pInfo->items[i]; if (pItem->taskInfo) { - tdFreeTaskHandle(pItem->taskInfo); - } - if (pItem->tmrHandle) { - taosTmrCleanUp(pItem->tmrHandle); + smaDebug("vgId:%d, stb %" PRIi64 " stop fetch-timer %p level %d", SMA_VID(pInfo->pSma), pInfo->suid, + pItem->tmrId, i + 1); + taosTmrStopA(&pItem->tmrId); + tdFreeTaskHandle(&pItem->taskInfo, SMA_VID(pInfo->pSma), i + 1); + } else { + smaDebug("vgId:%d, stb %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", + SMA_VID(pInfo->pSma), pInfo->suid, i + 1); } } taosMemoryFree(pInfo->pTSchema); taosMemoryFree(pInfo); + } else { + smaDebug("vgId:%d, stb %" PRIi64 " no need to destroy rsma info since empty", SMA_VID(pInfo->pSma), pInfo->suid); } return NULL; @@ -81,9 +124,9 @@ static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) { return TSDB_CODE_SUCCESS; } -static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) { +static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) { SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; if (!suid || !tbUids) { @@ -92,27 +135,31 @@ static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SA return TSDB_CODE_FAILED; } - pRSmaInfo = taosHashGet(SMA_RSMA_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); + pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid); terrno = TSDB_CODE_RSMA_INVALID_STAT; return TSDB_CODE_FAILED; } - if (pRSmaInfo->items[0].taskInfo && (qUpdateQualifiedTableId(pRSmaInfo->items[0].taskInfo, tbUids, true) < 0)) { - smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); - return TSDB_CODE_FAILED; - } else { - smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), - pRSmaInfo->items[0].taskInfo, *suid, *(int64_t *)taosArrayGet(tbUids, 0)); + if (pRSmaInfo->items[0].taskInfo) { + if ((qUpdateQualifiedTableId(pRSmaInfo->items[0].taskInfo, tbUids, true) < 0)) { + smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); + return TSDB_CODE_FAILED; + } else { + smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), + pRSmaInfo->items[0].taskInfo, *suid, *(int64_t *)taosArrayGet(tbUids, 0)); + } } - if (pRSmaInfo->items[1].taskInfo && (qUpdateQualifiedTableId(pRSmaInfo->items[1].taskInfo, tbUids, true) < 0)) { - smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); - return TSDB_CODE_FAILED; - } else { - smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), - pRSmaInfo->items[1].taskInfo, *suid, *(int64_t *)taosArrayGet(tbUids, 0)); + if (pRSmaInfo->items[1].taskInfo) { + if ((qUpdateQualifiedTableId(pRSmaInfo->items[1].taskInfo, tbUids, true) < 0)) { + smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); + return TSDB_CODE_FAILED; + } else { + smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), + pRSmaInfo->items[1].taskInfo, *suid, *(int64_t *)taosArrayGet(tbUids, 0)); + } } return TSDB_CODE_SUCCESS; @@ -159,9 +206,9 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui return TSDB_CODE_SUCCESS; } - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SHashObj *infoHash = NULL; - if (!pStat || !(infoHash = SMA_RSMA_INFO_HASH(pStat))) { + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + SHashObj *infoHash = NULL; + if (!pStat || !(infoHash = RSMA_INFO_HASH(pStat))) { terrno = TSDB_CODE_RSMA_INVALID_STAT; return TSDB_CODE_FAILED; } @@ -195,11 +242,11 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo if (param->qmsg[idx]) { SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]); pItem->pRsmaInfo = pRSmaInfo; - pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[0], pReadHandle); + pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], pReadHandle); if (!pItem->taskInfo) { goto _err; } - pItem->triggerStatus = TASK_TRIGGER_STATUS__IN_ACTIVE; + pItem->triggerStat = TASK_TRIGGER_STAT_INACTIVE; if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) { int64_t msInterval = convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND); @@ -211,10 +258,6 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo pItem->maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY; } pItem->level = (idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2); - pItem->tmrHandle = taosTmrInit(10000, 100, 10000, "RSMA"); - if (!pItem->tmrHandle) { - goto _err; - } } return TSDB_CODE_SUCCESS; _err: @@ -222,26 +265,21 @@ _err: } /** - * @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam. + * @brief for rsam create or restore * - * @param pTsdb - * @param pMeta - * @param pReq + * @param pSma + * @param param + * @param suid + * @param tbName * @return int32_t */ -int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { - SSma *pSma = pVnode->pSma; - if (!pReq->rollup) { - smaTrace("vgId:%d, return directly since no rollup for stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); - return TSDB_CODE_SUCCESS; - } - - SMeta *pMeta = pVnode->pMeta; - SMsgCb *pMsgCb = &pVnode->msgCb; - SRSmaParam *param = &pReq->pRSmaParam; +int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName) { + SVnode *pVnode = pSma->pVnode; + SMeta *pMeta = pVnode->pMeta; + SMsgCb *pMsgCb = &pVnode->msgCb; if ((param->qmsgLen[0] == 0) && (param->qmsgLen[1] == 0)) { - smaWarn("vgId:%d, no qmsg1/qmsg2 for rollup stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); + smaDebug("vgId:%d, no qmsg1/qmsg2 for rollup table %s %" PRIi64, SMA_VID(pSma), tbName, suid); return TSDB_CODE_SUCCESS; } @@ -251,13 +289,13 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { } SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; - pRSmaInfo = taosHashGet(SMA_RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t)); + pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (pRSmaInfo) { ASSERT(0); // TODO: free original pRSmaInfo is exists abnormally - smaWarn("vgId:%d, rsma info already exists for stb: %s, %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); + smaDebug("vgId:%d, rsma info already exists for table %s, %" PRIi64, SMA_VID(pSma), tbName, suid); return TSDB_CODE_SUCCESS; } @@ -281,26 +319,33 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { .vnode = pVnode, }; - STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), pReq->suid, -1); + STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), suid, -1); if (!pTSchema) { terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; goto _err; } pRSmaInfo->pTSchema = pTSchema; pRSmaInfo->pSma = pSma; - pRSmaInfo->suid = pReq->suid; + pRSmaInfo->suid = suid; if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 0) < 0) { goto _err; } + if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 1) < 0) { goto _err; } - if (taosHashPut(SMA_RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) { + if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) { goto _err; } else { - smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), pReq->suid); + smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), suid); + } + + // start the persist timer + if (TASK_TRIGGER_STAT_INIT == + atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_INIT, TASK_TRIGGER_STAT_ACTIVE)) { + taosTmrStart(tdRSmaPersistTrigger, RSMA_QTASKINFO_PERSIST_MS, pStat, RSMA_TMR_HANDLE(pStat)); } return TSDB_CODE_SUCCESS; @@ -310,6 +355,24 @@ _err: return TSDB_CODE_FAILED; } +/** + * @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam. + * + * @param pTsdb + * @param pMeta + * @param pReq + * @return int32_t + */ +int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { + SSma *pSma = pVnode->pSma; + if (!pReq->rollup) { + smaTrace("vgId:%d, return directly since no rollup for stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); + return TSDB_CODE_SUCCESS; + } + + return tdProcessRSmaCreateImpl(pSma, &pReq->rsmaParam, pReq->suid, pReq->name); +} + /** * @brief store suid/[uids], prefer to use array and then hash * @@ -432,6 +495,16 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { return 0; } +static void tdDestroySDataBlockArray(SArray *pArray) { +#if 0 + for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { + SSDataBlock *pDataBlock = taosArrayGet(pArray, i); + blockDestroyInner(pDataBlock); + } +#endif + taosArrayDestroy(pArray); +} + static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) { SArray *pResult = NULL; SRSmaInfo *pRSmaInfo = pItem->pRsmaInfo; @@ -462,10 +535,11 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) #if 1 char flag[10] = {0}; snprintf(flag, 10, "level %" PRIi8, pItem->level); - blockDebugShowData(pResult, flag); + blockDebugShowDataBlocks(pResult, flag); #endif STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2); SSubmitReq *pReq = NULL; + // TODO: the schema update should be handled if (buildSubmitReqFromDataBlock(&pReq, pResult, pRSmaInfo->pTSchema, SMA_VID(pSma), pRSmaInfo->suid) < 0) { goto _err; } @@ -476,18 +550,16 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) } taosMemoryFreeClear(pReq); + } else if (terrno == 0) { + smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched yet", SMA_VID(pSma), pItem->level); } else { - smaDebug("vgId:%d, no rsma % " PRIi8 " data generated since %s", SMA_VID(pSma), pItem->level, tstrerror(terrno)); + smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched since %s", SMA_VID(pSma), pItem->level, tstrerror(terrno)); } - if (blkType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { - atomic_store_8(&pItem->triggerStatus, TASK_TRIGGER_STATUS__ACTIVE); - } - - taosArrayDestroy(pResult); + tdDestroySDataBlockArray(pResult); return TSDB_CODE_SUCCESS; _err: - taosArrayDestroy(pResult); + tdDestroySDataBlockArray(pResult); return TSDB_CODE_FAILED; } @@ -499,26 +571,38 @@ _err: */ static void tdRSmaFetchTrigger(void *param, void *tmrId) { SRSmaInfoItem *pItem = param; + SSma *pSma = pItem->pRsmaInfo->pSma; + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)pSma->pRSmaEnv); - if (atomic_load_8(&pItem->triggerStatus) == TASK_TRIGGER_STATUS__ACTIVE) { - smaWarn("%s:%d THREAD:%" PRIi64 " level %" PRIi8 " status is active for tb suid:%" PRIi64, __func__, __LINE__, - taosGetSelfPthreadId(), pItem->level, pItem->pRsmaInfo->suid); - SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; - - atomic_store_8(&pItem->triggerStatus, TASK_TRIGGER_STATUS__IN_ACTIVE); - qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_DATA_TYPE_SSDATA_BLOCK, false); - - tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SSDATA_BLOCK); - } else { - smaWarn("%s:%d THREAD:%" PRIi64 " level %" PRIi8 " status is inactive for tb suid:%" PRIi64, __func__, __LINE__, - taosGetSelfPthreadId(), pItem->level, pItem->pRsmaInfo->suid); + int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat)); + if (rsmaTriggerStat == TASK_TRIGGER_STAT_CANCELLED || rsmaTriggerStat == TASK_TRIGGER_STAT_FINISHED) { + smaDebug("vgId:%d, level %" PRIi8 " not fetch since stat is cancelled for table suid:%" PRIi64, SMA_VID(pSma), + pItem->level, pItem->pRsmaInfo->suid); + return; } - // taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pItem->tmrHandle, &pItem->tmrId); + int8_t fetchTriggerStat = + atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE); + if (fetchTriggerStat == TASK_TRIGGER_STAT_ACTIVE) { + smaDebug("vgId:%d, level %" PRIi8 " stat is active for table suid:%" PRIi64, SMA_VID(pSma), pItem->level, + pItem->pRsmaInfo->suid); + + tdRefSmaStat(pSma, (SSmaStat *)pStat); + + SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; + qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_DATA_TYPE_SSDATA_BLOCK, false); + tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SSDATA_BLOCK); + + tdUnRefSmaStat(pSma, (SSmaStat *)pStat); + + } else { + smaDebug("vgId:%d, level %" PRIi8 " stat is inactive for table suid:%" PRIi64, SMA_VID(pSma), pItem->level, + pItem->pRsmaInfo->suid); + } } -static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, - tb_uid_t suid, int8_t level) { +static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, tb_uid_t suid, + int8_t level) { if (!pItem || !pItem->taskInfo) { smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid); return TSDB_CODE_SUCCESS; @@ -533,14 +617,17 @@ static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int3 } tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SUBMIT_BLOCK); - atomic_store_8(&pItem->triggerStatus, TASK_TRIGGER_STATUS__ACTIVE); - smaWarn("%s:%d THREAD:%" PRIi64 " process rsma insert", __func__, __LINE__, taosGetSelfPthreadId()); + atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE); + smaDebug("vgId:%d, process rsma insert", SMA_VID(pSma)); SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pStat = SMA_RSMA_STAT(pEnv->pStat); - taosTmrStart(tdRSmaPersistTrigger, 5000, pStat, pStat->tmrHandle); - taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pItem->tmrHandle, &pItem->tmrId); + if (pStat->tmrHandle) { + taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pStat->tmrHandle, &pItem->tmrId); + } else { + ASSERT(0); + } return TSDB_CODE_SUCCESS; } @@ -552,10 +639,10 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb return TSDB_CODE_SUCCESS; } - SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; - pRSmaInfo = taosHashGet(SMA_RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); + pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { smaDebug("vgId:%d, return as no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); @@ -608,7 +695,7 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { return TSDB_CODE_SUCCESS; } -void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char* outputName) { +static void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char *outputName) { tdGetVndFileName(vid, "rsma", tdQTaskInfoFname[ftype], outputName); } @@ -618,6 +705,11 @@ static void *tdRSmaPersistExec(void *param) { SSma *pSma = pRSmaStat->pSma; STfs *pTfs = pSma->pVnode->pTfs; int64_t toffset = 0; + bool isFileCreated = false; + + if (TASK_TRIGGER_STAT_CANCELLED == atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))) { + goto _end; + } void *infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL); if (!infoHash) { @@ -625,67 +717,143 @@ static void *tdRSmaPersistExec(void *param) { } STFile tFile = {0}; - int32_t vid = 2; - char qTaskInfoFName[TSDB_FILENAME_LEN]; - tdRSmaQTaskGetFName(vid, TD_QTASK_TMP_FILE, qTaskInfoFName); - tdInitTFile(&tFile, pTfs, qTaskInfoFName); - tdCreateTFile(&tFile, pTfs, true, -1); + int32_t vid = SMA_VID(pSma); while (infoHash) { SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; - char *pOutput = NULL; - int32_t len = 0; - if (qSerializeTaskStatus(pRSmaInfo->items[0].taskInfo, &pOutput, &len) < 0) { - smaError("serialize rsma task for table %" PRIi64 " failed since %s", pRSmaInfo->items[0].pRsmaInfo->suid, - terrstr(terrno)); - } else { - smaWarn("serialize rsma task for table %" PRIi64 " success and len is %d", pRSmaInfo->items[0].pRsmaInfo->suid, - len); - } - tdAppendTFile(&tFile, &len, sizeof(len), &toffset); - tdAppendTFile(&tFile, pOutput, len, &toffset); - taosMemoryFree(pOutput); +#if 0 + smaDebug("table %" PRIi64 " sleep 15s start ...", pRSmaInfo->items[0].pRsmaInfo->suid); + for (int32_t i = 15; i > 0; --i) { + taosSsleep(1); + smaDebug("table %" PRIi64 " countdown %d", pRSmaInfo->items[0].pRsmaInfo->suid, i); + } + smaDebug("table %" PRIi64 " sleep 15s end ...", pRSmaInfo->items[0].pRsmaInfo->suid); +#endif + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { + qTaskInfo_t taskInfo = pRSmaInfo->items[i].taskInfo; + if (!taskInfo) { + smaDebug("vgId:%d, table %" PRIi64 " level %d qTaskInfo is NULL", vid, pRSmaInfo->suid, i + 1); + continue; + } + char *pOutput = NULL; + int32_t len = 0; + int8_t type = (int8_t)(i + 1); + if (qSerializeTaskStatus(taskInfo, &pOutput, &len) < 0) { + smaError("vgId:%d, table %" PRIi64 " level %d serialize rsma task failed since %s", vid, pRSmaInfo->suid, i + 1, + terrstr(terrno)); + goto _err; + } else { + if (!pOutput) { + smaDebug("vgId:%d, table %" PRIi64 + " level %d serialize rsma task success but no output(len %d) and no need to persist", + vid, pRSmaInfo->suid, i + 1, len); + continue; + } else if (len <= 0) { + smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success with len %d and no need to persist", + vid, pRSmaInfo->suid, i + 1, len); + taosMemoryFree(pOutput); + } + smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success with len %d and need persist", vid, + pRSmaInfo->suid, i + 1, len); +#if 1 + if (qDeserializeTaskStatus(taskInfo, pOutput, len) < 0) { + smaError("vgId:%d, table %" PRIi64 "level %d deserialize rsma task failed since %s", vid, pRSmaInfo->suid, + i + 1, terrstr(terrno)); + } else { + smaDebug("vgId:%d, table %" PRIi64 " level %d deserialize rsma task success", vid, pRSmaInfo->suid, i + 1); + } +#endif + } + + if (!isFileCreated) { + char qTaskInfoFName[TSDB_FILENAME_LEN]; + tdRSmaQTaskGetFName(vid, TD_QTASK_TMP_FILE, qTaskInfoFName); + tdInitTFile(&tFile, pTfs, qTaskInfoFName); + tdCreateTFile(&tFile, pTfs, true, -1); + + isFileCreated = true; + } + len += (sizeof(len) + sizeof(type) + sizeof(pRSmaInfo->suid)); + tdAppendTFile(&tFile, &len, sizeof(len), &toffset); + tdAppendTFile(&tFile, &type, sizeof(type), &toffset); + tdAppendTFile(&tFile, &pRSmaInfo->suid, sizeof(pRSmaInfo->suid), &toffset); + tdAppendTFile(&tFile, pOutput, len, &toffset); + + taosMemoryFree(pOutput); + } infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), infoHash); } -_end: +_normal: + if (isFileCreated) { + if (tdUpdateTFileHeader(&tFile) < 0) { + smaError("vgId:%d, failed to update tfile %s header since %s", vid, TD_FILE_FULL_NAME(&tFile), tstrerror(terrno)); + tdCloseTFile(&tFile); + tdRemoveTFile(&tFile); + goto _err; + } else { + smaDebug("vgId:%d, succeed to update tfile %s header", vid, TD_FILE_FULL_NAME(&tFile)); + } - if (tdUpdateTFileHeader(&tFile) < 0) { - smaError("vgId:%d, failed to update tfile %s header since %s", vid, TD_FILE_FULL_NAME(&tFile), tstrerror(terrno)); tdCloseTFile(&tFile); - tdRemoveTFile(&tFile); - return NULL; + + char newFName[TSDB_FILENAME_LEN]; + strncpy(newFName, TD_FILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN); + char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_FILE]); + strncpy(pos, tdQTaskInfoFname[TD_QTASK_CUR_FILE], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName)); + if (taosRenameFile(TD_FILE_FULL_NAME(&tFile), newFName) != 0) { + smaError("vgId:%d, failed to rename %s to %s", vid, TD_FILE_FULL_NAME(&tFile), newFName); + goto _err; + } else { + smaDebug("vgId:%d, succeed to rename %s to %s", vid, TD_FILE_FULL_NAME(&tFile), newFName); + } } - - tdCloseTFile(&tFile); - - char newFName[TSDB_FILENAME_LEN]; - strncpy(newFName, TD_FILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN); - char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_FILE]); - strncpy(pos, tdQTaskInfoFname[TD_QTASK_CUR_FILE], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName)); - taosRenameFile(TD_FILE_FULL_NAME(&tFile), newFName); - - atomic_store_8(&pRSmaStat->tmrStat, TASK_TRIGGER_STATUS__ACTIVE); - return NULL; + goto _end; _err: - atomic_store_8(&pRSmaStat->tmrStat, TASK_TRIGGER_STATUS__ACTIVE); - // remove the .tmp file + if (isFileCreated) { + tdRemoveTFile(&tFile); + } +_end: + if (TASK_TRIGGER_STAT_INACTIVE == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), + TASK_TRIGGER_STAT_INACTIVE, + TASK_TRIGGER_STAT_ACTIVE)) { + smaDebug("vgId:%d, persist task is active again", vid); + } else if (TASK_TRIGGER_STAT_CANCELLED == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), + TASK_TRIGGER_STAT_CANCELLED, + TASK_TRIGGER_STAT_FINISHED)) { + smaDebug("vgId:%d, persist task is cancelled", vid); + } else { + smaWarn("vgId:%d, persist task in abnormal stat %" PRIi8, vid, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))); + ASSERT(0); + } + atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); + taosThreadExit(NULL); return NULL; } static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) { - smaWarn("%s:%d entry ", __func__, __LINE__); - TdThread threadId; TdThreadAttr thAttr; taosThreadAttrInit(&thAttr); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_DETACHED); + TdThread tid; - if (taosThreadCreate(&threadId, &thAttr, tdRSmaPersistExec, pRSmaStat) != 0) { - smaError("failed to create thread to persist rsma qTaskInfo since %s", strerror(errno)); + if (taosThreadCreate(&tid, &thAttr, tdRSmaPersistExec, pRSmaStat) != 0) { + if (TASK_TRIGGER_STAT_INACTIVE == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), + TASK_TRIGGER_STAT_INACTIVE, + TASK_TRIGGER_STAT_ACTIVE)) { + smaDebug("persist task is active again"); + } else if (TASK_TRIGGER_STAT_CANCELLED == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), + TASK_TRIGGER_STAT_CANCELLED, + TASK_TRIGGER_STAT_FINISHED)) { + smaDebug(" persist task is cancelled and set finished"); + } else { + smaWarn("persist task in abnormal stat %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))); + ASSERT(0); + } + atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); } taosThreadAttrDestroy(&thAttr); - smaWarn("%s:%d end ", __func__, __LINE__); } /** @@ -696,17 +864,283 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) { */ static void tdRSmaPersistTrigger(void *param, void *tmrId) { SRSmaStat *pRSmaStat = param; + int8_t tmrStat = + atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE); + switch (tmrStat) { + case TASK_TRIGGER_STAT_ACTIVE: { + atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 1); + if (TASK_TRIGGER_STAT_CANCELLED != atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), + TASK_TRIGGER_STAT_CANCELLED, + TASK_TRIGGER_STAT_FINISHED)) { + smaDebug("rsma persistence start since active"); - if (atomic_load_8(&pRSmaStat->tmrStat) == TASK_TRIGGER_STATUS__ACTIVE) { - smaWarn("%s:%d THREAD:%" PRIi64 " rsma persistence start since active", __func__, __LINE__, taosGetSelfPthreadId()); - atomic_store_8(&pRSmaStat->tmrStat, TASK_TRIGGER_STATUS__IN_ACTIVE); + // start persist task + tdRSmaPersistTask(pRSmaStat); - // execution - tdRSmaPersistTask(pRSmaStat); - } else { - smaWarn("%s:%d THREAD:%" PRIi64 " rsma persistence not start since inactive", __func__, __LINE__, - taosGetSelfPthreadId()); + taosTmrReset(tdRSmaPersistTrigger, RSMA_QTASKINFO_PERSIST_MS, pRSmaStat, pRSmaStat->tmrHandle, + &pRSmaStat->tmrId); + } else { + atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); + } + } break; + case TASK_TRIGGER_STAT_CANCELLED: { + atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_FINISHED); + smaDebug("rsma persistence not start since cancelled and finished"); + } break; + case TASK_TRIGGER_STAT_INACTIVE: { + smaDebug("rsma persistence not start since inactive"); + } break; + case TASK_TRIGGER_STAT_INIT: { + smaDebug("rsma persistence not start since init"); + } break; + default: { + smaWarn("rsma persistence not start since unknown stat %" PRIi8, tmrStat); + ASSERT(0); + } break; + } +} + +int32_t tdProcessRSmaRestoreImpl(SSma *pSma) { + SVnode *pVnode = pSma->pVnode; + + // step 1: iterate all stables to restore the rsma env + + SArray *suidList = taosArrayInit(1, sizeof(tb_uid_t)); + if (tsdbGetStbIdList(SMA_META(pSma), 0, suidList) < 0) { + smaError("vgId:%d, failed to restore rsma since get stb id list error: %s", TD_VID(pVnode), terrstr()); + return TSDB_CODE_FAILED; } - taosTmrReset(tdRSmaPersistTrigger, 3600000, pRSmaStat, pRSmaStat->tmrHandle, &pRSmaStat->tmrId); -} \ No newline at end of file + if (taosArrayGetSize(suidList) == 0) { + smaDebug("vgId:%d no need to restore rsma since empty stb id list", TD_VID(pVnode)); + return TSDB_CODE_SUCCESS; + } + + SMetaReader mr = {0}; + metaReaderInit(&mr, SMA_META(pSma), 0); + for (int32_t i = 0; i < taosArrayGetSize(suidList); ++i) { + tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); + smaDebug("suid [%d] is %" PRIi64, i, suid); + if (metaGetTableEntryByUid(&mr, suid) < 0) { + smaError("vgId:%d failed to get table meta for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr()); + goto _err; + } + ASSERT(mr.me.type == TSDB_SUPER_TABLE); + ASSERT(mr.me.uid == suid); + if (TABLE_IS_ROLLUP(mr.me.flags)) { + SRSmaParam *param = &mr.me.stbEntry.rsmaParam; + for (int i = 0; i < 2; ++i) { + smaDebug("vgId: %d table:%" PRIi64 " maxdelay[%d]:%" PRIi64 " watermark[%d]:%" PRIi64, TD_VID(pSma->pVnode), + suid, i, param->maxdelay[i], i, param->watermark[i]); + } + if (tdProcessRSmaCreateImpl(pSma, &mr.me.stbEntry.rsmaParam, suid, mr.me.name) < 0) { + smaError("vgId:%d failed to retore rsma env for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr()); + goto _err; + } + } + } + + // step 2: retrieve qtaskinfo object from the rsma/qtaskinfo file and restore + STFile tFile = {0}; + char qTaskInfoFName[TSDB_FILENAME_LEN]; + + tdRSmaQTaskGetFName(TD_VID(pVnode), TD_QTASK_CUR_FILE, qTaskInfoFName); + if (tdInitTFile(&tFile, pVnode->pTfs, qTaskInfoFName) < 0) { + goto _err; + } + if (tdOpenTFile(&tFile, TD_FILE_READ) < 0) { + goto _err; + } + SRSmaQTaskFIter fIter = {0}; + if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) { + goto _err; + } + SRSmaQTaskInfoItem infoItem = {0}; + bool isEnd = false; + int32_t code = 0; + while ((code = tdRSmaQTaskInfoIterNext(&fIter, &infoItem, &isEnd)) == 0) { + if (isEnd) { + break; + } + if ((code = tdRSmaQTaskInfoItemRestore(pSma, &infoItem)) < 0) { + break; + } + } + tdRSmaQTaskInfoIterDestroy(&fIter); + + if (code < 0) { + goto _err; + } + + metaReaderClear(&mr); + taosArrayDestroy(suidList); + return TSDB_CODE_SUCCESS; +_err: + ASSERT(0); + metaReaderClear(&mr); + taosArrayDestroy(suidList); + smaError("failed to restore rsma info since %s", terrstr()); + return TSDB_CODE_FAILED; +} + +static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *infoItem) { + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)pSma->pRSmaEnv); + SRSmaInfo *pRSmaInfo = NULL; + void *qTaskInfo = NULL; + + pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &infoItem->suid, sizeof(infoItem->suid)); + + if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { + smaDebug("vgId:%d, no restore as no rsma info for suid:%" PRIu64, SMA_VID(pSma), infoItem->suid); + return TSDB_CODE_SUCCESS; + } + + if (infoItem->type == 1) { + qTaskInfo = pRSmaInfo->items[0].taskInfo; + } else if (infoItem->type == 2) { + qTaskInfo = pRSmaInfo->items[1].taskInfo; + } else { + ASSERT(0); + } + + if (!qTaskInfo) { + smaDebug("vgId:%d, no restore as NULL rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), infoItem->suid); + return TSDB_CODE_SUCCESS; + } + + if (qDeserializeTaskStatus(qTaskInfo, infoItem->qTaskInfo, infoItem->len) < 0) { + smaError("vgId:%d, restore rsma failed for suid:%" PRIi64 " level %d since %s", SMA_VID(pSma), infoItem->suid, + infoItem->type, terrstr(terrno)); + return TSDB_CODE_FAILED; + } + smaDebug("vgId:%d, restore rsma success for suid:%" PRIi64 " level %d", SMA_VID(pSma), infoItem->suid, + infoItem->type); + + return TSDB_CODE_SUCCESS; +} + +static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskFIter *pIter, STFile *pTFile) { + memset(pIter, 0, sizeof(*pIter)); + pIter->pTFile = pTFile; + pIter->offset = TD_FILE_HEAD_SIZE; + + if (tdGetTFileSize(pTFile, &pIter->fsize) < 0) { + return TSDB_CODE_FAILED; + } + + if ((pIter->fsize - TD_FILE_HEAD_SIZE) < RSMA_QTASKINFO_BUFSIZE) { + pIter->nAlloc = pIter->fsize - TD_FILE_HEAD_SIZE; + } else { + pIter->nAlloc = RSMA_QTASKINFO_BUFSIZE; + } + + if (pIter->nAlloc < TD_FILE_HEAD_SIZE) { + pIter->nAlloc = TD_FILE_HEAD_SIZE; + } + + pIter->buf = taosMemoryMalloc(pIter->nAlloc); + if (!pIter->buf) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskFIter *pIter, bool *isFinish) { + STFile *pTFile = pIter->pTFile; + int64_t nBytes = RSMA_QTASKINFO_BUFSIZE; + + if (pIter->offset >= pIter->fsize) { + *isFinish = true; + return TSDB_CODE_SUCCESS; + } + + if ((pIter->fsize - pIter->offset) < RSMA_QTASKINFO_BUFSIZE) { + nBytes = pIter->fsize - pIter->offset; + } + + if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET) < 0) { + ASSERT(0); + return TSDB_CODE_FAILED; + } + + if (tdReadTFile(pTFile, pIter->buf, nBytes) != nBytes) { + ASSERT(0); + return TSDB_CODE_FAILED; + } + + int32_t infoLen = 0; + taosDecodeFixedI32(pIter->buf, &infoLen); + if (infoLen > nBytes) { + ASSERT(infoLen > RSMA_QTASKINFO_BUFSIZE); + pIter->nAlloc = infoLen; + void *pBuf = taosMemoryRealloc(pIter->buf, infoLen); + if (!pBuf) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + pIter->buf = pBuf; + nBytes = infoLen; + + if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET)) { + ASSERT(0); + return TSDB_CODE_FAILED; + } + + if (tdReadTFile(pTFile, pIter->buf, nBytes) != nBytes) { + ASSERT(0); + return TSDB_CODE_FAILED; + } + } + + pIter->offset += nBytes; + pIter->nBytes = nBytes; + pIter->nBufPos = 0; + + return TSDB_CODE_SUCCESS; +} + +static int32_t tdRSmaQTaskInfoIterNext(SRSmaQTaskFIter *pIter, SRSmaQTaskInfoItem *pItem, bool *isEnd) { + while (1) { + // block iter + bool isFinish = false; + if (tdRSmaQTaskInfoIterNextBlock(pIter, &isFinish) < 0) { + ASSERT(0); + return TSDB_CODE_FAILED; + } + if (isFinish) { + *isEnd = true; + return TSDB_CODE_SUCCESS; + } + + // consume the block + int32_t qTaskInfoLenWithHead = 0; + pIter->buf = taosDecodeFixedI32(pIter->buf, &qTaskInfoLenWithHead); + if (qTaskInfoLenWithHead < 0) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return TSDB_CODE_FAILED; + } + while (1) { + if ((pIter->nBufPos + qTaskInfoLenWithHead) <= pIter->nBytes) { + pIter->buf = taosDecodeFixedI8(pIter->buf, &pItem->type); + pIter->buf = taosDecodeFixedI64(pIter->buf, &pItem->suid); + pItem->qTaskInfo = pIter->buf; + pItem->len = tdRSmaQTaskInfoContLen(qTaskInfoLenWithHead); + // do the restore job + printf("%s:%d ###### restore the qtask info offset:%" PRIi64 "\n", __func__, __LINE__, pIter->offset); + + pIter->buf = POINTER_SHIFT(pIter->buf, pItem->len); + pIter->nBufPos += qTaskInfoLenWithHead; + + pIter->buf = taosDecodeFixedI32(pIter->buf, &qTaskInfoLenWithHead); + continue; + } + // prepare and load next block in the file + pIter->offset -= (pIter->nBytes - pIter->nBufPos); + break; + } + } + + return TSDB_CODE_SUCCESS; +} diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 76690a3bde..1f60da0b0a 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -15,7 +15,7 @@ #include "sma.h" -#define TD_FILE_HEAD_SIZE 512 +// smaFileUtil ================ #define TD_FILE_STATE_OK 0 #define TD_FILE_STATE_BAD 1 @@ -32,7 +32,7 @@ static int32_t tdEncodeTFInfo(void **buf, STFInfo *pInfo) { tlen += taosEncodeFixedU32(buf, pInfo->magic); tlen += taosEncodeFixedU32(buf, pInfo->ftype); tlen += taosEncodeFixedU32(buf, pInfo->fver); - tlen += taosEncodeFixedU64(buf, pInfo->fsize); + tlen += taosEncodeFixedI64(buf, pInfo->fsize); return tlen; } @@ -41,7 +41,7 @@ static void *tdDecodeTFInfo(void *buf, STFInfo *pInfo) { buf = taosDecodeFixedU32(buf, &(pInfo->magic)); buf = taosDecodeFixedU32(buf, &(pInfo->ftype)); buf = taosDecodeFixedU32(buf, &(pInfo->fver)); - buf = taosDecodeFixedU64(buf, &(pInfo->fsize)); + buf = taosDecodeFixedI64(buf, &(pInfo->fsize)); return buf; } @@ -69,6 +69,11 @@ int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence) { return loffset; } +int64_t tdGetTFileSize(STFile *pTFile, int64_t *size) { + ASSERT(TD_FILE_OPENED(pTFile)); + return taosFStatFile(pTFile->pFile, size, NULL); +} + int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte) { ASSERT(TD_FILE_OPENED(pTFile)); @@ -235,4 +240,7 @@ int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fTyp return 0; } -int32_t tdRemoveTFile(STFile *pTFile) { return tfsRemoveFile(TD_FILE_F(pTFile)); } \ No newline at end of file +int32_t tdRemoveTFile(STFile *pTFile) { return tfsRemoveFile(TD_FILE_F(pTFile)); } + +// smaXXXUtil ================ +// ... \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index ab31d65d68..03f18cc766 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2876,6 +2876,33 @@ int32_t tsdbGetCtbIdList(SMeta* pMeta, int64_t suid, SArray* list) { return TSDB_CODE_SUCCESS; } +/** + * @brief Get all suids since suid + * + * @param pMeta + * @param suid return all suids in one vnode if suid is 0 + * @param list + * @return int32_t + */ +int32_t tsdbGetStbIdList(SMeta* pMeta, int64_t suid, SArray* list) { + SMStbCursor* pCur = metaOpenStbCursor(pMeta, suid); + if(!pCur) { + return TSDB_CODE_FAILED; + } + + while (1) { + tb_uid_t id = metaStbCursorNext(pCur); + if (id == 0) { + break; + } + + taosArrayPush(list, &id); + } + + metaCloseStbCursor(pCur); + return TSDB_CODE_SUCCESS; +} + static void destroyHelper(void* param) { if (param == NULL) { return; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 3715866bb8..dbbe3d3750 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -72,7 +72,12 @@ int vnodeBegin(SVnode *pVnode) { return 0; } -int vnodeShouldCommit(SVnode *pVnode) { return pVnode->inUse->size > pVnode->config.szBuf / 3; } +int vnodeShouldCommit(SVnode *pVnode) { + if (pVnode->inUse) { + return pVnode->inUse->size > pVnode->config.szBuf / 3; + } + return false; +} int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { char fname[TSDB_FILENAME_LEN]; diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index dfc258b42b..4d73dbc406 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -152,12 +152,13 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { return pVnode; _err: + if (pVnode->pSma) smaClose(pVnode->pSma); if (pVnode->pQuery) vnodeQueryClose(pVnode); if (pVnode->pTq) tqClose(pVnode->pTq); if (pVnode->pWal) walClose(pVnode->pWal); if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb); if (pVnode->pMeta) metaClose(pVnode->pMeta); - if (pVnode->pSma) smaClose(pVnode->pSma); + tsem_destroy(&(pVnode->canCommit)); taosMemoryFree(pVnode); @@ -166,13 +167,14 @@ _err: void vnodeClose(SVnode *pVnode) { if (pVnode) { + smaCloseEnv(pVnode->pSma); vnodeCommit(pVnode); vnodeSyncClose(pVnode); vnodeQueryClose(pVnode); walClose(pVnode->pWal); tqClose(pVnode->pTq); if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb); - smaClose(pVnode->pSma); + smaCloseEx(pVnode->pSma); metaClose(pVnode->pMeta); vnodeCloseBufPool(pVnode); // destroy handle diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 88bdea3ae7..72c766e0ae 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -299,7 +299,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, int64_t version, SRpcMsg *pMsg, SRp void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { // TODO - blockDebugShowData(data, __func__); + blockDebugShowDataBlocks(data, __func__); tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data); } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 4c56180979..b998f40195 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -137,6 +137,25 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { vError("vgId:%d, failed to pre-process msg:%p since %s", vgId, pMsg, terrstr()); } else { code = syncPropose(pVnode->sync, pMsg, vnodeIsMsgWeak(pMsg->msgType)); + if (code == 1) { + do { + static int32_t cnt = 0; + if (cnt++ % 1000 == 1) { + vInfo("vgId:%d, msg:%p apply right now, apply index:%ld, msgtype:%s,%d", vgId, pMsg, + pMsg->info.conn.applyIndex, TMSG_INFO(pMsg->msgType), pMsg->msgType); + } + } while (0); + + SRpcMsg rsp = {.code = pMsg->code, .info = pMsg->info}; + if (vnodeProcessWriteReq(pVnode, pMsg, pMsg->info.conn.applyIndex, &rsp) < 0) { + rsp.code = terrno; + vInfo("vgId:%d, msg:%p failed to apply right now since %s", vgId, pMsg, terrstr()); + } + + if (rsp.info.handle != NULL) { + tmsgSendRsp(&rsp); + } + } } } @@ -163,10 +182,12 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info}; tmsgSendRedirectRsp(&rsp, &newEpSet); } else { - if (terrno != 0) code = terrno; - vError("vgId:%d, msg:%p failed to propose since %s, code:0x%x", vgId, pMsg, tstrerror(code), code); - SRpcMsg rsp = {.code = code, .info = pMsg->info}; - tmsgSendRsp(&rsp); + if (code != 1) { + if (terrno != 0) code = terrno; + vError("vgId:%d, msg:%p failed to propose since %s, code:0x%x", vgId, pMsg, tstrerror(code), code); + SRpcMsg rsp = {.code = code, .info = pMsg->info}; + tmsgSendRsp(&rsp); + } } vGTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, code); @@ -260,7 +281,7 @@ int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); - ret = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); + ret = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg, NULL); syncClientRequestDestroy(pSyncMsg); } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { @@ -359,34 +380,18 @@ static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta c SyncIndex beginIndex = SYNC_INDEX_INVALID; char logBuf[256] = {0}; - if (pFsm->FpGetSnapshotInfo != NULL) { - (*pFsm->FpGetSnapshotInfo)(pFsm, &snapshot); - beginIndex = snapshot.lastApplyIndex; - } + snprintf(logBuf, sizeof(logBuf), + "==callback== ==CommitCb== execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, beginIndex :%ld\n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), + beginIndex); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); - if (cbMeta.index > beginIndex) { - snprintf( - logBuf, sizeof(logBuf), - "==callback== ==CommitCb== execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, beginIndex :%ld\n", - pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex); - syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); - - SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen}; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen); - syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info); - rpcMsg.info.conn.applyIndex = cbMeta.index; - tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg); - - } else { - char logBuf[256] = {0}; - snprintf(logBuf, sizeof(logBuf), - "==callback== ==CommitCb== do not execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, " - "beginIndex :%ld\n", - pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), - beginIndex); - syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); - } + SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen}; + rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); + memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen); + syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info); + rpcMsg.info.conn.applyIndex = cbMeta.index; + tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg); } static void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 36f81e86ff..1cc3f9b874 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -403,7 +403,7 @@ typedef struct SIntervalAggOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode SOptrBasicInfo binfo; // basic info SAggSupporter aggSup; // aggregate supporter - + SExprSupp scalarSupp; // supporter for perform scalar function SGroupResInfo groupResInfo; // multiple results build supporter SInterval interval; // interval info int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. @@ -730,15 +730,12 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode *pNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams, SSDataBlock* pInputBlock, - SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pColMatchColInfo, - SExecTaskInfo* pTaskInfo); +SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo, bool isStream); + STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream); SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, @@ -808,9 +805,10 @@ int32_t getMaximumIdleDurationSec(); * ops: root operator * data: *data save the result of encode, need to be freed by caller * length: *length save the length of *data + * nOptrWithVal: *nOptrWithVal save the number of optr with value * return: result code, 0 means success */ -int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t *length); +int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t *length, int32_t *nOptrWithVal); /* * ops: root operator, created by caller diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h index 35bdd4ee55..972f4469e4 100644 --- a/source/libs/executor/inc/tsort.h +++ b/source/libs/executor/inc/tsort.h @@ -136,6 +136,13 @@ bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colId); */ void* tsortGetValue(STupleHandle* pVHandle, int32_t colId); +/** + * + * @param pVHandle + * @return + */ +uint64_t tsortGetGroupId(STupleHandle* pVHandle); + /** * * @param pSortHandle diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 374a3a736d..be6ca5145a 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -159,7 +159,12 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) { } SArray* createSortInfo(SNodeList* pNodeList) { - size_t numOfCols = LIST_LENGTH(pNodeList); + size_t numOfCols = 0; + if (pNodeList != NULL) { + numOfCols = LIST_LENGTH(pNodeList); + } else { + numOfCols = 0; + } SArray* pList = taosArrayInit(numOfCols, sizeof(SBlockOrderInfo)); if (pList == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index a9e1e03178..e4c0959185 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -222,7 +222,13 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) { return TSDB_CODE_INVALID_PARA; } - return encodeOperator(pTaskInfo->pRoot, pOutput, len); + int32_t nOptrWithVal = 0; + int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal); + if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal = 0)) { + taosMemoryFreeClear(*pOutput); + *len = 0; + } + return code; } int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index f18da72f58..17e5482b60 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2929,6 +2929,13 @@ int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* len int32_t totalSize = sizeof(int32_t) + sizeof(int32_t) + size * (sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize); + // no result + if (getTotalBufSize(pSup->pResultBuf) == 0) { + *result = NULL; + *length = 0; + return TSDB_CODE_SUCCESS; + } + *result = (char*)taosMemoryCalloc(1, totalSize); if (*result == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -4217,8 +4224,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; bool isStream = (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type); - pOptr = - createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pTaskInfo, isStream); + pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pIntervalPhyNode, + pTaskInfo, isStream); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) { SMergeAlignedIntervalPhysiNode* pIntervalPhyNode = (SMergeAlignedIntervalPhysiNode*)pPhyNode; @@ -4262,17 +4269,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createGroupSortOperatorInfo(ops[0], (SGroupSortPhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE == type) { SMergePhysiNode* pMergePhyNode = (SMergePhysiNode*)pPhyNode; - - SDataBlockDescNode* pDescNode = pPhyNode->pOutputDataBlockDesc; - SSDataBlock* pResBlock = createResDataBlock(pDescNode); - - SArray* sortInfo = createSortInfo(pMergePhyNode->pMergeKeys); - int32_t numOfOutputCols = 0; - SArray* pColList = - extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); - SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0); - SSDataBlock* pInputDataBlock = createResDataBlock(pChildNode->pOutputDataBlockDesc); - pOptr = createMultiwaySortMergeOperatorInfo(ops, size, pInputDataBlock, pResBlock, sortInfo, pColList, pTaskInfo); + pOptr = createMultiwayMergeOperatorInfo(ops, size, pMergePhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION == type) { SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; @@ -4465,12 +4462,12 @@ int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHa return 0; } -int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) { +int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length, int32_t* nOptrWithVal) { int32_t code = TDB_CODE_SUCCESS; char* pCurrent = NULL; int32_t currLength = 0; if (ops->fpSet.encodeResultRow) { - if (result == NULL || length == NULL) { + if (result == NULL || length == NULL || nOptrWithVal == NULL) { return TSDB_CODE_TSC_INVALID_INPUT; } code = ops->fpSet.encodeResultRow(ops, &pCurrent, &currLength); @@ -4481,8 +4478,15 @@ int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) { *result = NULL; } return code; + } else if (currLength == 0) { + ASSERT(!pCurrent); + goto _downstream; } + ++(*nOptrWithVal); + + ASSERT(currLength >= 0); + if (*result == NULL) { *result = (char*)taosMemoryCalloc(1, currLength + sizeof(int32_t)); if (*result == NULL) { @@ -4508,8 +4512,9 @@ int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) { *length = *(int32_t*)(*result); } +_downstream: for (int32_t i = 0; i < ops->numOfDownstream; ++i) { - code = encodeOperator(ops->pDownstream[i], result, length); + code = encodeOperator(ops->pDownstream[i], result, length, nOptrWithVal); if (code != TDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 1ac890517c..51bed90bde 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -448,7 +448,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) { qDebug("%s start to repeat ascending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) { STimeWindow* pWin = &pTableScanInfo->cond.twindows[i]; - qDebug("%s\t qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); + qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); } // do prepare for the next round table scan operation tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); @@ -467,7 +467,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) { qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) { STimeWindow* pWin = &pTableScanInfo->cond.twindows[i]; - qDebug("%s\t qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); + qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); } while (pTableScanInfo->scanTimes < total) { @@ -492,7 +492,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) { GET_TASKID(pTaskInfo)); for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) { STimeWindow* pWin = &pTableScanInfo->cond.twindows[i]; - qDebug("%s\t qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); + qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); } tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); pTableScanInfo->curTWinIdx = 0; @@ -2053,7 +2053,7 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle qDebug("no table qualified for query, TID:0x%" PRIx64 ", QID:0x%" PRIx64, taskId, queryId); return TSDB_CODE_SUCCESS; } - code = generateGroupIdMap(pTableListInfo, pHandle, pTableScanNode->pPartitionTags); + code = generateGroupIdMap(pTableListInfo, pHandle, pTableScanNode->pGroupTags); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2457,7 +2457,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN if (pInfo == NULL || pOperator == NULL) { goto _error; } - if (pTableScanNode->pPartitionTags) { + if (pTableScanNode->pGroupTags) { taosArraySort(pTableListInfo->pTableList, compareTableKeyInfoByGid); } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 09cc4e47be..44ff4c1c90 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -142,7 +142,8 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i SSDataBlock* loadNextDataBlock(void* param) { SOperatorInfo* pOperator = (SOperatorInfo*)param; - return pOperator->fpSet.getNextFn(pOperator); + SSDataBlock* pBlock = pOperator->fpSet.getNextFn(pOperator); + return pBlock; } // todo refactor: merged with fetch fp @@ -442,7 +443,7 @@ void destroyGroupSortOperatorInfo(void* param, int32_t numOfOutput) { SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo) { SGroupSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupSortOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL /* || rowSize > 100 * 1024 * 1024*/) { goto _error; } @@ -474,8 +475,8 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort pOperator->exprSupp.numOfExprs = numOfCols; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyGroupSortOperatorInfo, NULL, - NULL, getGroupSortExplainExecInfo); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyGroupSortOperatorInfo, + NULL, NULL, getGroupSortExplainExecInfo); int32_t code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -493,7 +494,7 @@ _error: //===================================================================================== // Multiway Sort Merge operator -typedef struct SMultiwaySortMergeOperatorInfo { +typedef struct SMultiwayMergeOperatorInfo { SOptrBasicInfo binfo; int32_t bufPageSize; @@ -503,14 +504,17 @@ typedef struct SMultiwaySortMergeOperatorInfo { SSortHandle* pSortHandle; SArray* pColMatchInfo; // for index map from table scan output - SSDataBlock* pInputBlock; - int64_t startTs; // sort start time - uint64_t groupId; -} SMultiwaySortMergeOperatorInfo; + SSDataBlock* pInputBlock; + int64_t startTs; // sort start time + bool groupSort; + bool hasGroupId; + uint64_t groupId; + STupleHandle* prefetchedTuple; +} SMultiwayMergeOperatorInfo; -int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { - SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; +int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) { + SMultiwayMergeOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; if (OPTR_IS_OPENED(pOperator)) { return TSDB_CODE_SUCCESS; @@ -524,8 +528,9 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { pInfo->pInputBlock, pTaskInfo->id.str); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, NULL, NULL); - tsortSetCompareGroupId(pInfo->pSortHandle, true); - + + tsortSetCompareGroupId(pInfo->pSortHandle, pInfo->groupSort); + for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) { SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); ps->param = pOperator->pDownstream[i]; @@ -547,8 +552,8 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo, SOperatorInfo* pOperator) { - SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SMultiwayMergeOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; blockDataCleanup(pDataBlock); @@ -560,18 +565,47 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData blockDataEnsureCapacity(p, capacity); while (1) { - STupleHandle* pTupleHandle = tsortNextTuple(pHandle); + STupleHandle* pTupleHandle = NULL; + if (pInfo->groupSort) { + if (pInfo->prefetchedTuple == NULL) { + pTupleHandle = tsortNextTuple(pHandle); + } else { + pTupleHandle = pInfo->prefetchedTuple; + pInfo->groupId = tsortGetGroupId(pTupleHandle); + pInfo->prefetchedTuple = NULL; + } + } + else { + pTupleHandle = tsortNextTuple(pHandle); + pInfo->groupId = 0; + } + if (pTupleHandle == NULL) { break; } - appendOneRowToDataBlock(p, pTupleHandle); + if (pInfo->groupSort) + { + uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle); + if (!pInfo->hasGroupId) { + pInfo->groupId = tupleGroupId; + pInfo->hasGroupId = true; + appendOneRowToDataBlock(p, pTupleHandle); + } else if (pInfo->groupId == tupleGroupId) { + appendOneRowToDataBlock(p, pTupleHandle); + } else { + pInfo->prefetchedTuple = pTupleHandle; + break; + } + } else { + appendOneRowToDataBlock(p, pTupleHandle); + } if (p->info.rows >= capacity) { break; } } - if (p->info.rows > 0) {// todo extract method + if (p->info.rows > 0) { // todo extract method blockDataEnsureCapacity(pDataBlock, p->info.rows); int32_t numOfCols = taosArrayGetSize(pColMatchInfo); for (int32_t i = 0; i < numOfCols; ++i) { @@ -593,13 +627,13 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; } -SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) { +SSDataBlock* doMultiwayMerge(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SMultiwayMergeOperatorInfo* pInfo = pOperator->info; int32_t code = pOperator->fpSet._openFn(pOperator); if (code != TSDB_CODE_SUCCESS) { @@ -608,7 +642,6 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) { SSDataBlock* pBlock = getMultiwaySortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo, pOperator); - if (pBlock != NULL) { pOperator->resultInfo.totalRows += pBlock->info.rows; } else { @@ -617,8 +650,8 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) { return pBlock; } -void destroyMultiwaySortMergeOperatorInfo(void* param, int32_t numOfOutput) { - SMultiwaySortMergeOperatorInfo* pInfo = (SMultiwaySortMergeOperatorInfo*)param; +void destroyMultiwayMergeOperatorInfo(void* param, int32_t numOfOutput) { + SMultiwayMergeOperatorInfo* pInfo = (SMultiwayMergeOperatorInfo*)param; pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); pInfo->pInputBlock = blockDataDestroy(pInfo->pInputBlock); @@ -626,11 +659,11 @@ void destroyMultiwaySortMergeOperatorInfo(void* param, int32_t numOfOutput) { taosArrayDestroy(pInfo->pColMatchInfo); } -int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { +int32_t getMultiwayMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { ASSERT(pOptr != NULL); SSortExecInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo)); - SMultiwaySortMergeOperatorInfo* pOperatorInfo = (SMultiwaySortMergeOperatorInfo*)pOptr->info; + SMultiwayMergeOperatorInfo* pOperatorInfo = (SMultiwayMergeOperatorInfo*)pOptr->info; *pInfo = tsortGetSortExecInfo(pOperatorInfo->pSortHandle); *pOptrExplain = pInfo; @@ -638,24 +671,35 @@ int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrEx return TSDB_CODE_SUCCESS; } -SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams, - SSDataBlock* pInputBlock, SSDataBlock* pResBlock, SArray* pSortInfo, - SArray* pColMatchColInfo, SExecTaskInfo* pTaskInfo) { - SMultiwaySortMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwaySortMergeOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - int32_t rowSize = pResBlock->info.rowSize; +SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size_t numStreams, + SMergePhysiNode* pMergePhyNode, SExecTaskInfo* pTaskInfo) { + SPhysiNode* pPhyNode = (SPhysiNode*)pMergePhyNode; + + SMultiwayMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwayMergeOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SDataBlockDescNode* pDescNode = pPhyNode->pOutputDataBlockDesc; + SSDataBlock* pResBlock = createResDataBlock(pDescNode); + + int32_t rowSize = pResBlock->info.rowSize; if (pInfo == NULL || pOperator == NULL || rowSize > 100 * 1024 * 1024) { goto _error; } + SArray* pSortInfo = createSortInfo(pMergePhyNode->pMergeKeys); + int32_t numOfOutputCols = 0; + SArray* pColMatchColInfo = + extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); + SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0); + SSDataBlock* pInputBlock = createResDataBlock(pChildNode->pOutputDataBlockDesc); initResultSizeInfo(pOperator, 1024); + pInfo->groupSort = pMergePhyNode->groupSort; pInfo->binfo.pRes = pResBlock; pInfo->pSortInfo = pSortInfo; pInfo->pColMatchInfo = pColMatchColInfo; pInfo->pInputBlock = pInputBlock; - pOperator->name = "MultiwaySortMerge"; + pOperator->name = "MultiwayMerge"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE; pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; @@ -667,9 +711,8 @@ SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, // one additional is reserved for merged result. pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1); - pOperator->fpSet = - createOperatorFpSet(doOpenMultiwaySortMergeOperator, doMultiwaySortMerge, NULL, NULL, - destroyMultiwaySortMergeOperatorInfo, NULL, NULL, getMultiwaySortMergeExplainExecInfo); + pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL, NULL, + destroyMultiwayMergeOperatorInfo, NULL, NULL, getMultiwayMergeExplainExecInfo); int32_t code = appendDownstream(pOperator, downStreams, numStreams); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index b7d39207d8..963e714972 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -969,6 +969,12 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { getTableScanInfo(pOperator, &pInfo->order, &scanFlag); + if (pInfo->scalarSupp.pExprInfo != NULL) { + SExprSupp* pExprSup =& pInfo->scalarSupp; + projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, + pExprSup->numOfExprs, NULL); + } + // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, scanFlag, true); hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag, NULL); @@ -1381,6 +1387,11 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { continue; } + if (pInfo->scalarSupp.pExprInfo != NULL) { + SExprSupp* pExprSup = &pInfo->scalarSupp; + projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + } + // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the // caller. Note that all the time window are not close till now. // the pDataBlock are always the same one, no need to call this again @@ -1498,18 +1509,27 @@ void increaseTs(SqlFunctionCtx* pCtx) { SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo, bool isStream) { + STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream) { SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->win = pTaskInfo->window; - pInfo->order = TSDB_ORDER_ASC; - pInfo->interval = *pInterval; + pInfo->win = pTaskInfo->window; + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; pInfo->execModel = pTaskInfo->execModel; - pInfo->twAggSup = *pTwAggSupp; + pInfo->twAggSup = *pTwAggSupp; + + if (pPhyNode->window.pExprs != NULL) { + int32_t numOfScalar = 0; + SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar); + int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } pInfo->primaryTsIndex = primaryTsSlotId; @@ -2473,7 +2493,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { if (IS_FINAL_OP(pInfo)) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); - SIntervalAggOperatorInfo* pChildInfo = pChildOp->info; + SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info; SExprSupp* pChildSup = &pChildOp->exprSupp; doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildInfo->primaryTsIndex, diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 50474ff62e..7d44d41b55 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -752,6 +752,10 @@ void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) { } } +uint64_t tsortGetGroupId(STupleHandle* pVHandle) { + return pVHandle->pBlock->info.groupId; +} + SSortExecInfo tsortGetSortExecInfo(SSortHandle* pHandle) { SSortExecInfo info = {0}; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 983cffe9dc..364ee0692f 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -110,7 +110,7 @@ static void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf); static int32_t udfdConnectToMnode(); static int32_t udfdLoadUdf(char *udfName, SUdf *udf); -static bool udfdRpcRfp(int32_t code); +static bool udfdRpcRfp(int32_t code, tmsg_t msgType); static int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static int32_t udfdOpenClientRpc(); static int32_t udfdCloseClientRpc(); @@ -546,9 +546,12 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { } return 0; } -static bool udfdRpcRfp(int32_t code) { +static bool udfdRpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { + if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) { + return false; + } return true; } else { return false; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index ac197912cf..9d8baf472b 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -351,7 +351,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_SCALAR_FIELD(watermark); COPY_SCALAR_FIELD(tsColId); COPY_SCALAR_FIELD(filesFactor); - CLONE_NODE_LIST_FIELD(pPartTags); + CLONE_NODE_LIST_FIELD(pGroupTags); return TSDB_CODE_SUCCESS; } @@ -401,6 +401,7 @@ static int32_t logicMergeCopy(const SMergeLogicNode* pSrc, SMergeLogicNode* pDst CLONE_NODE_LIST_FIELD(pInputs); COPY_SCALAR_FIELD(numOfChannels); COPY_SCALAR_FIELD(srcGroupId); + COPY_SCALAR_FIELD(groupSort); return TSDB_CODE_SUCCESS; } @@ -436,6 +437,7 @@ static int32_t logicFillCopy(const SFillLogicNode* pSrc, SFillLogicNode* pDst) { static int32_t logicSortCopy(const SSortLogicNode* pSrc, SSortLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pSortKeys); + COPY_SCALAR_FIELD(groupSort); return TSDB_CODE_SUCCESS; } @@ -500,7 +502,7 @@ static int32_t physiTableScanCopy(const STableScanPhysiNode* pSrc, STableScanPhy COPY_SCALAR_FIELD(ratio); COPY_SCALAR_FIELD(dataRequired); CLONE_NODE_LIST_FIELD(pDynamicScanFuncs); - CLONE_NODE_LIST_FIELD(pPartitionTags); + CLONE_NODE_LIST_FIELD(pGroupTags); COPY_SCALAR_FIELD(interval); COPY_SCALAR_FIELD(offset); COPY_SCALAR_FIELD(sliding); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 514d65d52e..df7429bd88 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -234,6 +234,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiMerge"; case QUERY_NODE_PHYSICAL_PLAN_SORT: return "PhysiSort"; + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: + return "PhysiGroupSort"; case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: return "PhysiHashInterval"; case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: @@ -539,7 +541,7 @@ static const char* jkScanLogicPlanScanPseudoCols = "ScanPseudoCols"; static const char* jkScanLogicPlanTableId = "TableId"; static const char* jkScanLogicPlanTableType = "TableType"; static const char* jkScanLogicPlanTagCond = "TagCond"; -static const char* jkScanLogicPlanPartTags = "PartTags"; +static const char* jkScanLogicPlanGroupTags = "GroupTags"; static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { const SScanLogicNode* pNode = (const SScanLogicNode*)pObj; @@ -561,7 +563,7 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddObject(pJson, jkScanLogicPlanTagCond, nodeToJson, pNode->pTagCond); } if (TSDB_CODE_SUCCESS == code) { - code = nodeListToJson(pJson, jkScanLogicPlanPartTags, pNode->pPartTags); + code = nodeListToJson(pJson, jkScanLogicPlanGroupTags, pNode->pGroupTags); } return code; @@ -588,7 +590,7 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { code = jsonToNodeObject(pJson, jkScanLogicPlanTagCond, &pNode->pTagCond); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeList(pJson, jkScanLogicPlanPartTags, &pNode->pPartTags); + code = jsonToNodeList(pJson, jkScanLogicPlanGroupTags, &pNode->pGroupTags); } return code; @@ -1430,7 +1432,8 @@ static const char* jkTableScanPhysiPlanTriggerType = "triggerType"; static const char* jkTableScanPhysiPlanWatermark = "watermark"; static const char* jkTableScanPhysiPlanTsColId = "tsColId"; static const char* jkTableScanPhysiPlanFilesFactor = "FilesFactor"; -static const char* jkTableScanPhysiPlanPartitionTags = "PartitionTags"; +static const char* jkTableScanPhysiPlanGroupTags = "GroupTags"; +static const char* jkTableScanPhysiPlanGroupSort = "GroupSort"; static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj; @@ -1485,7 +1488,10 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddDoubleToObject(pJson, jkTableScanPhysiPlanFilesFactor, pNode->filesFactor); } if (TSDB_CODE_SUCCESS == code) { - code = nodeListToJson(pJson, jkTableScanPhysiPlanPartitionTags, pNode->pPartitionTags); + code = nodeListToJson(pJson, jkTableScanPhysiPlanGroupTags, pNode->pGroupTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanGroupSort, pNode->groupSort); } return code; @@ -1544,7 +1550,10 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { code = tjsonGetDoubleValue(pJson, jkTableScanPhysiPlanFilesFactor, &pNode->filesFactor); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeList(pJson, jkTableScanPhysiPlanPartitionTags, &pNode->pPartitionTags); + code = jsonToNodeList(pJson, jkTableScanPhysiPlanGroupTags, &pNode->pGroupTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanGroupSort, &pNode->groupSort); } return code; @@ -1725,6 +1734,7 @@ static const char* jkMergePhysiPlanMergeKeys = "MergeKeys"; static const char* jkMergePhysiPlanTargets = "Targets"; static const char* jkMergePhysiPlanNumOfChannels = "NumOfChannels"; static const char* jkMergePhysiPlanSrcGroupId = "SrcGroupId"; +static const char* jkMergePhysiPlanGroupSort = "GroupSort"; static int32_t physiMergeNodeToJson(const void* pObj, SJson* pJson) { const SMergePhysiNode* pNode = (const SMergePhysiNode*)pObj; @@ -1742,6 +1752,9 @@ static int32_t physiMergeNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkMergePhysiPlanSrcGroupId, pNode->srcGroupId); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkMergePhysiPlanGroupSort, pNode->groupSort); + } return code; } @@ -1762,6 +1775,9 @@ static int32_t jsonToPhysiMergeNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkMergePhysiPlanSrcGroupId, &pNode->srcGroupId); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkMergePhysiPlanGroupSort, &pNode->groupSort); + } return code; } @@ -3369,6 +3385,7 @@ static const char* jkSlotDescSlotId = "SlotId"; static const char* jkSlotDescDataType = "DataType"; static const char* jkSlotDescReserve = "Reserve"; static const char* jkSlotDescOutput = "Output"; +static const char* jkSlotDescName = "Name"; static int32_t slotDescNodeToJson(const void* pObj, SJson* pJson) { const SSlotDescNode* pNode = (const SSlotDescNode*)pObj; @@ -3383,6 +3400,9 @@ static int32_t slotDescNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkSlotDescOutput, pNode->output); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkSlotDescName, pNode->name); + } return code; } @@ -3400,6 +3420,9 @@ static int32_t jsonToSlotDescNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkSlotDescOutput, &pNode->output); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkSlotDescName, pNode->name); + } return code; } @@ -4137,6 +4160,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN_MERGE: return physiMergeNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_SORT: + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: return physiSortNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: @@ -4280,6 +4304,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { case QUERY_NODE_PHYSICAL_PLAN_MERGE: return jsonToPhysiMergeNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_SORT: + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: return jsonToPhysiSortNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index 1fb2db9f23..3747dde9ed 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -500,7 +500,8 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk } break; } - case QUERY_NODE_PHYSICAL_PLAN_SORT: { + case QUERY_NODE_PHYSICAL_PLAN_SORT: + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: { SSortPhysiNode* pSort = (SSortPhysiNode*)pNode; res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext); if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 734f287cfb..be6faa92cb 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -288,6 +288,8 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SMergePhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_SORT: return makeNode(type, sizeof(SSortPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: + return makeNode(type, sizeof(SGroupSortPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: return makeNode(type, sizeof(SIntervalPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: @@ -709,7 +711,7 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pLogicNode->pTagCond); nodesDestroyNode(pLogicNode->pTagIndexCond); taosArrayDestroy(pLogicNode->pSmaIndexes); - nodesDestroyList(pLogicNode->pPartTags); + nodesDestroyList(pLogicNode->pGroupTags); break; } case QUERY_NODE_LOGIC_PLAN_JOIN: { @@ -813,7 +815,7 @@ void nodesDestroyNode(SNode* pNode) { STableScanPhysiNode* pPhyNode = (STableScanPhysiNode*)pNode; destroyScanPhysiNode((SScanPhysiNode*)pNode); nodesDestroyList(pPhyNode->pDynamicScanFuncs); - nodesDestroyList(pPhyNode->pPartitionTags); + nodesDestroyList(pPhyNode->pGroupTags); break; } case QUERY_NODE_PHYSICAL_PLAN_PROJECT: { @@ -850,7 +852,8 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyList(pPhyNode->pTargets); break; } - case QUERY_NODE_PHYSICAL_PLAN_SORT: { + case QUERY_NODE_PHYSICAL_PLAN_SORT: + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: { SSortPhysiNode* pPhyNode = (SSortPhysiNode*)pNode; destroyPhysiNode((SPhysiNode*)pPhyNode); nodesDestroyList(pPhyNode->pExprs); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b68a4cc568..45240536e3 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2456,90 +2456,6 @@ static int32_t rewriteUniqueStmt(STranslateContext* pCxt, SSelectStmt* pSelect) return cxt.pTranslateCxt->errCode; } -typedef struct SRwriteTailCxt { - STranslateContext* pTranslateCxt; - int64_t limit; - int64_t offset; -} SRwriteTailCxt; - -static EDealRes rewriteTailFunc(SNode** pNode, void* pContext) { - SRwriteTailCxt* pCxt = pContext; - if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { - SFunctionNode* pFunc = (SFunctionNode*)*pNode; - if (FUNCTION_TYPE_TAIL == pFunc->funcType) { - pCxt->limit = ((SValueNode*)nodesListGetNode(pFunc->pParameterList, 1))->datum.i; - if (3 == LIST_LENGTH(pFunc->pParameterList)) { - pCxt->offset = ((SValueNode*)nodesListGetNode(pFunc->pParameterList, 2))->datum.i; - } - SNode* pExpr = nodesListGetNode(pFunc->pParameterList, 0); - strcpy(((SExprNode*)pExpr)->aliasName, ((SExprNode*)*pNode)->aliasName); - NODES_CLEAR_LIST(pFunc->pParameterList); - nodesDestroyNode(*pNode); - *pNode = pExpr; - return DEAL_RES_IGNORE_CHILD; - } - } - return DEAL_RES_CONTINUE; -} - -static int32_t createLimieNode(SRwriteTailCxt* pCxt, SLimitNode** pOutput) { - *pOutput = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT); - if (NULL == *pOutput) { - return TSDB_CODE_OUT_OF_MEMORY; - } - (*pOutput)->limit = pCxt->limit; - (*pOutput)->offset = pCxt->offset; - return TSDB_CODE_SUCCESS; -} - -static SNode* createOrderByExpr(STranslateContext* pCxt) { - SOrderByExprNode* pOrder = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); - if (NULL == pOrder) { - return NULL; - } - pCxt->errCode = createPrimaryKeyCol(pCxt, &pOrder->pExpr); - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - nodesDestroyNode((SNode*)pOrder); - return NULL; - } - pOrder->order = ORDER_DESC; - pOrder->nullOrder = NULL_ORDER_FIRST; - return (SNode*)pOrder; -} - -/* case 1: - * in: select tail(expr, k, f) from t where_clause - * out: select expr from t where_clause order by _rowts desc limit k offset f - * - * case 2: - * in: select tail(expr, k, f) from t where_clause partition_by_clause - * out: select expr from t where_clause partition_by_clause sort by _rowts desc limit k offset f - * - * case 3: - * in: select tail(expr, k, f) from t where_clause order_by_clause limit_clause - * out: select expr from ( - * select expr from t where_clause order by _rowts desc limit k offset f - * ) order_by_clause limit_clause - * - * case 4: - * in: select tail(expr, k, f) from t where_clause partition_by_clause limit_clause - * out: - */ -static int32_t rewriteTailStmt(STranslateContext* pCxt, SSelectStmt* pSelect) { - if (!pSelect->hasTailFunc) { - return TSDB_CODE_SUCCESS; - } - - SRwriteTailCxt cxt = {.pTranslateCxt = pCxt, .limit = -1, .offset = -1}; - nodesRewriteExprs(pSelect->pProjectionList, rewriteTailFunc, &cxt); - int32_t code = nodesListMakeStrictAppend(&pSelect->pOrderByList, createOrderByExpr(pCxt)); - if (TSDB_CODE_SUCCESS == code) { - code = createLimieNode(&cxt, &pSelect->pLimit); - } - pSelect->hasIndefiniteRowsFunc = false; - return code; -} - typedef struct SReplaceOrderByAliasCxt { STranslateContext* pTranslateCxt; SNodeList* pProjectionList; @@ -2616,9 +2532,6 @@ static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect if (TSDB_CODE_SUCCESS == code) { code = rewriteUniqueStmt(pCxt, pSelect); } - // if (TSDB_CODE_SUCCESS == code) { - // code = rewriteTailStmt(pCxt, pSelect); - // } if (TSDB_CODE_SUCCESS == code) { code = rewriteTimelineFunc(pCxt, pSelect); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index c4f4624355..202dd96581 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -337,7 +337,11 @@ int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) { static bool isValidateTag(char* input) { if (!input) return false; for (size_t i = 0; i < strlen(input); ++i) { + #ifdef WINDOWS + if (input[i] < 0x20 || input[i] > 0x7E) return false; + #else if (isprint(input[i]) == 0) return false; + #endif } return true; } @@ -377,6 +381,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, SMs char* jsonKey = item->string; if (!isValidateTag(jsonKey)) { + fprintf(stdout,"%s(%d) %s %08" PRId64 "\n", __FILE__, __LINE__,__func__,taosGetSelfPthreadId());fflush(stdout); retCode = buildSyntaxErrMsg(pMsgBuf, "json key not validate", jsonKey); goto end; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index a4cdcd35d3..396a7b6193 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -507,6 +507,8 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt return TSDB_CODE_OUT_OF_MEMORY; } + pIdfRowsFunc->isTailFunc = pSelect->hasTailFunc; + // indefinite rows functions and _select_values functions int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsVectorFunc, &pIdfRowsFunc->pFuncs); if (TSDB_CODE_SUCCESS == code) { @@ -733,6 +735,8 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect return TSDB_CODE_OUT_OF_MEMORY; } + pSort->groupSort = pSelect->groupSort; + int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, COLLECT_COL_TYPE_ALL, &pSort->node.pTargets); if (TSDB_CODE_SUCCESS == code && NULL == pSort->node.pTargets) { code = nodesListMakeStrictAppend(&pSort->node.pTargets, diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index f1cad6010c..53c9ef8dd0 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -20,9 +20,8 @@ #define OPTIMIZE_FLAG_MASK(n) (1 << n) -#define OPTIMIZE_FLAG_OSD OPTIMIZE_FLAG_MASK(0) -#define OPTIMIZE_FLAG_CPD OPTIMIZE_FLAG_MASK(1) -#define OPTIMIZE_FLAG_OPK OPTIMIZE_FLAG_MASK(2) +#define OPTIMIZE_FLAG_SCAN_PATH OPTIMIZE_FLAG_MASK(0) +#define OPTIMIZE_FLAG_PUSH_DOWN_CONDE OPTIMIZE_FLAG_MASK(1) #define OPTIMIZE_FLAG_SET_MASK(val, mask) (val) |= (mask) #define OPTIMIZE_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0) @@ -76,7 +75,7 @@ static SLogicNode* optFindPossibleNode(SLogicNode* pNode, FMayBeOptimized func) return NULL; } -EDealRes osdHaveNormalColImpl(SNode* pNode, void* pContext) { +EDealRes scanPathOptHaveNormalColImpl(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { // *((bool*)pContext) = (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType); *((bool*)pContext) = true; @@ -85,14 +84,14 @@ EDealRes osdHaveNormalColImpl(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static bool osdHaveNormalCol(SNodeList* pList) { +static bool scanPathOptHaveNormalCol(SNodeList* pList) { bool res = false; - nodesWalkExprsPostOrder(pList, osdHaveNormalColImpl, &res); + nodesWalkExprsPostOrder(pList, scanPathOptHaveNormalColImpl, &res); return res; } -static bool osdMayBeOptimized(SLogicNode* pNode) { - if (OPTIMIZE_FLAG_TEST_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_OSD)) { +static bool scanPathOptMayBeOptimized(SLogicNode* pNode) { + if (OPTIMIZE_FLAG_TEST_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_SCAN_PATH)) { return false; } if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode)) { @@ -108,10 +107,10 @@ static bool osdMayBeOptimized(SLogicNode* pNode) { QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent->pParent))) { return true; } - return !osdHaveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys); + return !scanPathOptHaveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys); } -static SNodeList* osdGetAllFuncs(SLogicNode* pNode) { +static SNodeList* scanPathOptGetAllFuncs(SLogicNode* pNode) { switch (nodeType(pNode)) { case QUERY_NODE_LOGIC_PLAN_WINDOW: return ((SWindowLogicNode*)pNode)->pFuncs; @@ -123,7 +122,7 @@ static SNodeList* osdGetAllFuncs(SLogicNode* pNode) { return NULL; } -static bool needOptimizeDataRequire(const SFunctionNode* pFunc) { +static bool scanPathOptNeedOptimizeDataRequire(const SFunctionNode* pFunc) { if (!fmIsSpecialDataRequiredFunc(pFunc->funcId)) { return false; } @@ -136,7 +135,7 @@ static bool needOptimizeDataRequire(const SFunctionNode* pFunc) { return true; } -static bool needOptimizeDynamicScan(const SFunctionNode* pFunc) { +static bool scanPathOptNeedDynOptimize(const SFunctionNode* pFunc) { if (!fmIsDynamicScanOptimizedFunc(pFunc->funcId)) { return false; } @@ -149,17 +148,17 @@ static bool needOptimizeDynamicScan(const SFunctionNode* pFunc) { return true; } -static int32_t osdGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs, SNodeList** pDsoFuncs) { - SNodeList* pAllFuncs = osdGetAllFuncs(pScan->node.pParent); +static int32_t scanPathOptGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs, SNodeList** pDsoFuncs) { + SNodeList* pAllFuncs = scanPathOptGetAllFuncs(pScan->node.pParent); SNodeList* pTmpSdrFuncs = NULL; SNodeList* pTmpDsoFuncs = NULL; SNode* pFunc = NULL; bool otherFunc = false; FOREACH(pFunc, pAllFuncs) { int32_t code = TSDB_CODE_SUCCESS; - if (needOptimizeDataRequire((SFunctionNode*)pFunc)) { + if (scanPathOptNeedOptimizeDataRequire((SFunctionNode*)pFunc)) { code = nodesListMakeStrictAppend(&pTmpSdrFuncs, nodesCloneNode(pFunc)); - } else if (needOptimizeDynamicScan((SFunctionNode*)pFunc)) { + } else if (scanPathOptNeedDynOptimize((SFunctionNode*)pFunc)) { code = nodesListMakeStrictAppend(&pTmpDsoFuncs, nodesCloneNode(pFunc)); } else { otherFunc = true; @@ -180,15 +179,15 @@ static int32_t osdGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs, return TSDB_CODE_SUCCESS; } -static int32_t osdMatch(SOptimizeContext* pCxt, SLogicNode* pLogicNode, SOsdInfo* pInfo) { - pInfo->pScan = (SScanLogicNode*)optFindPossibleNode(pLogicNode, osdMayBeOptimized); +static int32_t scanPathOptMatch(SOptimizeContext* pCxt, SLogicNode* pLogicNode, SOsdInfo* pInfo) { + pInfo->pScan = (SScanLogicNode*)optFindPossibleNode(pLogicNode, scanPathOptMayBeOptimized); if (NULL == pInfo->pScan) { return TSDB_CODE_SUCCESS; } - return osdGetRelatedFuncs(pInfo->pScan, &pInfo->pSdrFuncs, &pInfo->pDsoFuncs); + return scanPathOptGetRelatedFuncs(pInfo->pScan, &pInfo->pSdrFuncs, &pInfo->pDsoFuncs); } -static EFuncDataRequired osdPromoteDataRequired(EFuncDataRequired l, EFuncDataRequired r) { +static EFuncDataRequired scanPathOptPromoteDataRequired(EFuncDataRequired l, EFuncDataRequired r) { switch (l) { case FUNC_DATA_REQUIRED_DATA_LOAD: return l; @@ -202,19 +201,19 @@ static EFuncDataRequired osdPromoteDataRequired(EFuncDataRequired l, EFuncDataRe return r; } -static int32_t osdGetDataRequired(SNodeList* pFuncs) { +static int32_t scanPathOptGetDataRequired(SNodeList* pFuncs) { if (NULL == pFuncs) { return FUNC_DATA_REQUIRED_DATA_LOAD; } EFuncDataRequired dataRequired = FUNC_DATA_REQUIRED_FILTEROUT; SNode* pFunc = NULL; FOREACH(pFunc, pFuncs) { - dataRequired = osdPromoteDataRequired(dataRequired, fmFuncDataRequired((SFunctionNode*)pFunc, NULL)); + dataRequired = scanPathOptPromoteDataRequired(dataRequired, fmFuncDataRequired((SFunctionNode*)pFunc, NULL)); } return dataRequired; } -static void setScanWindowInfo(SScanLogicNode* pScan) { +static void scanPathOptSetScanWin(SScanLogicNode* pScan) { SLogicNode* pParent = pScan->node.pParent; if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pParent) && pParent->pParent && QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pParent->pParent)) { @@ -233,23 +232,23 @@ static void setScanWindowInfo(SScanLogicNode* pScan) { } } -static int32_t osdOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { +static int32_t scanPathOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { SOsdInfo info = {0}; - int32_t code = osdMatch(pCxt, pLogicSubplan->pNode, &info); + int32_t code = scanPathOptMatch(pCxt, pLogicSubplan->pNode, &info); if (TSDB_CODE_SUCCESS == code && info.pScan) { - setScanWindowInfo((SScanLogicNode*)info.pScan); + scanPathOptSetScanWin((SScanLogicNode*)info.pScan); } if (TSDB_CODE_SUCCESS == code && (NULL != info.pDsoFuncs || NULL != info.pSdrFuncs)) { - info.pScan->dataRequired = osdGetDataRequired(info.pSdrFuncs); + info.pScan->dataRequired = scanPathOptGetDataRequired(info.pSdrFuncs); info.pScan->pDynamicScanFuncs = info.pDsoFuncs; - OPTIMIZE_FLAG_SET_MASK(info.pScan->node.optimizedFlag, OPTIMIZE_FLAG_OSD); + OPTIMIZE_FLAG_SET_MASK(info.pScan->node.optimizedFlag, OPTIMIZE_FLAG_SCAN_PATH); pCxt->optimized = true; } nodesDestroyList(info.pSdrFuncs); return code; } -static int32_t cpdMergeCond(SNode** pDst, SNode** pSrc) { +static int32_t pushDownCondOptMergeCond(SNode** pDst, SNode** pSrc) { SLogicConditionNode* pLogicCond = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); if (NULL == pLogicCond) { return TSDB_CODE_OUT_OF_MEMORY; @@ -270,7 +269,7 @@ static int32_t cpdMergeCond(SNode** pDst, SNode** pSrc) { return code; } -static int32_t cpdCondAppend(SNode** pCond, SNode** pAdditionalCond) { +static int32_t pushDownCondOptAppendCond(SNode** pCond, SNode** pAdditionalCond) { if (NULL == *pCond) { TSWAP(*pCond, *pAdditionalCond); return TSDB_CODE_SUCCESS; @@ -283,16 +282,16 @@ static int32_t cpdCondAppend(SNode** pCond, SNode** pAdditionalCond) { *pAdditionalCond = NULL; } } else { - code = cpdMergeCond(pCond, pAdditionalCond); + code = pushDownCondOptMergeCond(pCond, pAdditionalCond); } return code; } -static int32_t cpdCalcTimeRange(SOptimizeContext* pCxt, SScanLogicNode* pScan, SNode** pPrimaryKeyCond, - SNode** pOtherCond) { +static int32_t pushDownCondOptCalcTimeRange(SOptimizeContext* pCxt, SScanLogicNode* pScan, SNode** pPrimaryKeyCond, + SNode** pOtherCond) { int32_t code = TSDB_CODE_SUCCESS; if (pCxt->pPlanCxt->topicQuery || pCxt->pPlanCxt->streamQuery) { - code = cpdCondAppend(pOtherCond, pPrimaryKeyCond); + code = pushDownCondOptAppendCond(pOtherCond, pPrimaryKeyCond); } else { bool isStrict = false; code = filterGetTimeRange(*pPrimaryKeyCond, &pScan->scanRange, &isStrict); @@ -300,7 +299,7 @@ static int32_t cpdCalcTimeRange(SOptimizeContext* pCxt, SScanLogicNode* pScan, S if (isStrict) { nodesDestroyNode(*pPrimaryKeyCond); } else { - code = cpdCondAppend(pOtherCond, pPrimaryKeyCond); + code = pushDownCondOptAppendCond(pOtherCond, pPrimaryKeyCond); } *pPrimaryKeyCond = NULL; } @@ -308,8 +307,9 @@ static int32_t cpdCalcTimeRange(SOptimizeContext* pCxt, SScanLogicNode* pScan, S return code; } -static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan) { - if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD) || +static int32_t pushDownCondOptDealScan(SOptimizeContext* pCxt, SScanLogicNode* pScan) { + if (NULL == pScan->node.pConditions || + OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE) || TSDB_SYSTEM_TABLE == pScan->tableType) { return TSDB_CODE_SUCCESS; } @@ -319,14 +319,14 @@ static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* int32_t code = nodesPartitionCond(&pScan->node.pConditions, &pPrimaryKeyCond, &pScan->pTagIndexCond, &pScan->pTagCond, &pOtherCond); if (TSDB_CODE_SUCCESS == code && NULL != pPrimaryKeyCond) { - code = cpdCalcTimeRange(pCxt, pScan, &pPrimaryKeyCond, &pOtherCond); + code = pushDownCondOptCalcTimeRange(pCxt, pScan, &pPrimaryKeyCond, &pOtherCond); } if (TSDB_CODE_SUCCESS == code) { pScan->node.pConditions = pOtherCond; } if (TSDB_CODE_SUCCESS == code) { - OPTIMIZE_FLAG_SET_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD); + OPTIMIZE_FLAG_SET_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE); pCxt->optimized = true; } else { nodesDestroyNode(pPrimaryKeyCond); @@ -336,7 +336,7 @@ static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* return code; } -static bool cpdBelongThisTable(SNode* pCondCol, SNodeList* pTableCols) { +static bool pushDownCondOptBelongThisTable(SNode* pCondCol, SNodeList* pTableCols) { SNode* pTableCol = NULL; FOREACH(pTableCol, pTableCols) { if (nodesEqualNode(pCondCol, pTableCol)) { @@ -346,12 +346,12 @@ static bool cpdBelongThisTable(SNode* pCondCol, SNodeList* pTableCols) { return false; } -static EDealRes cpdIsMultiTableCondImpl(SNode* pNode, void* pContext) { +static EDealRes pushDownCondOptIsCrossTableCond(SNode* pNode, void* pContext) { SCpdIsMultiTableCondCxt* pCxt = pContext; if (QUERY_NODE_COLUMN == nodeType(pNode)) { - if (cpdBelongThisTable(pNode, pCxt->pLeftCols)) { + if (pushDownCondOptBelongThisTable(pNode, pCxt->pLeftCols)) { pCxt->havaLeftCol = true; - } else if (cpdBelongThisTable(pNode, pCxt->pRightCols)) { + } else if (pushDownCondOptBelongThisTable(pNode, pCxt->pRightCols)) { pCxt->haveRightCol = true; } return pCxt->havaLeftCol && pCxt->haveRightCol ? DEAL_RES_END : DEAL_RES_CONTINUE; @@ -359,10 +359,11 @@ static EDealRes cpdIsMultiTableCondImpl(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static ECondAction cpdCondAction(EJoinType joinType, SNodeList* pLeftCols, SNodeList* pRightCols, SNode* pNode) { +static ECondAction pushDownCondOptGetCondAction(EJoinType joinType, SNodeList* pLeftCols, SNodeList* pRightCols, + SNode* pNode) { SCpdIsMultiTableCondCxt cxt = { .pLeftCols = pLeftCols, .pRightCols = pRightCols, .havaLeftCol = false, .haveRightCol = false}; - nodesWalkExpr(pNode, cpdIsMultiTableCondImpl, &cxt); + nodesWalkExpr(pNode, pushDownCondOptIsCrossTableCond, &cxt); return (JOIN_TYPE_INNER != joinType ? COND_ACTION_STAY : (cxt.havaLeftCol && cxt.haveRightCol @@ -370,8 +371,8 @@ static ECondAction cpdCondAction(EJoinType joinType, SNodeList* pLeftCols, SNode : (cxt.havaLeftCol ? COND_ACTION_PUSH_LEFT_CHILD : COND_ACTION_PUSH_RIGHT_CHILD))); } -static int32_t cpdPartitionLogicCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, - SNode** pRightChildCond) { +static int32_t pushDownCondOptPartLogicCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, + SNode** pRightChildCond) { SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pJoin->node.pConditions; if (LOGIC_COND_TYPE_AND != pLogicCond->condType) { return TSDB_CODE_SUCCESS; @@ -387,7 +388,7 @@ static int32_t cpdPartitionLogicCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNo SNodeList* pRemainConds = NULL; SNode* pCond = NULL; FOREACH(pCond, pLogicCond->pParameterList) { - ECondAction condAction = cpdCondAction(pJoin->joinType, pLeftCols, pRightCols, pCond); + ECondAction condAction = pushDownCondOptGetCondAction(pJoin->joinType, pLeftCols, pRightCols, pCond); if (COND_ACTION_PUSH_JOIN == condAction) { code = nodesListMakeAppend(&pOnConds, nodesCloneNode(pCond)); } else if (COND_ACTION_PUSH_LEFT_CHILD == condAction) { @@ -439,11 +440,12 @@ static int32_t cpdPartitionLogicCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNo return code; } -static int32_t cpdPartitionOpCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, - SNode** pRightChildCond) { +static int32_t pushDownCondOptPartOpCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, + SNode** pRightChildCond) { SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; - ECondAction condAction = cpdCondAction(pJoin->joinType, pLeftCols, pRightCols, pJoin->node.pConditions); + ECondAction condAction = + pushDownCondOptGetCondAction(pJoin->joinType, pLeftCols, pRightCols, pJoin->node.pConditions); if (COND_ACTION_STAY == condAction) { return TSDB_CODE_SUCCESS; } else if (COND_ACTION_PUSH_JOIN == condAction) { @@ -457,35 +459,35 @@ static int32_t cpdPartitionOpCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode* return TSDB_CODE_SUCCESS; } -static int32_t cpdPartitionCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, - SNode** pRightChildCond) { +static int32_t pushDownCondOptPartCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, + SNode** pRightChildCond) { if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->node.pConditions)) { - return cpdPartitionLogicCond(pJoin, pOnCond, pLeftChildCond, pRightChildCond); + return pushDownCondOptPartLogicCond(pJoin, pOnCond, pLeftChildCond, pRightChildCond); } else { - return cpdPartitionOpCond(pJoin, pOnCond, pLeftChildCond, pRightChildCond); + return pushDownCondOptPartOpCond(pJoin, pOnCond, pLeftChildCond, pRightChildCond); } } -static int32_t cpdPushCondToOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode** pCond) { - return cpdCondAppend(&pJoin->pOnConditions, pCond); +static int32_t pushDownCondOptPushCondToOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode** pCond) { + return pushDownCondOptAppendCond(&pJoin->pOnConditions, pCond); } -static int32_t cpdPushCondToScan(SOptimizeContext* pCxt, SScanLogicNode* pScan, SNode** pCond) { - return cpdCondAppend(&pScan->node.pConditions, pCond); +static int32_t pushDownCondOptPushCondToScan(SOptimizeContext* pCxt, SScanLogicNode* pScan, SNode** pCond) { + return pushDownCondOptAppendCond(&pScan->node.pConditions, pCond); } -static int32_t cpdPushCondToChild(SOptimizeContext* pCxt, SLogicNode* pChild, SNode** pCond) { +static int32_t pushDownCondOptPushCondToChild(SOptimizeContext* pCxt, SLogicNode* pChild, SNode** pCond) { switch (nodeType(pChild)) { case QUERY_NODE_LOGIC_PLAN_SCAN: - return cpdPushCondToScan(pCxt, (SScanLogicNode*)pChild, pCond); + return pushDownCondOptPushCondToScan(pCxt, (SScanLogicNode*)pChild, pCond); default: break; } - planError("cpdPushCondToChild failed, invalid logic plan node %s", nodesNodeName(nodeType(pChild))); + planError("pushDownCondOptPushCondToChild failed, invalid logic plan node %s", nodesNodeName(nodeType(pChild))); return TSDB_CODE_PLAN_INTERNAL_ERROR; } -static bool cpdIsPrimaryKey(SNode* pNode, SNodeList* pTableCols) { +static bool pushDownCondOptIsPriKey(SNode* pNode, SNodeList* pTableCols) { if (QUERY_NODE_COLUMN != nodeType(pNode)) { return false; } @@ -493,10 +495,10 @@ static bool cpdIsPrimaryKey(SNode* pNode, SNodeList* pTableCols) { if (PRIMARYKEY_TIMESTAMP_COL_ID != pCol->colId) { return false; } - return cpdBelongThisTable(pNode, pTableCols); + return pushDownCondOptBelongThisTable(pNode, pTableCols); } -static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { +static bool pushDownCondOptIsPriKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { if (QUERY_NODE_OPERATOR != nodeType(pCond)) { return false; } @@ -508,15 +510,15 @@ static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; - if (cpdIsPrimaryKey(pOper->pLeft, pLeftCols)) { - return cpdIsPrimaryKey(pOper->pRight, pRightCols); - } else if (cpdIsPrimaryKey(pOper->pLeft, pRightCols)) { - return cpdIsPrimaryKey(pOper->pRight, pLeftCols); + if (pushDownCondOptIsPriKey(pOper->pLeft, pLeftCols)) { + return pushDownCondOptIsPriKey(pOper->pRight, pRightCols); + } else if (pushDownCondOptIsPriKey(pOper->pLeft, pRightCols)) { + return pushDownCondOptIsPriKey(pOper->pRight, pLeftCols); } return false; } -static bool cpdContainPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { +static bool pushDownCondOptContainPriKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { if (QUERY_NODE_LOGIC_CONDITION == nodeType(pCond)) { SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pCond; if (LOGIC_COND_TYPE_AND != pLogicCond->condType) { @@ -525,54 +527,56 @@ static bool cpdContainPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { bool hasPrimaryKeyEqualCond = false; SNode* pCond = NULL; FOREACH(pCond, pLogicCond->pParameterList) { - if (cpdContainPrimaryKeyEqualCond(pJoin, pCond)) { + if (pushDownCondOptContainPriKeyEqualCond(pJoin, pCond)) { hasPrimaryKeyEqualCond = true; break; } } return hasPrimaryKeyEqualCond; } else { - return cpdIsPrimaryKeyEqualCond(pJoin, pCond); + return pushDownCondOptIsPriKeyEqualCond(pJoin, pCond); } } -static int32_t cpdCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { +static int32_t pushDownCondOptCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (NULL == pJoin->pOnConditions) { return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN); } - if (!cpdContainPrimaryKeyEqualCond(pJoin, pJoin->pOnConditions)) { + if (!pushDownCondOptContainPriKeyEqualCond(pJoin, pJoin->pOnConditions)) { return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); } return TSDB_CODE_SUCCESS; } -static int32_t cpdPushJoinCondition(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { - if (OPTIMIZE_FLAG_TEST_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_CPD)) { +static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { + if (OPTIMIZE_FLAG_TEST_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) { return TSDB_CODE_SUCCESS; } if (NULL == pJoin->node.pConditions) { - return cpdCheckJoinOnCond(pCxt, pJoin); + return pushDownCondOptCheckJoinOnCond(pCxt, pJoin); } SNode* pOnCond = NULL; SNode* pLeftChildCond = NULL; SNode* pRightChildCond = NULL; - int32_t code = cpdPartitionCond(pJoin, &pOnCond, &pLeftChildCond, &pRightChildCond); + int32_t code = pushDownCondOptPartCond(pJoin, &pOnCond, &pLeftChildCond, &pRightChildCond); if (TSDB_CODE_SUCCESS == code && NULL != pOnCond) { - code = cpdPushCondToOnCond(pCxt, pJoin, &pOnCond); + code = pushDownCondOptPushCondToOnCond(pCxt, pJoin, &pOnCond); } if (TSDB_CODE_SUCCESS == code && NULL != pLeftChildCond) { - code = cpdPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0), &pLeftChildCond); + code = + pushDownCondOptPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0), &pLeftChildCond); } if (TSDB_CODE_SUCCESS == code && NULL != pRightChildCond) { - code = cpdPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1), &pRightChildCond); + code = + pushDownCondOptPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1), &pRightChildCond); } if (TSDB_CODE_SUCCESS == code) { - OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_CPD); + OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE); pCxt->optimized = true; - code = cpdCheckJoinOnCond(pCxt, pJoin); + code = pushDownCondOptCheckJoinOnCond(pCxt, pJoin); } else { nodesDestroyNode(pOnCond); nodesDestroyNode(pLeftChildCond); @@ -582,22 +586,22 @@ static int32_t cpdPushJoinCondition(SOptimizeContext* pCxt, SJoinLogicNode* pJoi return code; } -static int32_t cpdPushAggCondition(SOptimizeContext* pCxt, SAggLogicNode* pAgg) { +static int32_t pushDownCondOptDealAgg(SOptimizeContext* pCxt, SAggLogicNode* pAgg) { // todo return TSDB_CODE_SUCCESS; } -static int32_t cpdPushCondition(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { +static int32_t pushDownCondOptimizeImpl(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pLogicNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: - code = cpdOptimizeScanCondition(pCxt, (SScanLogicNode*)pLogicNode); + code = pushDownCondOptDealScan(pCxt, (SScanLogicNode*)pLogicNode); break; case QUERY_NODE_LOGIC_PLAN_JOIN: - code = cpdPushJoinCondition(pCxt, (SJoinLogicNode*)pLogicNode); + code = pushDownCondOptDealJoin(pCxt, (SJoinLogicNode*)pLogicNode); break; case QUERY_NODE_LOGIC_PLAN_AGG: - code = cpdPushAggCondition(pCxt, (SAggLogicNode*)pLogicNode); + code = pushDownCondOptDealAgg(pCxt, (SAggLogicNode*)pLogicNode); break; default: break; @@ -605,7 +609,7 @@ static int32_t cpdPushCondition(SOptimizeContext* pCxt, SLogicNode* pLogicNode) if (TSDB_CODE_SUCCESS == code) { SNode* pChild = NULL; FOREACH(pChild, pLogicNode->pChildren) { - code = cpdPushCondition(pCxt, (SLogicNode*)pChild); + code = pushDownCondOptimizeImpl(pCxt, (SLogicNode*)pChild); if (TSDB_CODE_SUCCESS != code) { break; } @@ -614,11 +618,11 @@ static int32_t cpdPushCondition(SOptimizeContext* pCxt, SLogicNode* pLogicNode) return code; } -static int32_t cpdOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { - return cpdPushCondition(pCxt, pLogicSubplan->pNode); +static int32_t pushDownCondOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + return pushDownCondOptimizeImpl(pCxt, pLogicSubplan->pNode); } -static bool opkIsPrimaryKeyOrderBy(SNodeList* pSortKeys) { +static bool sortPriKeyOptIsPriKeyOrderBy(SNodeList* pSortKeys) { if (1 != LIST_LENGTH(pSortKeys)) { return false; } @@ -626,17 +630,18 @@ static bool opkIsPrimaryKeyOrderBy(SNodeList* pSortKeys) { return (QUERY_NODE_COLUMN == nodeType(pNode) ? (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) : false); } -static bool opkSortMayBeOptimized(SLogicNode* pNode) { +static bool sortPriKeyOptMayBeOptimized(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SORT != nodeType(pNode)) { return false; } - if (OPTIMIZE_FLAG_TEST_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_OPK)) { - return false; + SSortLogicNode* pSort = (SSortLogicNode*)pNode; + if (pSort->groupSort || !sortPriKeyOptIsPriKeyOrderBy(pSort->pSortKeys) || 1 != LIST_LENGTH(pSort->node.pChildren)) { + return TSDB_CODE_SUCCESS; } return true; } -static int32_t opkGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeList** pScanNodes) { +static int32_t sortPriKeyOptGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeList** pScanNodes) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pNode)) { @@ -646,9 +651,11 @@ static int32_t opkGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeL } break; case QUERY_NODE_LOGIC_PLAN_JOIN: - code = opkGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes); + code = + sortPriKeyOptGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes); if (TSDB_CODE_SUCCESS == code) { - code = opkGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 1), pNotOptimize, pScanNodes); + code = + sortPriKeyOptGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 1), pNotOptimize, pScanNodes); } return code; case QUERY_NODE_LOGIC_PLAN_AGG: @@ -663,77 +670,58 @@ static int32_t opkGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeL return TSDB_CODE_SUCCESS; } - return opkGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes); + return sortPriKeyOptGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes); } -static int32_t opkGetScanNodes(SLogicNode* pNode, SNodeList** pScanNodes) { +static int32_t sortPriKeyOptGetScanNodes(SLogicNode* pNode, SNodeList** pScanNodes) { bool notOptimize = false; - int32_t code = opkGetScanNodesImpl(pNode, ¬Optimize, pScanNodes); + int32_t code = sortPriKeyOptGetScanNodesImpl(pNode, ¬Optimize, pScanNodes); if (TSDB_CODE_SUCCESS != code || notOptimize) { nodesClearList(*pScanNodes); } return code; } -static EOrder opkGetPrimaryKeyOrder(SSortLogicNode* pSort) { +static EOrder sortPriKeyOptGetPriKeyOrder(SSortLogicNode* pSort) { return ((SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0))->order; } -static SNode* opkRewriteDownNode(SSortLogicNode* pSort) { - SNode* pDownNode = nodesListGetNode(pSort->node.pChildren, 0); - // todo - NODES_CLEAR_LIST(pSort->node.pChildren); - return pDownNode; -} - -static int32_t opkDoOptimized(SOptimizeContext* pCxt, SSortLogicNode* pSort, SNodeList* pScanNodes) { - EOrder order = opkGetPrimaryKeyOrder(pSort); +static int32_t sortPriKeyOptApply(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SSortLogicNode* pSort, + SNodeList* pScanNodes) { + EOrder order = sortPriKeyOptGetPriKeyOrder(pSort); if (ORDER_DESC == order) { SNode* pScan = NULL; FOREACH(pScan, pScanNodes) { TSWAP(((SScanLogicNode*)pScan)->scanSeq[0], ((SScanLogicNode*)pScan)->scanSeq[1]); } } - if (NULL == pSort->node.pParent) { - // todo - return TSDB_CODE_SUCCESS; + int32_t code = + replaceLogicNode(pLogicSubplan, (SLogicNode*)pSort, (SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0)); + if (TSDB_CODE_SUCCESS == code) { + NODES_CLEAR_LIST(pSort->node.pChildren); + nodesDestroyNode((SNode*)pSort); } - - SNode* pDownNode = opkRewriteDownNode(pSort); - SNode* pNode; - FOREACH(pNode, pSort->node.pParent->pChildren) { - if (nodesEqualNode(pNode, (SNode*)pSort)) { - REPLACE_NODE(pDownNode); - ((SLogicNode*)pDownNode)->pParent = pSort->node.pParent; - break; - } - } - nodesDestroyNode((SNode*)pSort); - return TSDB_CODE_SUCCESS; + return code; } -static int32_t opkOptimizeImpl(SOptimizeContext* pCxt, SSortLogicNode* pSort) { - OPTIMIZE_FLAG_SET_MASK(pSort->node.optimizedFlag, OPTIMIZE_FLAG_OPK); - if (!opkIsPrimaryKeyOrderBy(pSort->pSortKeys) || 1 != LIST_LENGTH(pSort->node.pChildren)) { - return TSDB_CODE_SUCCESS; - } +static int32_t sortPrimaryKeyOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SSortLogicNode* pSort) { SNodeList* pScanNodes = NULL; - int32_t code = opkGetScanNodes((SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0), &pScanNodes); + int32_t code = sortPriKeyOptGetScanNodes((SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0), &pScanNodes); if (TSDB_CODE_SUCCESS == code && NULL != pScanNodes) { - code = opkDoOptimized(pCxt, pSort, pScanNodes); + code = sortPriKeyOptApply(pCxt, pLogicSubplan, pSort, pScanNodes); } nodesClearList(pScanNodes); return code; } -static int32_t opkOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { - SSortLogicNode* pSort = (SSortLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, opkSortMayBeOptimized); +static int32_t sortPrimaryKeyOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SSortLogicNode* pSort = (SSortLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, sortPriKeyOptMayBeOptimized); if (NULL == pSort) { return TSDB_CODE_SUCCESS; } - return opkOptimizeImpl(pCxt, pSort); + return sortPrimaryKeyOptimizeImpl(pCxt, pLogicSubplan, pSort); } -static bool smaOptMayBeOptimized(SLogicNode* pNode) { +static bool smaIndexOptMayBeOptimized(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode) || NULL == pNode->pParent || QUERY_NODE_LOGIC_PLAN_WINDOW != nodeType(pNode->pParent) || WINDOW_TYPE_INTERVAL != ((SWindowLogicNode*)pNode->pParent)->winType) { @@ -748,7 +736,8 @@ static bool smaOptMayBeOptimized(SLogicNode* pNode) { return true; } -static int32_t smaOptCreateMerge(SLogicNode* pChild, SNodeList* pMergeKeys, SNodeList* pTargets, SLogicNode** pOutput) { +static int32_t smaIndexOptCreateMerge(SLogicNode* pChild, SNodeList* pMergeKeys, SNodeList* pTargets, + SLogicNode** pOutput) { SMergeLogicNode* pMerge = (SMergeLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_MERGE); if (NULL == pMerge) { return TSDB_CODE_OUT_OF_MEMORY; @@ -767,8 +756,8 @@ static int32_t smaOptCreateMerge(SLogicNode* pChild, SNodeList* pMergeKeys, SNod return TSDB_CODE_SUCCESS; } -static int32_t smaOptRecombinationNode(SLogicSubplan* pLogicSubplan, SLogicNode* pInterval, SLogicNode* pMerge, - SLogicNode* pSmaScan) { +static int32_t smaIndexOptRecombinationNode(SLogicSubplan* pLogicSubplan, SLogicNode* pInterval, SLogicNode* pMerge, + SLogicNode* pSmaScan) { int32_t code = nodesListMakeAppend(&pMerge->pChildren, (SNode*)pInterval); if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeAppend(&pMerge->pChildren, (SNode*)pSmaScan); @@ -781,8 +770,8 @@ static int32_t smaOptRecombinationNode(SLogicSubplan* pLogicSubplan, SLogicNode* return code; } -static int32_t smaOptCreateSmaScan(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList* pCols, - SLogicNode** pOutput) { +static int32_t smaIndexOptCreateSmaScan(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList* pCols, + SLogicNode** pOutput) { SScanLogicNode* pSmaScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN); if (NULL == pSmaScan) { return TSDB_CODE_OUT_OF_MEMORY; @@ -811,7 +800,7 @@ static int32_t smaOptCreateSmaScan(SScanLogicNode* pScan, STableIndexInfo* pInde return TSDB_CODE_SUCCESS; } -static bool smaOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pWindow, STableIndexInfo* pIndex) { +static bool smaIndexOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pWindow, STableIndexInfo* pIndex) { if (pWindow->interval != pIndex->interval || pWindow->intervalUnit != pIndex->intervalUnit || pWindow->offset != pIndex->offset || pWindow->sliding != pIndex->sliding || pWindow->slidingUnit != pIndex->slidingUnit) { @@ -831,7 +820,7 @@ static bool smaOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pWindow return true; } -static SNode* smaOptCreateSmaCol(SNode* pFunc, uint64_t tableId, int32_t colId) { +static SNode* smaIndexOptCreateSmaCol(SNode* pFunc, uint64_t tableId, int32_t colId) { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { return NULL; @@ -846,7 +835,7 @@ static SNode* smaOptCreateSmaCol(SNode* pFunc, uint64_t tableId, int32_t colId) return (SNode*)pCol; } -static int32_t smaOptFindSmaFunc(SNode* pQueryFunc, SNodeList* pSmaFuncs) { +static int32_t smaIndexOptFindSmaFunc(SNode* pQueryFunc, SNodeList* pSmaFuncs) { int32_t index = 0; SNode* pSmaFunc = NULL; FOREACH(pSmaFunc, pSmaFuncs) { @@ -858,8 +847,8 @@ static int32_t smaOptFindSmaFunc(SNode* pQueryFunc, SNodeList* pSmaFuncs) { return -1; } -static int32_t smaOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, SNodeList** pOutput, - int32_t* pWStrartIndex) { +static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, SNodeList** pOutput, + int32_t* pWStrartIndex) { SNodeList* pCols = NULL; SNode* pFunc = NULL; int32_t code = TSDB_CODE_SUCCESS; @@ -870,11 +859,11 @@ static int32_t smaOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeLis if (FUNCTION_TYPE_WSTARTTS == ((SFunctionNode*)pFunc)->funcType) { *pWStrartIndex = index; } - smaFuncIndex = smaOptFindSmaFunc(pFunc, pSmaFuncs); + smaFuncIndex = smaIndexOptFindSmaFunc(pFunc, pSmaFuncs); if (smaFuncIndex < 0) { break; } else { - code = nodesListMakeStrictAppend(&pCols, smaOptCreateSmaCol(pFunc, tableId, smaFuncIndex + 2)); + code = nodesListMakeStrictAppend(&pCols, smaIndexOptCreateSmaCol(pFunc, tableId, smaFuncIndex + 2)); if (TSDB_CODE_SUCCESS != code) { break; } @@ -891,22 +880,22 @@ static int32_t smaOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeLis return code; } -static int32_t smaOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList** pCols, - int32_t* pWStrartIndex) { +static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList** pCols, + int32_t* pWStrartIndex) { SWindowLogicNode* pWindow = (SWindowLogicNode*)pScan->node.pParent; - if (!smaOptEqualInterval(pScan, pWindow, pIndex)) { + if (!smaIndexOptEqualInterval(pScan, pWindow, pIndex)) { return TSDB_CODE_SUCCESS; } SNodeList* pSmaFuncs = NULL; int32_t code = nodesStringToList(pIndex->expr, &pSmaFuncs); if (TSDB_CODE_SUCCESS == code) { - code = smaOptCreateSmaCols(pWindow->pFuncs, pIndex->dstTbUid, pSmaFuncs, pCols, pWStrartIndex); + code = smaIndexOptCreateSmaCols(pWindow->pFuncs, pIndex->dstTbUid, pSmaFuncs, pCols, pWStrartIndex); } nodesDestroyList(pSmaFuncs); return code; } -static SNode* smaOptCreateWStartTs() { +static SNode* smaIndexOptCreateWStartTs() { SFunctionNode* pWStart = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); if (NULL == pWStart) { return NULL; @@ -920,7 +909,7 @@ static SNode* smaOptCreateWStartTs() { return (SNode*)pWStart; } -static int32_t smaOptCreateMergeKey(SNode* pCol, SNodeList** pMergeKeys) { +static int32_t smaIndexOptCreateMergeKey(SNode* pCol, SNodeList** pMergeKeys) { SOrderByExprNode* pMergeKey = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); if (NULL == pMergeKey) { return TSDB_CODE_OUT_OF_MEMORY; @@ -935,9 +924,9 @@ static int32_t smaOptCreateMergeKey(SNode* pCol, SNodeList** pMergeKeys) { return nodesListMakeStrictAppend(pMergeKeys, (SNode*)pMergeKey); } -static int32_t smaOptRewriteInterval(SWindowLogicNode* pInterval, int32_t wstrartIndex, SNodeList** pMergeKeys) { +static int32_t smaIndexOptRewriteInterval(SWindowLogicNode* pInterval, int32_t wstrartIndex, SNodeList** pMergeKeys) { if (wstrartIndex < 0) { - SNode* pWStart = smaOptCreateWStartTs(); + SNode* pWStart = smaIndexOptCreateWStartTs(); if (NULL == pWStart) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -948,11 +937,11 @@ static int32_t smaOptRewriteInterval(SWindowLogicNode* pInterval, int32_t wstrar } wstrartIndex = LIST_LENGTH(pInterval->node.pTargets) - 1; } - return smaOptCreateMergeKey(nodesListGetNode(pInterval->node.pTargets, wstrartIndex), pMergeKeys); + return smaIndexOptCreateMergeKey(nodesListGetNode(pInterval->node.pTargets, wstrartIndex), pMergeKeys); } -static int32_t smaOptApplyIndexExt(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, - SNodeList* pSmaCols, int32_t wstrartIndex) { +static int32_t smaIndexOptApplyIndexExt(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, + SNodeList* pSmaCols, int32_t wstrartIndex) { SWindowLogicNode* pInterval = (SWindowLogicNode*)pScan->node.pParent; SNodeList* pMergeTargets = nodesCloneList(pInterval->node.pTargets); if (NULL == pMergeTargets) { @@ -961,42 +950,42 @@ static int32_t smaOptApplyIndexExt(SLogicSubplan* pLogicSubplan, SScanLogicNode* SLogicNode* pSmaScan = NULL; SLogicNode* pMerge = NULL; SNodeList* pMergeKeys = NULL; - int32_t code = smaOptRewriteInterval(pInterval, wstrartIndex, &pMergeKeys); + int32_t code = smaIndexOptRewriteInterval(pInterval, wstrartIndex, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { - code = smaOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); + code = smaIndexOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); } if (TSDB_CODE_SUCCESS == code) { - code = smaOptCreateMerge(pScan->node.pParent, pMergeKeys, pMergeTargets, &pMerge); + code = smaIndexOptCreateMerge(pScan->node.pParent, pMergeKeys, pMergeTargets, &pMerge); } if (TSDB_CODE_SUCCESS == code) { - code = smaOptRecombinationNode(pLogicSubplan, pScan->node.pParent, pMerge, pSmaScan); + code = smaIndexOptRecombinationNode(pLogicSubplan, pScan->node.pParent, pMerge, pSmaScan); } return code; } -static int32_t smaOptApplyIndex(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, - SNodeList* pSmaCols, int32_t wstrartIndex) { +static int32_t smaIndexOptApplyIndex(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, + SNodeList* pSmaCols, int32_t wstrartIndex) { SLogicNode* pSmaScan = NULL; - int32_t code = smaOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); + int32_t code = smaIndexOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); if (TSDB_CODE_SUCCESS == code) { code = replaceLogicNode(pLogicSubplan, pScan->node.pParent, pSmaScan); } return code; } -static void smaOptDestroySmaIndex(void* p) { taosMemoryFree(((STableIndexInfo*)p)->expr); } +static void smaIndexOptDestroySmaIndex(void* p) { taosMemoryFree(((STableIndexInfo*)p)->expr); } -static int32_t smaOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan) { +static int32_t smaIndexOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan) { int32_t code = TSDB_CODE_SUCCESS; int32_t nindexes = taosArrayGetSize(pScan->pSmaIndexes); for (int32_t i = 0; i < nindexes; ++i) { STableIndexInfo* pIndex = taosArrayGet(pScan->pSmaIndexes, i); SNodeList* pSmaCols = NULL; int32_t wstrartIndex = -1; - code = smaOptCouldApplyIndex(pScan, pIndex, &pSmaCols, &wstrartIndex); + code = smaIndexOptCouldApplyIndex(pScan, pIndex, &pSmaCols, &wstrartIndex); if (TSDB_CODE_SUCCESS == code && NULL != pSmaCols) { - code = smaOptApplyIndex(pLogicSubplan, pScan, pIndex, pSmaCols, wstrartIndex); - taosArrayDestroyEx(pScan->pSmaIndexes, smaOptDestroySmaIndex); + code = smaIndexOptApplyIndex(pLogicSubplan, pScan, pIndex, pSmaCols, wstrartIndex); + taosArrayDestroyEx(pScan->pSmaIndexes, smaIndexOptDestroySmaIndex); pScan->pSmaIndexes = NULL; break; } @@ -1004,12 +993,12 @@ static int32_t smaOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubp return code; } -static int32_t smaOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { - SScanLogicNode* pScan = (SScanLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, smaOptMayBeOptimized); +static int32_t smaIndexOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SScanLogicNode* pScan = (SScanLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, smaIndexOptMayBeOptimized); if (NULL == pScan) { return TSDB_CODE_SUCCESS; } - return smaOptimizeImpl(pCxt, pLogicSubplan, pScan); + return smaIndexOptimizeImpl(pCxt, pLogicSubplan, pScan); } static EDealRes partTagsOptHasColImpl(SNode* pNode, void* pContext) { @@ -1084,7 +1073,7 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub int32_t code = TSDB_CODE_SUCCESS; SScanLogicNode* pScan = (SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0); if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { - TSWAP(((SPartitionLogicNode*)pNode)->pPartitionKeys, pScan->pPartTags); + TSWAP(((SPartitionLogicNode*)pNode)->pPartitionKeys, pScan->pGroupTags); int32_t code = replaceLogicNode(pLogicSubplan, pNode, (SLogicNode*)pScan); if (TSDB_CODE_SUCCESS == code) { NODES_CLEAR_LIST(pNode->pChildren); @@ -1094,7 +1083,7 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub SNode* pGroupKey = NULL; FOREACH(pGroupKey, ((SAggLogicNode*)pNode)->pGroupKeys) { code = nodesListMakeStrictAppend( - &pScan->pPartTags, nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pGroupKey)->pParameterList, 0))); + &pScan->pGroupTags, nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pGroupKey)->pParameterList, 0))); if (TSDB_CODE_SUCCESS != code) { break; } @@ -1102,13 +1091,11 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub NODES_DESTORY_LIST(((SAggLogicNode*)pNode)->pGroupKeys); } if (TSDB_CODE_SUCCESS == code) { - code = partTagsOptRebuildTbanme(pScan->pPartTags); + code = partTagsOptRebuildTbanme(pScan->pGroupTags); } return code; } -//===================================================================================================================== -// eliminate project optimization static bool eliminateProjOptCheckProjColumnNames(SProjectLogicNode* pProjectNode) { SHashObj* pProjColNameHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); SNode* pProjection; @@ -1142,7 +1129,7 @@ static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) { return false; } - SNode* pProjection; + SNode* pProjection; FOREACH(pProjection, pProjectNode->pProjections) { SExprNode* pExprNode = (SExprNode*)pProjection; if (QUERY_NODE_COLUMN != nodeType(pExprNode)) { @@ -1190,31 +1177,169 @@ static int32_t eliminateProjOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLog return eliminateProjOptimizeImpl(pCxt, pLogicSubplan, pProjectNode); } -//===================================================================================================================== -// eliminate Set Operator optimization +static bool rewriteTailOptMayBeOptimized(SLogicNode* pNode) { + return QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC == nodeType(pNode) && ((SIndefRowsFuncLogicNode*)pNode)->isTailFunc; +} + +static SNode* rewriteTailOptCreateOrderByExpr(SNode* pSortKey) { + SOrderByExprNode* pOrder = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); + if (NULL == pOrder) { + return NULL; + } + pOrder->order = ORDER_DESC; + pOrder->pExpr = nodesCloneNode(pSortKey); + if (NULL == pOrder->pExpr) { + nodesDestroyNode((SNode*)pOrder); + return NULL; + } + return (SNode*)pOrder; +} + +static int32_t rewriteTailOptCreateLimit(SNode* pLimit, SNode* pOffset, SNode** pOutput) { + SLimitNode* pLimitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT); + if (NULL == pLimitNode) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pLimitNode->limit = NULL == pLimit ? -1 : ((SValueNode*)pLimit)->datum.i; + pLimitNode->offset = NULL == pOffset ? -1 : ((SValueNode*)pOffset)->datum.i; + *pOutput = (SNode*)pLimitNode; + return TSDB_CODE_SUCCESS; +} + +static bool rewriteTailOptNeedGroupSort(SIndefRowsFuncLogicNode* pIndef) { + if (1 != LIST_LENGTH(pIndef->node.pChildren)) { + return false; + } + SNode* pChild = nodesListGetNode(pIndef->node.pChildren, 0); + return QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pChild) || + (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && NULL != ((SScanLogicNode*)pChild)->pGroupTags); +} + +static int32_t rewriteTailOptCreateSort(SIndefRowsFuncLogicNode* pIndef, SLogicNode** pOutput) { + SSortLogicNode* pSort = (SSortLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SORT); + if (NULL == pSort) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pSort->groupSort = rewriteTailOptNeedGroupSort(pIndef); + TSWAP(pSort->node.pChildren, pIndef->node.pChildren); + pSort->node.precision = pIndef->node.precision; + + // tail(expr, [limit, offset,] _rowts) + SFunctionNode* pTail = (SFunctionNode*)nodesListGetNode(pIndef->pFuncs, 0); + int32_t rowtsIndex = LIST_LENGTH(pTail->pParameterList) - 1; + + int32_t code = nodesListMakeStrictAppend( + &pSort->pSortKeys, rewriteTailOptCreateOrderByExpr(nodesListGetNode(pTail->pParameterList, rowtsIndex))); + if (TSDB_CODE_SUCCESS == code) { + pSort->node.pTargets = nodesCloneList(((SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0))->pTargets); + if (NULL == pSort->node.pTargets) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pOutput = (SLogicNode*)pSort; + } else { + nodesDestroyNode((SNode*)pSort); + } + + return code; +} + +static SNode* rewriteTailOptCreateProjectExpr(SFunctionNode* pTail) { + SNode* pExpr = nodesCloneNode(nodesListGetNode(pTail->pParameterList, 0)); + if (NULL == pExpr) { + return NULL; + } + strcpy(((SExprNode*)pExpr)->aliasName, pTail->node.aliasName); + return pExpr; +} + +static int32_t rewriteTailOptCreateProject(SIndefRowsFuncLogicNode* pIndef, SLogicNode** pOutput) { + SProjectLogicNode* pProject = (SProjectLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_PROJECT); + if (NULL == pProject) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + TSWAP(pProject->node.pTargets, pIndef->node.pTargets); + pProject->node.precision = pIndef->node.precision; + + // tail(expr, [limit, offset,] _rowts) + SFunctionNode* pTail = (SFunctionNode*)nodesListGetNode(pIndef->pFuncs, 0); + int32_t limitIndex = LIST_LENGTH(pTail->pParameterList) > 2 ? 1 : -1; + int32_t offsetIndex = LIST_LENGTH(pTail->pParameterList) > 3 ? 2 : -1; + + int32_t code = nodesListMakeStrictAppend(&pProject->pProjections, rewriteTailOptCreateProjectExpr(pTail)); + if (TSDB_CODE_SUCCESS == code) { + code = rewriteTailOptCreateLimit(limitIndex < 0 ? NULL : nodesListGetNode(pTail->pParameterList, limitIndex), + offsetIndex < 0 ? NULL : nodesListGetNode(pTail->pParameterList, offsetIndex), + &pProject->node.pLimit); + } + if (TSDB_CODE_SUCCESS == code) { + *pOutput = (SLogicNode*)pProject; + } else { + nodesDestroyNode((SNode*)pProject); + } + return code; +} + +static int32_t rewriteTailOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, + SIndefRowsFuncLogicNode* pIndef) { + SLogicNode* pSort = NULL; + SLogicNode* pProject = NULL; + int32_t code = rewriteTailOptCreateSort(pIndef, &pSort); + if (TSDB_CODE_SUCCESS == code) { + code = rewriteTailOptCreateProject(pIndef, &pProject); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeAppend(&pProject->pChildren, (SNode*)pSort); + pSort->pParent = pProject; + pSort = NULL; + } + if (TSDB_CODE_SUCCESS == code) { + code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pIndef, pProject); + } + if (TSDB_CODE_SUCCESS == code) { + nodesDestroyNode((SNode*)pIndef); + } else { + nodesDestroyNode((SNode*)pSort); + nodesDestroyNode((SNode*)pProject); + } + return code; +} + +static int32_t rewriteTailOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SIndefRowsFuncLogicNode* pIndef = + (SIndefRowsFuncLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, rewriteTailOptMayBeOptimized); + + if (NULL == pIndef) { + return TSDB_CODE_SUCCESS; + } + + return rewriteTailOptimizeImpl(pCxt, pLogicSubplan, pIndef); +} static bool eliminateSetOpMayBeOptimized(SLogicNode* pNode) { SLogicNode* pParent = pNode->pParent; if (NULL == pParent || - QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pParent) && QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pParent) || + QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pParent) && QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pParent) || LIST_LENGTH(pParent->pChildren) < 2) { return false; } - if (nodeType(pNode) != nodeType(pNode->pParent) || - LIST_LENGTH(pNode->pChildren) < 2) { + if (nodeType(pNode) != nodeType(pNode->pParent) || LIST_LENGTH(pNode->pChildren) < 2) { return false; } return true; } -static int32_t eliminateSetOpOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SLogicNode* pSetOpNode) { +static int32_t eliminateSetOpOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, + SLogicNode* pSetOpNode) { SNode* pSibling; FOREACH(pSibling, pSetOpNode->pParent->pChildren) { if (nodesEqualNode(pSibling, (SNode*)pSetOpNode)) { SNode* pChild; - FOREACH(pChild, pSetOpNode->pChildren) { - ((SLogicNode*)pChild)->pParent = pSetOpNode->pParent; - } + FOREACH(pChild, pSetOpNode->pChildren) { ((SLogicNode*)pChild)->pParent = pSetOpNode->pParent; } INSERT_LIST(pSetOpNode->pParent->pChildren, pSetOpNode->pChildren); pSetOpNode->pChildren = NULL; @@ -1237,13 +1362,14 @@ static int32_t eliminateSetOpOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLo // clang-format off static const SOptimizeRule optimizeRuleSet[] = { - {.pName = "OptimizeScanData", .optimizeFunc = osdOptimize}, - {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, - {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}, - {.pName = "SmaIndex", .optimizeFunc = smaOptimize}, + {.pName = "ScanPath", .optimizeFunc = scanPathOptimize}, + {.pName = "PushDownCondition", .optimizeFunc = pushDownCondOptimize}, + {.pName = "SortPrimaryKey", .optimizeFunc = sortPrimaryKeyOptimize}, + {.pName = "SmaIndex", .optimizeFunc = smaIndexOptimize}, {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize}, - {.pName = "EliminateSetOperator", .optimizeFunc = eliminateSetOpOptimize} + {.pName = "EliminateSetOperator", .optimizeFunc = eliminateSetOpOptimize}, + {.pName = "RewriteTail", .optimizeFunc = rewriteTailOptimize} }; // clang-format on diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 4f46abd8da..46747af3a9 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -41,8 +41,12 @@ typedef struct SPhysiPlanContext { static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { SColumnNode* pCol = (SColumnNode*)pNode; - if (NULL != pStmtName && '\0' != pStmtName[0]) { - return sprintf(pKey, "%s.%s", pStmtName, pCol->node.aliasName); + if (NULL != pStmtName) { + if ('\0' != pStmtName[0]) { + return sprintf(pKey, "%s.%s", pStmtName, pCol->node.aliasName); + } else { + return sprintf(pKey, "%s", pCol->node.aliasName); + } } if ('\0' == pCol->tableAlias[0]) { return sprintf(pKey, "%s", pCol->colName); @@ -56,11 +60,13 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) { return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName); } -static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_t slotId, bool output, bool reserve) { +static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const char* pName, const SNode* pNode, int16_t slotId, + bool output, bool reserve) { SSlotDescNode* pSlot = (SSlotDescNode*)nodesMakeNode(QUERY_NODE_SLOT_DESC); if (NULL == pSlot) { return NULL; } + strcpy(pSlot->name, pName); pSlot->slotId = slotId; pSlot->dataType = ((SExprNode*)pNode)->resType; pSlot->reserve = reserve; @@ -99,10 +105,8 @@ static int32_t putSlotToHashImpl(int16_t dataBlockId, int16_t slotId, const char return taosHashPut(pHash, pName, len, &index, sizeof(SSlotIndex)); } -static int32_t putSlotToHash(int16_t dataBlockId, int16_t slotId, SNode* pNode, SHashObj* pHash) { - char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; - int32_t len = getSlotKey(pNode, NULL, name); - return putSlotToHashImpl(dataBlockId, slotId, name, len, pHash); +static int32_t putSlotToHash(const char* pName, int16_t dataBlockId, int16_t slotId, SNode* pNode, SHashObj* pHash) { + return putSlotToHashImpl(dataBlockId, slotId, pName, strlen(pName), pHash); } static int32_t createDataBlockDescHash(SPhysiPlanContext* pCxt, int32_t capacity, int16_t dataBlockId, @@ -131,9 +135,11 @@ static int32_t buildDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SD int16_t slotId = 0; SNode* pNode = NULL; FOREACH(pNode, pList) { - code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pNode, slotId, true, false)); + char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; + getSlotKey(pNode, NULL, name); + code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, name, pNode, slotId, true, false)); if (TSDB_CODE_SUCCESS == code) { - code = putSlotToHash(pDataBlockDesc->dataBlockId, slotId, pNode, pHash); + code = putSlotToHash(name, pDataBlockDesc->dataBlockId, slotId, pNode, pHash); } if (TSDB_CODE_SUCCESS == code) { pDataBlockDesc->totalRowSize += ((SExprNode*)pNode)->resType.bytes; @@ -196,7 +202,8 @@ static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList, int32_t len = getSlotKey(pExpr, pStmtName, name); SSlotIndex* pIndex = taosHashGet(pHash, name, len); if (NULL == pIndex) { - code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pExpr, nextSlotId, output, reserve)); + code = + nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, name, pExpr, nextSlotId, output, reserve)); if (TSDB_CODE_SUCCESS == code) { code = putSlotToHashImpl(pDataBlockDesc->dataBlockId, nextSlotId, name, len, pHash); } @@ -513,12 +520,13 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName); pTableScan->dataRequired = pScanLogicNode->dataRequired; pTableScan->pDynamicScanFuncs = nodesCloneList(pScanLogicNode->pDynamicScanFuncs); - pTableScan->pPartitionTags = nodesCloneList(pScanLogicNode->pPartTags); + pTableScan->pGroupTags = nodesCloneList(pScanLogicNode->pGroupTags); if ((NULL != pScanLogicNode->pDynamicScanFuncs && NULL == pTableScan->pDynamicScanFuncs) || - (NULL != pScanLogicNode->pPartTags && NULL == pTableScan->pPartitionTags)) { + (NULL != pScanLogicNode->pGroupTags && NULL == pTableScan->pGroupTags)) { nodesDestroyNode((SNode*)pTableScan); return TSDB_CODE_OUT_OF_MEMORY; } + pTableScan->groupSort = pScanLogicNode->groupSort; pTableScan->interval = pScanLogicNode->interval; pTableScan->offset = pScanLogicNode->offset; pTableScan->sliding = pScanLogicNode->sliding; @@ -1170,8 +1178,9 @@ static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildr static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SSortLogicNode* pSortLogicNode, SPhysiNode** pPhyNode) { - SSortPhysiNode* pSort = - (SSortPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pSortLogicNode, QUERY_NODE_PHYSICAL_PLAN_SORT); + SSortPhysiNode* pSort = (SSortPhysiNode*)makePhysiNode( + pCxt, (SLogicNode*)pSortLogicNode, + pSortLogicNode->groupSort ? QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT : QUERY_NODE_PHYSICAL_PLAN_SORT); if (NULL == pSort) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1185,7 +1194,7 @@ static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) { code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pSort->pExprs); if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockSlots(pCxt, pSort->pExprs, pChildTupe); + code = pushdownDataBlockSlots(pCxt, pSort->pExprs, pChildTupe); } } @@ -1322,6 +1331,7 @@ static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SMergeLogicNode* pM pMerge->numOfChannels = pMergeLogicNode->numOfChannels; pMerge->srcGroupId = pMergeLogicNode->srcGroupId; + pMerge->groupSort = pMergeLogicNode->groupSort; int32_t code = addDataBlockSlots(pCxt, pMergeLogicNode->pInputs, pMerge->node.pOutputDataBlockDesc); diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index b4a966f206..49ab50f913 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -362,7 +362,7 @@ static int32_t stbSplGetNumOfVgroups(SLogicNode* pNode) { } static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pSplitNode, - SNodeList* pMergeKeys, SLogicNode* pPartChild) { + SNodeList* pMergeKeys, SLogicNode* pPartChild, bool groupSort) { SMergeLogicNode* pMerge = (SMergeLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_MERGE); if (NULL == pMerge) { return TSDB_CODE_OUT_OF_MEMORY; @@ -371,6 +371,7 @@ static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubpla pMerge->srcGroupId = pCxt->groupId; pMerge->node.precision = pPartChild->precision; pMerge->pMergeKeys = pMergeKeys; + pMerge->groupSort = groupSort; int32_t code = TSDB_CODE_SUCCESS; pMerge->pInputs = nodesCloneList(pPartChild->pTargets); @@ -430,7 +431,7 @@ static int32_t stbSplSplitIntervalForBatch(SSplitContext* pCxt, SStableSplitInfo SNodeList* pMergeKeys = NULL; code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pInfo->pSplitNode)->pTspk, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { - code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, pMergeKeys, pPartWindow); + code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, pMergeKeys, pPartWindow, false); } if (TSDB_CODE_SUCCESS != code) { nodesDestroyList(pMergeKeys); @@ -497,12 +498,16 @@ static int32_t stbSplSplitSessionForStream(SSplitContext* pCxt, SStableSplitInfo return code; } -static void splSetTableScanType(SLogicNode* pNode, EScanType scanType) { +static void stbSplSetTableMergeScan(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { - ((SScanLogicNode*)pNode)->scanType = scanType; + SScanLogicNode* pScan = (SScanLogicNode*)pNode; + pScan->scanType = SCAN_TYPE_TABLE_MERGE; + if (NULL != pScan->pGroupTags) { + pScan->groupSort = true; + } } else { if (1 == LIST_LENGTH(pNode->pChildren)) { - splSetTableScanType((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), scanType); + stbSplSetTableMergeScan((SLogicNode*)nodesListGetNode(pNode->pChildren, 0)); } } } @@ -515,7 +520,7 @@ static int32_t stbSplSplitSessionOrStateForBatch(SSplitContext* pCxt, SStableSpl int32_t code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pWindow)->pTspk, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { - code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pChild, pMergeKeys, (SLogicNode*)pChild); + code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pChild, pMergeKeys, (SLogicNode*)pChild, true); } if (TSDB_CODE_SUCCESS == code) { @@ -524,13 +529,10 @@ static int32_t stbSplSplitSessionOrStateForBatch(SSplitContext* pCxt, SStableSpl } if (TSDB_CODE_SUCCESS == code) { - splSetTableScanType(pChild, SCAN_TYPE_TABLE_MERGE); - ++(pCxt->groupId); - } - - if (TSDB_CODE_SUCCESS == code) { + stbSplSetTableMergeScan(pChild); pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE; SPLIT_FLAG_SET_MASK(pInfo->pSubplan->splitFlag, SPLIT_FLAG_STABLE_SPLIT); + ++(pCxt->groupId); } else { nodesDestroyList(pMergeKeys); } @@ -560,7 +562,7 @@ static int32_t stbSplSplitState(SSplitContext* pCxt, SStableSplitInfo* pInfo) { static SNodeList* stbSplGetPartKeys(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { - return ((SScanLogicNode*)pNode)->pPartTags; + return ((SScanLogicNode*)pNode)->pGroupTags; } else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { return ((SPartitionLogicNode*)pNode)->pPartitionKeys; } else { @@ -775,6 +777,7 @@ static int32_t stbSplCreatePartSortNode(SSortLogicNode* pSort, SLogicNode** pOut pPartSort->node.pChildren = pChildren; splSetParent((SLogicNode*)pPartSort); pPartSort->pSortKeys = pSortKeys; + pPartSort->groupSort = pSort->groupSort; code = stbSplCreateMergeKeys(pPartSort->pSortKeys, pPartSort->node.pTargets, &pMergeKeys); } @@ -789,12 +792,29 @@ static int32_t stbSplCreatePartSortNode(SSortLogicNode* pSort, SLogicNode** pOut return code; } +static void stbSplSetScanPartSort(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + SScanLogicNode* pScan = (SScanLogicNode*)pNode; + if (NULL != pScan->pGroupTags) { + pScan->groupSort = true; + } + } else { + if (1 == LIST_LENGTH(pNode->pChildren)) { + stbSplSetScanPartSort((SLogicNode*)nodesListGetNode(pNode->pChildren, 0)); + } + } +} + static int32_t stbSplSplitSortNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { SLogicNode* pPartSort = NULL; SNodeList* pMergeKeys = NULL; + bool groupSort = ((SSortLogicNode*)pInfo->pSplitNode)->groupSort; int32_t code = stbSplCreatePartSortNode((SSortLogicNode*)pInfo->pSplitNode, &pPartSort, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { - code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, pMergeKeys, pPartSort); + code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, pMergeKeys, pPartSort, groupSort); + } + if (TSDB_CODE_SUCCESS == code && groupSort) { + stbSplSetScanPartSort(pPartSort); } if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, @@ -829,7 +849,7 @@ static int32_t stbSplSplitScanNodeForJoin(SSplitContext* pCxt, SLogicSubplan* pS SNodeList* pMergeKeys = NULL; int32_t code = stbSplCreateMergeKeysByPrimaryKey(stbSplFindPrimaryKeyFromScan(pScan), &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { - code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, (SLogicNode*)pScan); + code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, (SLogicNode*)pScan, false); } if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&pSubplan->pChildren, diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp index ea40567dae..8c19b52a09 100644 --- a/source/libs/planner/test/planBasicTest.cpp +++ b/source/libs/planner/test/planBasicTest.cpp @@ -67,6 +67,14 @@ TEST_F(PlanBasicTest, tailFunc) { run("SELECT TAIL(c1, 10) FROM t1"); run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10"); + + run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 PARTITION BY c1"); + + run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 ORDER BY 1"); + + run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 LIMIT 5"); + + run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 PARTITION BY c1 LIMIT 5"); } TEST_F(PlanBasicTest, interpFunc) { diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index 43b6bf5772..46c2f33048 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -76,6 +76,7 @@ static void parseArg(int argc, char* argv[]) { static struct option long_options[] = { {"dump", optional_argument, NULL, 'd'}, {"skipSql", required_argument, NULL, 's'}, + {"limitSql", required_argument, NULL, 'i'}, {"log", required_argument, NULL, 'l'}, {0, 0, 0, 0} }; @@ -88,6 +89,9 @@ static void parseArg(int argc, char* argv[]) { case 's': setSkipSqlNum(optarg); break; + case 'i': + setLimitSqlNum(optarg); + break; case 'l': setLogLevel(optarg); break; diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index c197a02dd1..d54859c296 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -51,6 +51,7 @@ enum DumpModule { DumpModule g_dumpModule = DUMP_MODULE_NOTHING; int32_t g_skipSql = 0; +int32_t g_limitSql = 0; int32_t g_logLevel = 131; void setDumpModule(const char* pModule) { @@ -76,28 +77,33 @@ void setDumpModule(const char* pModule) { } void setSkipSqlNum(const char* pNum) { g_skipSql = stoi(pNum); } - +void setLimitSqlNum(const char* pNum) { g_limitSql = stoi(pNum); } void setLogLevel(const char* pLogLevel) { g_logLevel = stoi(pLogLevel); } int32_t getLogLevel() { return g_logLevel; } class PlannerTestBaseImpl { public: - PlannerTestBaseImpl() : sqlNo_(0) {} + PlannerTestBaseImpl() : sqlNo_(0), sqlNum_(0) {} void useDb(const string& user, const string& db) { caseEnv_.acctId_ = 0; caseEnv_.user_ = user; caseEnv_.db_ = db; - caseEnv_.nsql_ = g_skipSql; + caseEnv_.numOfSkipSql_ = g_skipSql; + caseEnv_.numOfLimitSql_ = g_limitSql; } void run(const string& sql) { ++sqlNo_; - if (caseEnv_.nsql_ > 0) { - --(caseEnv_.nsql_); + if (caseEnv_.numOfSkipSql_ > 0) { + --(caseEnv_.numOfSkipSql_); return; } + if (caseEnv_.numOfLimitSql_ > 0 && caseEnv_.numOfLimitSql_ == sqlNum_) { + return; + } + ++sqlNum_; reset(); try { @@ -134,7 +140,7 @@ class PlannerTestBaseImpl { } void prepare(const string& sql) { - if (caseEnv_.nsql_ > 0) { + if (caseEnv_.numOfSkipSql_ > 0) { return; } @@ -148,7 +154,7 @@ class PlannerTestBaseImpl { } void bindParams(TAOS_MULTI_BIND* pParams, int32_t colIdx) { - if (caseEnv_.nsql_ > 0) { + if (caseEnv_.numOfSkipSql_ > 0) { return; } @@ -161,8 +167,8 @@ class PlannerTestBaseImpl { } void exec() { - if (caseEnv_.nsql_ > 0) { - --(caseEnv_.nsql_); + if (caseEnv_.numOfSkipSql_ > 0) { + --(caseEnv_.numOfSkipSql_); return; } @@ -197,9 +203,10 @@ class PlannerTestBaseImpl { int32_t acctId_; string user_; string db_; - int32_t nsql_; + int32_t numOfSkipSql_; + int32_t numOfLimitSql_; - caseEnv() : nsql_(0) {} + caseEnv() : numOfSkipSql_(0) {} }; struct stmtEnv { @@ -401,6 +408,7 @@ class PlannerTestBaseImpl { stmtEnv stmtEnv_; stmtRes res_; int32_t sqlNo_; + int32_t sqlNum_; }; PlannerTestBase::PlannerTestBase() : impl_(new PlannerTestBaseImpl()) {} diff --git a/source/libs/planner/test/planTestUtil.h b/source/libs/planner/test/planTestUtil.h index 7d2a9e533f..f9942c93a7 100644 --- a/source/libs/planner/test/planTestUtil.h +++ b/source/libs/planner/test/planTestUtil.h @@ -43,6 +43,7 @@ class PlannerTestBase : public testing::Test { extern void setDumpModule(const char* pModule); extern void setSkipSqlNum(const char* pNum); +extern void setLimitSqlNum(const char* pNum); extern void setLogLevel(const char* pLogLevel); extern int32_t getLogLevel(); diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index 2e8e0e1227..beddec64c5 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -28,13 +28,13 @@ extern "C" { #include "trpc.h" #include "ttimer.h" -#define TIMER_MAX_MS 0x7FFFFFFF -#define ENV_TICK_TIMER_MS 1000 -#define PING_TIMER_MS 1000 -#define ELECT_TIMER_MS_MIN 1300 -#define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2) +#define TIMER_MAX_MS 0x7FFFFFFF +#define ENV_TICK_TIMER_MS 1000 +#define PING_TIMER_MS 1000 +#define ELECT_TIMER_MS_MIN 1300 +#define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2) #define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN) -#define HEARTBEAT_TIMER_MS 900 +#define HEARTBEAT_TIMER_MS 900 #define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0}) diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index 88d3065375..f3064e668d 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -50,7 +50,7 @@ typedef struct SSyncIO { void *pSyncNode; int32_t (*FpOnSyncPing)(SSyncNode *pSyncNode, SyncPing *pMsg); int32_t (*FpOnSyncPingReply)(SSyncNode *pSyncNode, SyncPingReply *pMsg); - int32_t (*FpOnSyncClientRequest)(SSyncNode *pSyncNode, SyncClientRequest *pMsg); + int32_t (*FpOnSyncClientRequest)(SSyncNode *pSyncNode, SyncClientRequest *pMsg, SyncIndex *pRetIndex); int32_t (*FpOnSyncRequestVote)(SSyncNode *pSyncNode, SyncRequestVote *pMsg); int32_t (*FpOnSyncRequestVoteReply)(SSyncNode *pSyncNode, SyncRequestVoteReply *pMsg); int32_t (*FpOnSyncAppendEntries)(SSyncNode *pSyncNode, SyncAppendEntries *pMsg); diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 1bd0671fb5..d351dc50f4 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -169,7 +169,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); void syncNodeStart(SSyncNode* pSyncNode); void syncNodeStartStandBy(SSyncNode* pSyncNode); void syncNodeClose(SSyncNode* pSyncNode); -int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak); +int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak); // option bool syncNodeSnapshotEnable(SSyncNode* pSyncNode); @@ -233,6 +233,7 @@ SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index); SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index); int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm); +bool syncNodeIsOptimizedOneReplica(SSyncNode* ths, SRpcMsg* pMsg); int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag); int32_t syncNodeUpdateNewConfigIndex(SSyncNode* ths, SSyncCfg* pNewCfg); diff --git a/source/libs/sync/inc/syncRaftCfg.h b/source/libs/sync/inc/syncRaftCfg.h index efe8a65b77..521ca6068d 100644 --- a/source/libs/sync/inc/syncRaftCfg.h +++ b/source/libs/sync/inc/syncRaftCfg.h @@ -49,14 +49,14 @@ int32_t raftCfgClose(SRaftCfg *pRaftCfg); int32_t raftCfgPersist(SRaftCfg *pRaftCfg); int32_t raftCfgAddConfigIndex(SRaftCfg *pRaftCfg, SyncIndex configIndex); -cJSON *syncCfg2Json(SSyncCfg *pSyncCfg); -char *syncCfg2Str(SSyncCfg *pSyncCfg); -char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg); +cJSON * syncCfg2Json(SSyncCfg *pSyncCfg); +char * syncCfg2Str(SSyncCfg *pSyncCfg); +char * syncCfg2SimpleStr(SSyncCfg *pSyncCfg); int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg); int32_t syncCfgFromStr(const char *s, SSyncCfg *pSyncCfg); -cJSON *raftCfg2Json(SRaftCfg *pRaftCfg); -char *raftCfg2Str(SRaftCfg *pRaftCfg); +cJSON * raftCfg2Json(SRaftCfg *pRaftCfg); +char * raftCfg2Str(SRaftCfg *pRaftCfg); int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg); int32_t raftCfgFromStr(const char *s, SRaftCfg *pRaftCfg); diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index b130186516..37f18a6388 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -56,6 +56,31 @@ void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj); void syncEntryLog(const SSyncRaftEntry* pObj); void syncEntryLog2(char* s, const SSyncRaftEntry* pObj); +//----------------------------------- +typedef struct SRaftEntryCache { + SHashObj* pEntryHash; + int32_t maxCount; + int32_t currentCount; + TdThreadMutex mutex; + SSyncNode* pSyncNode; +} SRaftEntryCache; + +SRaftEntryCache* raftCacheCreate(SSyncNode* pSyncNode, int32_t maxCount); +void raftCacheDestroy(SRaftEntryCache* pCache); +int32_t raftCachePutEntry(struct SRaftEntryCache* pCache, SSyncRaftEntry* pEntry); +int32_t raftCacheGetEntry(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); +int32_t raftCacheGetEntryP(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); +int32_t raftCacheDelEntry(struct SRaftEntryCache* pCache, SyncIndex index); +int32_t raftCacheGetAndDel(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); +int32_t raftCacheClear(struct SRaftEntryCache* pCache); + +cJSON* raftCache2Json(SRaftEntryCache* pObj); +char* raftCache2Str(SRaftEntryCache* pObj); +void raftCachePrint(SRaftEntryCache* pObj); +void raftCachePrint2(char* s, SRaftEntryCache* pObj); +void raftCacheLog(SRaftEntryCache* pObj); +void raftCacheLog2(char* s, SRaftEntryCache* pObj); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index f124867097..f3ed9e302b 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -29,9 +29,12 @@ extern "C" { #include "wal.h" typedef struct SSyncLogStoreData { - SSyncNode* pSyncNode; - SWal* pWal; + SSyncNode* pSyncNode; + SWal* pWal; + + TdThreadMutex mutex; SWalReadHandle* pWalHandle; + // SyncIndex beginIndex; // valid begin index, default 0, may be set beginIndex > 0 } SSyncLogStoreData; diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index f00dfe6f50..55e0755a89 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -35,12 +35,13 @@ extern "C" { #define SYNC_SNAPSHOT_RETRY_MS 5000 +//--------------------------------------------------- typedef struct SSyncSnapshotSender { bool start; int32_t seq; int32_t ack; - void * pReader; - void * pCurrentBlock; + void *pReader; + void *pCurrentBlock; int32_t blockLen; SSnapshot snapshot; SSyncCfg lastConfig; @@ -59,33 +60,36 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot void snapshotSenderStop(SSyncSnapshotSender *pSender); int32_t snapshotSend(SSyncSnapshotSender *pSender); int32_t snapshotReSend(SSyncSnapshotSender *pSender); -cJSON * snapshotSender2Json(SSyncSnapshotSender *pSender); -char * snapshotSender2Str(SSyncSnapshotSender *pSender); -char * snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event); +cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); +char *snapshotSender2Str(SSyncSnapshotSender *pSender); +char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event); + +//--------------------------------------------------- typedef struct SSyncSnapshotReceiver { - bool start; - - int32_t ack; - void * pWriter; - SyncTerm term; - SyncTerm privateTerm; - SSnapshot snapshot; - - SSyncNode *pSyncNode; + bool start; + int32_t ack; + void *pWriter; + SyncTerm term; + SyncTerm privateTerm; + SSnapshot snapshot; SRaftId fromId; + SSyncNode *pSyncNode; } SSyncSnapshotReceiver; SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId); void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver); -void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg); -bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); -void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply); -cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); -char * snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); -char * snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event); +void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg); +bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); +void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply); +cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); +char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); +char *snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event); + +//--------------------------------------------------- +// on message int32_t syncNodeOnSnapshotSendCb(SSyncNode *ths, SyncSnapshotSend *pMsg); int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg); diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index c92edae381..982d37826a 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -102,6 +102,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { } } + // maybe execute fsm if (newCommitIndex > pSyncNode->commitIndex) { SyncIndex beginIndex = pSyncNode->commitIndex + 1; SyncIndex endIndex = newCommitIndex; diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 038c36c417..663745a7d7 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -30,7 +30,7 @@ static int32_t syncIODestroy(SSyncIO *io); static int32_t syncIOStartInternal(SSyncIO *io); static int32_t syncIOStopInternal(SSyncIO *io); -static void *syncIOConsumerFunc(void *param); +static void * syncIOConsumerFunc(void *param); static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); @@ -242,9 +242,9 @@ static int32_t syncIOStopInternal(SSyncIO *io) { } static void *syncIOConsumerFunc(void *param) { - SSyncIO *io = param; + SSyncIO * io = param; STaosQall *qall; - SRpcMsg *pRpcMsg, rpcMsg; + SRpcMsg * pRpcMsg, rpcMsg; qall = taosAllocateQall(); while (1) { @@ -281,7 +281,7 @@ static void *syncIOConsumerFunc(void *param) { if (io->FpOnSyncClientRequest != NULL) { SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg); ASSERT(pSyncMsg != NULL); - io->FpOnSyncClientRequest(io->pSyncNode, pSyncMsg); + io->FpOnSyncClientRequest(io->pSyncNode, pSyncMsg, NULL); syncClientRequestDestroy(pSyncMsg); } diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index 5b432aeec4..10f0e0e335 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -126,7 +126,7 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr) { cJSON *pJson = syncIndexMgr2Json(pSyncIndexMgr); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 3100d0525c..60f6ca7e7b 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -50,7 +50,7 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths); // process message ---- int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); -int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg); +int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex); // life cycle static void syncFreeNode(void* param); @@ -627,7 +627,7 @@ int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak) { return ret; } -int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak) { +int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { int32_t ret = 0; char eventLog[128]; @@ -664,13 +664,34 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak) SRpcMsg rpcMsg; syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); - if (pSyncNode->FpEqMsg != NULL && (*pSyncNode->FpEqMsg)(pSyncNode->msgcb, &rpcMsg) == 0) { - ret = 0; + // optimized one replica + if (syncNodeIsOptimizedOneReplica(pSyncNode, pMsg)) { + SyncIndex retIndex; + int32_t code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg, &retIndex); + if (code == 0) { + pMsg->info.conn.applyIndex = retIndex; + rpcFreeCont(rpcMsg.pCont); + syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum); + ret = 1; + sDebug("vgId:%d optimized index:%ld success, msgtype:%s,%d", pSyncNode->vgId, retIndex, + TMSG_INFO(pMsg->msgType), pMsg->msgType); + } else { + ret = -1; + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + sError("vgId:%d optimized index:%ld error, msgtype:%s,%d", pSyncNode->vgId, retIndex, TMSG_INFO(pMsg->msgType), + pMsg->msgType); + } + } else { - ret = -1; - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - sError("syncPropose pSyncNode->FpEqMsg is NULL"); + if (pSyncNode->FpEqMsg != NULL && (*pSyncNode->FpEqMsg)(pSyncNode->msgcb, &rpcMsg) == 0) { + ret = 0; + } else { + ret = -1; + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + sError("enqueue msg error, FpEqMsg is NULL"); + } } + syncClientRequestDestroy(pSyncMsg); goto _END; @@ -1334,11 +1355,16 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { int32_t userStrLen = strlen(str); SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; - if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { + if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); } - SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); - SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); + + SyncIndex logLastIndex = SYNC_INDEX_INVALID; + SyncIndex logBeginIndex = SYNC_INDEX_INVALID; + if (pSyncNode->pLogStore != NULL) { + logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); + logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); + } char* pCfgStr = syncCfg2SimpleStr(&(pSyncNode->pRaftCfg->cfg)); char* printStr = ""; @@ -2377,7 +2403,7 @@ int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { // /\ UNCHANGED <> // -int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) { +int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex) { int32_t ret = 0; syncClientRequestLog2("==syncNodeOnClientRequestCb==", pMsg); @@ -2436,6 +2462,14 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) { rpcFreeCont(rpcMsg.pCont); } + if (pRetIndex != NULL) { + if (ret == 0 && pEntry != NULL) { + *pRetIndex = pEntry->index; + } else { + *pRetIndex = SYNC_INDEX_INVALID; + } + } + syncEntryDestory(pEntry); return ret; } @@ -2600,6 +2634,10 @@ static int32_t syncNodeProposeConfigChangeFinish(SSyncNode* ths, SyncReconfigFin return 0; } +bool syncNodeIsOptimizedOneReplica(SSyncNode* ths, SRpcMsg* pMsg) { + return (ths->replicaNum == 1 && syncUtilUserCommit(pMsg->msgType) && ths->vgId != 1); +} + int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag) { int32_t code = 0; ESyncState state = flag; @@ -2621,19 +2659,33 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, syncEntry2OriginalRpc(pEntry, &rpcMsg); // user commit - if (ths->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) { - SFsmCbMeta cbMeta = {0}; - cbMeta.index = pEntry->index; - cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index); - cbMeta.isWeak = pEntry->isWeak; - cbMeta.code = 0; - cbMeta.state = ths->state; - cbMeta.seqNum = pEntry->seqNum; - cbMeta.term = pEntry->term; - cbMeta.currentTerm = ths->pRaftStore->currentTerm; - cbMeta.flag = flag; + if ((ths->pFsm->FpCommitCb != NULL) && syncUtilUserCommit(pEntry->originalRpcType)) { + bool internalExecute = true; + if ((ths->replicaNum == 1) && ths->restoreFinish && (ths->vgId != 1)) { + internalExecute = false; + } - ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, cbMeta); + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "index:%ld, internalExecute:%d", i, internalExecute); + syncNodeEventLog(ths, logBuf); + } while (0); + + // execute fsm in apply thread, or execute outside syncPropose + if (internalExecute) { + SFsmCbMeta cbMeta = {0}; + cbMeta.index = pEntry->index; + cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index); + cbMeta.isWeak = pEntry->isWeak; + cbMeta.code = 0; + cbMeta.state = ths->state; + cbMeta.seqNum = pEntry->seqNum; + cbMeta.term = pEntry->term; + cbMeta.currentTerm = ths->pRaftStore->currentTerm; + cbMeta.flag = flag; + + ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, cbMeta); + } } // config change diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index f40eff57d9..9d16bed6c1 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -101,7 +101,7 @@ cJSON *syncCfg2Json(SSyncCfg *pSyncCfg) { char *syncCfg2Str(SSyncCfg *pSyncCfg) { cJSON *pJson = syncCfg2Json(pSyncCfg); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -109,7 +109,7 @@ char *syncCfg2Str(SSyncCfg *pSyncCfg) { char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) { if (pSyncCfg != NULL) { int32_t len = 512; - char *s = taosMemoryMalloc(len); + char * s = taosMemoryMalloc(len); memset(s, 0, len); snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex); @@ -205,7 +205,7 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) { char *raftCfg2Str(SRaftCfg *pRaftCfg) { cJSON *pJson = raftCfg2Json(pRaftCfg); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -271,7 +271,7 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) { (pRaftCfg->configIndexArr)[i] = atoll(pIndex->valuestring); } - cJSON *pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); + cJSON * pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg)); ASSERT(code == 0); diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index ff334a76bb..225360630c 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -180,3 +180,247 @@ void syncEntryLog2(char* s, const SSyncRaftEntry* pObj) { sTrace("syncEntryLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } + +//----------------------------------- +SRaftEntryCache* raftCacheCreate(SSyncNode* pSyncNode, int32_t maxCount) { + SRaftEntryCache* pCache = taosMemoryMalloc(sizeof(SRaftEntryCache)); + if (pCache == NULL) { + sError("vgId:%d raft cache create error", pSyncNode->vgId); + return NULL; + } + + pCache->pEntryHash = + taosHashInit(sizeof(SyncIndex), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (pCache->pEntryHash == NULL) { + sError("vgId:%d raft cache create hash error", pSyncNode->vgId); + return NULL; + } + + taosThreadMutexInit(&(pCache->mutex), NULL); + pCache->maxCount = maxCount; + pCache->currentCount = 0; + pCache->pSyncNode = pSyncNode; + + return pCache; +} + +void raftCacheDestroy(SRaftEntryCache* pCache) { + if (pCache != NULL) { + taosThreadMutexLock(&(pCache->mutex)); + taosHashCleanup(pCache->pEntryHash); + taosThreadMutexUnlock(&(pCache->mutex)); + taosThreadMutexDestroy(&(pCache->mutex)); + taosMemoryFree(pCache); + } +} + +// success, return 1 +// max count, return 0 +// error, return -1 +int32_t raftCachePutEntry(struct SRaftEntryCache* pCache, SSyncRaftEntry* pEntry) { + taosThreadMutexLock(&(pCache->mutex)); + + if (pCache->currentCount >= pCache->maxCount) { + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; + } + + taosHashPut(pCache->pEntryHash, &(pEntry->index), sizeof(pEntry->index), pEntry, pEntry->bytes); + ++(pCache->currentCount); + + do { + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "raft cache add, type:%s,%d, type2:%s,%d, index:%ld, bytes:%d", + TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType, + pEntry->index, pEntry->bytes); + syncNodeEventLog(pCache->pSyncNode, eventLog); + } while (0); + + taosThreadMutexUnlock(&(pCache->mutex)); + return 1; +} + +// success, return 0 +// error, return -1 +// not exist, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST +int32_t raftCacheGetEntry(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) { + if (ppEntry == NULL) { + return -1; + } + *ppEntry = NULL; + + taosThreadMutexLock(&(pCache->mutex)); + void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index)); + if (pTmp != NULL) { + SSyncRaftEntry* pEntry = pTmp; + *ppEntry = taosMemoryMalloc(pEntry->bytes); + memcpy(*ppEntry, pTmp, pEntry->bytes); + + do { + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "raft cache get, type:%s,%d, type2:%s,%d, index:%ld", + TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType), + (*ppEntry)->originalRpcType, (*ppEntry)->index); + syncNodeEventLog(pCache->pSyncNode, eventLog); + } while (0); + + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; + } + + taosThreadMutexUnlock(&(pCache->mutex)); + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + return -1; +} + +// success, return 0 +// error, return -1 +// not exist, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST +int32_t raftCacheGetEntryP(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) { + if (ppEntry == NULL) { + return -1; + } + *ppEntry = NULL; + + taosThreadMutexLock(&(pCache->mutex)); + void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index)); + if (pTmp != NULL) { + SSyncRaftEntry* pEntry = pTmp; + *ppEntry = pEntry; + + do { + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "raft cache get, type:%s,%d, type2:%s,%d, index:%ld", + TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType), + (*ppEntry)->originalRpcType, (*ppEntry)->index); + syncNodeEventLog(pCache->pSyncNode, eventLog); + } while (0); + + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; + } + + taosThreadMutexUnlock(&(pCache->mutex)); + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + return -1; +} + +int32_t raftCacheDelEntry(struct SRaftEntryCache* pCache, SyncIndex index) { + taosThreadMutexLock(&(pCache->mutex)); + taosHashRemove(pCache->pEntryHash, &index, sizeof(index)); + --(pCache->currentCount); + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; +} + +int32_t raftCacheGetAndDel(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) { + if (ppEntry == NULL) { + return -1; + } + *ppEntry = NULL; + + taosThreadMutexLock(&(pCache->mutex)); + void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index)); + if (pTmp != NULL) { + SSyncRaftEntry* pEntry = pTmp; + *ppEntry = taosMemoryMalloc(pEntry->bytes); + memcpy(*ppEntry, pTmp, pEntry->bytes); + + do { + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "raft cache get-and-del, type:%s,%d, type2:%s,%d, index:%ld", + TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType), + (*ppEntry)->originalRpcType, (*ppEntry)->index); + syncNodeEventLog(pCache->pSyncNode, eventLog); + } while (0); + + taosHashRemove(pCache->pEntryHash, &index, sizeof(index)); + --(pCache->currentCount); + + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; + } + + taosThreadMutexUnlock(&(pCache->mutex)); + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + return -1; +} + +int32_t raftCacheClear(struct SRaftEntryCache* pCache) { + taosThreadMutexLock(&(pCache->mutex)); + taosHashClear(pCache->pEntryHash); + pCache->currentCount = 0; + taosThreadMutexUnlock(&(pCache->mutex)); + return 0; +} + +//----------------------------------- +cJSON* raftCache2Json(SRaftEntryCache* pCache) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pCache != NULL) { + taosThreadMutexLock(&(pCache->mutex)); + + snprintf(u64buf, sizeof(u64buf), "%p", pCache->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + cJSON_AddNumberToObject(pRoot, "currentCount", pCache->currentCount); + cJSON_AddNumberToObject(pRoot, "maxCount", pCache->maxCount); + cJSON* pEntries = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "entries", pEntries); + + SSyncRaftEntry* pIter = (SSyncRaftEntry*)taosHashIterate(pCache->pEntryHash, NULL); + if (pIter != NULL) { + SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter; + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + } + while (pIter) { + pIter = taosHashIterate(pCache->pEntryHash, pIter); + if (pIter != NULL) { + SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter; + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + } + } + + taosThreadMutexUnlock(&(pCache->mutex)); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SRaftEntryCache", pRoot); + return pJson; +} + +char* raftCache2Str(SRaftEntryCache* pCache) { + cJSON* pJson = raftCache2Json(pCache); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +void raftCachePrint(SRaftEntryCache* pCache) { + char* serialized = raftCache2Str(pCache); + printf("raftCachePrint | len:%lu | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftCachePrint2(char* s, SRaftEntryCache* pCache) { + char* serialized = raftCache2Str(pCache); + printf("raftCachePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftCacheLog(SRaftEntryCache* pCache) { + char* serialized = raftCache2Str(pCache); + sTrace("raftCacheLog | len:%lu | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void raftCacheLog2(char* s, SRaftEntryCache* pCache) { + if (gRaftDetailLog) { + char* serialized = raftCache2Str(pCache); + sTraceLong("raftCacheLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} \ No newline at end of file diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index cc606f4df8..d84c576714 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -257,6 +257,8 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, return -1; } + taosThreadMutexLock(&(pData->mutex)); + code = walReadWithHandle(pWalHandle, index); if (code != 0) { int32_t err = terrno; @@ -281,6 +283,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, terrno = saveErr; */ + taosThreadMutexUnlock(&(pData->mutex)); return code; } @@ -301,6 +304,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, terrno = saveErr; */ + taosThreadMutexUnlock(&(pData->mutex)); return code; } @@ -364,6 +368,7 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { pData->pWal = pSyncNode->pWal; ASSERT(pData->pWal != NULL); + taosThreadMutexInit(&(pData->mutex), NULL); pData->pWalHandle = walOpenReadHandle(pData->pWal); ASSERT(pData->pWalHandle != NULL); @@ -408,9 +413,14 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { void logStoreDestory(SSyncLogStore* pLogStore) { if (pLogStore != NULL) { SSyncLogStoreData* pData = pLogStore->data; + + taosThreadMutexLock(&(pData->mutex)); if (pData->pWalHandle != NULL) { walCloseReadHandle(pData->pWalHandle); + pData->pWalHandle = NULL; } + taosThreadMutexUnlock(&(pData->mutex)); + taosThreadMutexDestroy(&(pData->mutex)); taosMemoryFree(pLogStore->data); taosMemoryFree(pLogStore); @@ -460,6 +470,8 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { SWal* pWal = pData->pWal; if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) { + taosThreadMutexLock(&(pData->mutex)); + // SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); SWalReadHandle* pWalHandle = pData->pWalHandle; ASSERT(pWalHandle != NULL); @@ -503,6 +515,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { terrno = saveErr; */ + taosThreadMutexUnlock(&(pData->mutex)); return pEntry; } else { diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index a1ab95c00f..9f5cba6c66 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -216,7 +216,7 @@ cJSON *raftStore2Json(SRaftStore *pRaftStore) { char *raftStore2Str(SRaftStore *pRaftStore) { cJSON *pJson = raftStore2Json(pRaftStore); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index 48c1b70a04..d2cbabe226 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -32,11 +32,13 @@ SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) { } void syncRespMgrDestroy(SSyncRespMgr *pObj) { - taosThreadMutexLock(&(pObj->mutex)); - taosHashCleanup(pObj->pRespHash); - taosThreadMutexUnlock(&(pObj->mutex)); - taosThreadMutexDestroy(&(pObj->mutex)); - taosMemoryFree(pObj); + if (pObj != NULL) { + taosThreadMutexLock(&(pObj->mutex)); + taosHashCleanup(pObj->pRespHash); + taosThreadMutexUnlock(&(pObj->mutex)); + taosThreadMutexDestroy(&(pObj->mutex)); + taosMemoryFree(pObj); + } } int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub) { diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 4fc0128400..39e972c4c4 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -21,6 +21,7 @@ #include "syncUtil.h" #include "wal.h" +//---------------------------------- static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg); @@ -49,7 +50,7 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI pSender->pSyncNode->pFsm->FpGetSnapshotInfo(pSender->pSyncNode->pFsm, &(pSender->snapshot)); pSender->finish = false; } else { - sError("snapshotSenderCreate cannot create sender"); + sError("vgId:%d cannot create snapshot sender", pSyncNode->vgId); } return pSender; @@ -57,39 +58,59 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { if (pSender != NULL) { + // free current block if (pSender->pCurrentBlock != NULL) { taosMemoryFree(pSender->pCurrentBlock); + pSender->pCurrentBlock = NULL; } + + // close reader + if (pSender->pReader != NULL) { + int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader); + ASSERT(ret == 0); + pSender->pReader = NULL; + } + + // free sender taosMemoryFree(pSender); } } bool snapshotSenderIsStart(SSyncSnapshotSender *pSender) { return pSender->start; } -// begin send snapshot (current term, seq begin) +// begin send snapshot by snapshot, pReader void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void *pReader) { ASSERT(!snapshotSenderIsStart(pSender)); - pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; - pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; - // init snapshot and reader ASSERT(pSender->pReader == NULL); pSender->pReader = pReader; pSender->snapshot = snapshot; + // init current block if (pSender->pCurrentBlock != NULL) { taosMemoryFree(pSender->pCurrentBlock); } pSender->blockLen = 0; + // update term + pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; + ++(pSender->privateTerm); + + // update state + pSender->finish = false; + pSender->start = true; + pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; + pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; + + // init last config if (pSender->snapshot.lastConfigIndex != SYNC_INDEX_INVALID) { int32_t code = 0; SSyncRaftEntry *pEntry = NULL; + bool getLastConfig = false; + code = pSender->pSyncNode->pLogStore->syncLogGetEntry(pSender->pSyncNode->pLogStore, pSender->snapshot.lastConfigIndex, &pEntry); - - bool getLastConfig = false; if (code == 0) { ASSERT(pEntry != NULL); @@ -112,29 +133,29 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void } } + // last config not found in wal, update to -1 if (!getLastConfig) { - char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "snapshot sender update lcindex from %ld to -1", - pSender->snapshot.lastConfigIndex); - pSender->snapshot.lastConfigIndex = -1; - - char *eventLog = snapshotSender2SimpleStr(pSender, logBuf); - syncNodeEventLog(pSender->pSyncNode, eventLog); - taosMemoryFree(eventLog); - + SyncIndex oldLastConfigIndex = pSender->snapshot.lastConfigIndex; + SyncIndex newLastConfigIndex = SYNC_INDEX_INVALID; + pSender->snapshot.lastConfigIndex = SYNC_INDEX_INVALID; memset(&(pSender->lastConfig), 0, sizeof(SSyncCfg)); + + // event log + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "snapshot sender update lcindex from %ld to %ld", oldLastConfigIndex, + newLastConfigIndex); + char *eventLog = snapshotSender2SimpleStr(pSender, logBuf); + syncNodeEventLog(pSender->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); } } else { + // no last config memset(&(pSender->lastConfig), 0, sizeof(SSyncCfg)); } - pSender->sendingMS = SYNC_SNAPSHOT_RETRY_MS; - pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; - ++(pSender->privateTerm); - pSender->finish = false; - pSender->start = true; - // build begin msg SyncSnapshotSend *pMsg = syncSnapshotSendBuild(0, pSender->pSyncNode->vgId); pMsg->srcId = pSender->pSyncNode->myRaftId; @@ -151,40 +172,39 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void SRpcMsg rpcMsg; syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); - - char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender send"); - syncNodeEventLog(pSender->pSyncNode, eventLog); - taosMemoryFree(eventLog); - syncSnapshotSendDestroy(pMsg); + + // event log + do { + char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender send"); + syncNodeEventLog(pSender->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); } void snapshotSenderStop(SSyncSnapshotSender *pSender) { + // close reader if (pSender->pReader != NULL) { int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader); ASSERT(ret == 0); pSender->pReader = NULL; } + // free current block if (pSender->pCurrentBlock != NULL) { taosMemoryFree(pSender->pCurrentBlock); pSender->pCurrentBlock = NULL; pSender->blockLen = 0; } + // update flag pSender->start = false; - - if (gRaftDetailLog) { - char *s = snapshotSender2Str(pSender); - sInfo("snapshotSenderStop %s", s); - taosMemoryFree(s); - } } -// when sender receiver ack, call this function to send msg from seq +// when sender receive ack, call this function to send msg from seq // seq = ack + 1, already updated int32_t snapshotSend(SSyncSnapshotSender *pSender) { - // free memory last time (seq - 1) + // free memory last time (current seq - 1) if (pSender->pCurrentBlock != NULL) { taosMemoryFree(pSender->pCurrentBlock); pSender->pCurrentBlock = NULL; @@ -198,7 +218,7 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { if (pSender->blockLen > 0) { // has read data } else { - // read finish + // read finish, update seq to end pSender->seq = SYNC_SNAPSHOT_SEQ_END; } @@ -219,25 +239,28 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { SRpcMsg rpcMsg; syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); - - if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) { - char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender finish"); - syncNodeEventLog(pSender->pSyncNode, eventLog); - taosMemoryFree(eventLog); - - } else { - char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender sending"); - syncNodeEventLog(pSender->pSyncNode, eventLog); - taosMemoryFree(eventLog); - } - syncSnapshotSendDestroy(pMsg); + + // event log + do { + char *eventLog = NULL; + if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) { + eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender finish"); + } else { + eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender sending"); + } + syncNodeEventLog(pSender->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); + return 0; } // send snapshot data from cache int32_t snapshotReSend(SSyncSnapshotSender *pSender) { - if (pSender->pCurrentBlock != NULL) { + // send current block data + if (pSender->pCurrentBlock != NULL && pSender->blockLen > 0) { + // build msg SyncSnapshotSend *pMsg = syncSnapshotSendBuild(pSender->blockLen, pSender->pSyncNode->vgId); pMsg->srcId = pSender->pSyncNode->myRaftId; pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; @@ -249,16 +272,20 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { pMsg->seq = pSender->seq; memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); + // send msg SRpcMsg rpcMsg; syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); - - char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender resend"); - syncNodeEventLog(pSender->pSyncNode, eventLog); - taosMemoryFree(eventLog); - syncSnapshotSendDestroy(pMsg); + + // event log + do { + char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender resend"); + syncNodeEventLog(pSender->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); } + return 0; } @@ -294,7 +321,6 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { snprintf(u64buf, sizeof(u64buf), "%lu", pSender->snapshot.lastApplyTerm); cJSON_AddStringToObject(pSnapshot, "lastApplyTerm", u64buf); cJSON_AddItemToObject(pRoot, "snapshot", pSnapshot); - snprintf(u64buf, sizeof(u64buf), "%lu", pSender->sendingMS); cJSON_AddStringToObject(pRoot, "sendingMS", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pSender->pSyncNode); @@ -324,7 +350,7 @@ char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event) { char *s = taosMemoryMalloc(len); SRaftId destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; - char host[128]; + char host[64]; uint16_t port; syncUtilU642Addr(destId.addr, host, sizeof(host), &port); @@ -355,12 +381,12 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId from pReceiver->term = pSyncNode->pRaftStore->currentTerm; pReceiver->privateTerm = 0; pReceiver->snapshot.data = NULL; - pReceiver->snapshot.lastApplyIndex = -1; + pReceiver->snapshot.lastApplyIndex = SYNC_INDEX_INVALID; pReceiver->snapshot.lastApplyTerm = 0; - pReceiver->snapshot.lastConfigIndex = -1; + pReceiver->snapshot.lastConfigIndex = SYNC_INDEX_INVALID; } else { - sInfo("snapshotReceiverCreate cannot create receiver"); + sError("vgId:%d cannot create snapshot receiver", pSyncNode->vgId); } return pReceiver; @@ -368,29 +394,55 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId from void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) { if (pReceiver != NULL) { + // close writer + if (pReceiver->pWriter != NULL) { + int32_t ret = + pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false); + ASSERT(ret == 0); + pReceiver->pWriter = NULL; + } + + // free receiver taosMemoryFree(pReceiver); } } bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { return pReceiver->start; } -// begin receive snapshot msg (current term, seq begin) +// static do start +// receive first snapshot data +// privateTerm, pBeginMsg static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg) { + // update state pReceiver->term = pReceiver->pSyncNode->pRaftStore->currentTerm; pReceiver->privateTerm = privateTerm; pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; pReceiver->fromId = pBeginMsg->srcId; + // update snapshot pReceiver->snapshot.lastApplyIndex = pBeginMsg->lastIndex; pReceiver->snapshot.lastApplyTerm = pBeginMsg->lastTerm; pReceiver->snapshot.lastConfigIndex = pBeginMsg->lastConfigIndex; + // write data ASSERT(pReceiver->pWriter == NULL); int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStartWrite(pReceiver->pSyncNode->pFsm, &(pReceiver->pWriter)); ASSERT(ret == 0); } +static void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) { + // force close, abandon incomplete data + if (pReceiver->pWriter != NULL) { + int32_t ret = + pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false); + ASSERT(ret == 0); + pReceiver->pWriter = NULL; + } + + pReceiver->start = false; +} + // if receiver receive msg from seq = SYNC_SNAPSHOT_SEQ_BEGIN, start receiver // if already start, force close, start again void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg) { @@ -413,12 +465,6 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTer snapshotReceiverDoStart(pReceiver, privateTerm, pBeginMsg); pReceiver->start = true; } - - if (gRaftDetailLog) { - char *s = snapshotReceiver2Str(pReceiver); - sInfo("snapshotReceiverStart %s", s); - taosMemoryFree(s); - } } void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply) { @@ -434,12 +480,6 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply) { if (apply) { // ++(pReceiver->privateTerm); } - - if (gRaftDetailLog) { - char *s = snapshotReceiver2Str(pReceiver); - sInfo("snapshotReceiverStop %s", s); - taosMemoryFree(s); - } } cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index d6c2cbd34e..2c43312064 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -127,7 +127,7 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { char *voteGranted2Str(SVotesGranted *pVotesGranted) { cJSON *pJson = voteGranted2Json(pVotesGranted); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -256,7 +256,7 @@ cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { char *votesRespond2Str(SVotesRespond *pVotesRespond) { cJSON *pJson = votesRespond2Json(pVotesRespond); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 27084286da..bc63462a9e 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -17,6 +17,7 @@ add_executable(syncVotesRespondTest "") add_executable(syncIndexMgrTest "") add_executable(syncLogStoreTest "") add_executable(syncEntryTest "") +add_executable(syncEntryCacheTest "") add_executable(syncRequestVoteTest "") add_executable(syncRequestVoteReplyTest "") add_executable(syncAppendEntriesTest "") @@ -129,6 +130,10 @@ target_sources(syncEntryTest PRIVATE "syncEntryTest.cpp" ) +target_sources(syncEntryCacheTest + PRIVATE + "syncEntryCacheTest.cpp" +) target_sources(syncRequestVoteTest PRIVATE "syncRequestVoteTest.cpp" @@ -362,6 +367,11 @@ target_include_directories(syncEntryTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncEntryCacheTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_include_directories(syncRequestVoteTest PUBLIC "${TD_SOURCE_DIR}/include/libs/sync" @@ -610,6 +620,10 @@ target_link_libraries(syncEntryTest sync gtest_main ) +target_link_libraries(syncEntryCacheTest + sync + gtest_main +) target_link_libraries(syncRequestVoteTest sync gtest_main diff --git a/source/libs/sync/test/syncEntryCacheTest.cpp b/source/libs/sync/test/syncEntryCacheTest.cpp new file mode 100644 index 0000000000..787c08e507 --- /dev/null +++ b/source/libs/sync/test/syncEntryCacheTest.cpp @@ -0,0 +1,162 @@ +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SSyncRaftEntry* createEntry(int i) { + int32_t dataLen = 20; + SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); + assert(pEntry != NULL); + pEntry->msgType = 88; + pEntry->originalRpcType = 99; + pEntry->seqNum = 3; + pEntry->isWeak = true; + pEntry->term = 100 + i; + pEntry->index = i; + snprintf(pEntry->data, dataLen, "value%d", i); + + return pEntry; +} + +SSyncNode* createFakeNode() { + SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); + ASSERT(pSyncNode != NULL); + memset(pSyncNode, 0, sizeof(SSyncNode)); + + return pSyncNode; +} + +SRaftEntryCache* createCache(int maxCount) { + SSyncNode* pSyncNode = createFakeNode(); + ASSERT(pSyncNode != NULL); + + SRaftEntryCache* pCache = raftCacheCreate(pSyncNode, maxCount); + ASSERT(pCache != NULL); + + return pCache; +} + +void test1() { + int32_t code = 0; + SRaftEntryCache* pCache = createCache(5); + for (int i = 0; i < 5; ++i) { + SSyncRaftEntry* pEntry = createEntry(i); + code = raftCachePutEntry(pCache, pEntry); + ASSERT(code == 1); + syncEntryDestory(pEntry); + } + raftCacheLog2((char*)"==test1 write 5 entries==", pCache); + + SyncIndex index; + index = 1; + code = raftCacheDelEntry(pCache, index); + ASSERT(code == 0); + index = 3; + code = raftCacheDelEntry(pCache, index); + ASSERT(code == 0); + raftCacheLog2((char*)"==test1 delete 1,3==", pCache); + + code = raftCacheClear(pCache); + ASSERT(code == 0); + raftCacheLog2((char*)"==clear all==", pCache); +} + +void test2() { + int32_t code = 0; + SRaftEntryCache* pCache = createCache(5); + for (int i = 0; i < 5; ++i) { + SSyncRaftEntry* pEntry = createEntry(i); + code = raftCachePutEntry(pCache, pEntry); + ASSERT(code == 1); + syncEntryDestory(pEntry); + } + raftCacheLog2((char*)"==test2 write 5 entries==", pCache); + + SyncIndex index; + index = 1; + SSyncRaftEntry* pEntry; + code = raftCacheGetEntry(pCache, index, &pEntry); + ASSERT(code == 0); + syncEntryDestory(pEntry); + syncEntryLog2((char*)"==test2 get entry 1==", pEntry); + + index = 2; + code = raftCacheGetEntryP(pCache, index, &pEntry); + ASSERT(code == 0); + syncEntryLog2((char*)"==test2 get entry pointer 2==", pEntry); + + // not found + index = 8; + code = raftCacheGetEntry(pCache, index, &pEntry); + ASSERT(code == -1 && terrno == TSDB_CODE_WAL_LOG_NOT_EXIST); + sTrace("==test2 get entry 8 not found=="); + + // not found + index = 9; + code = raftCacheGetEntryP(pCache, index, &pEntry); + ASSERT(code == -1 && terrno == TSDB_CODE_WAL_LOG_NOT_EXIST); + sTrace("==test2 get entry pointer 9 not found=="); +} + +void test3() { + int32_t code = 0; + SRaftEntryCache* pCache = createCache(5); + for (int i = 0; i < 5; ++i) { + SSyncRaftEntry* pEntry = createEntry(i); + code = raftCachePutEntry(pCache, pEntry); + ASSERT(code == 1); + syncEntryDestory(pEntry); + } + for (int i = 6; i < 10; ++i) { + SSyncRaftEntry* pEntry = createEntry(i); + code = raftCachePutEntry(pCache, pEntry); + ASSERT(code == 0); + syncEntryDestory(pEntry); + } + raftCacheLog2((char*)"==test3 write 10 entries, max count is 5==", pCache); +} + +void test4() { + int32_t code = 0; + SRaftEntryCache* pCache = createCache(5); + for (int i = 0; i < 5; ++i) { + SSyncRaftEntry* pEntry = createEntry(i); + code = raftCachePutEntry(pCache, pEntry); + ASSERT(code == 1); + syncEntryDestory(pEntry); + } + raftCacheLog2((char*)"==test4 write 5 entries==", pCache); + + SyncIndex index; + index = 3; + SSyncRaftEntry* pEntry; + code = raftCacheGetAndDel(pCache, index, &pEntry); + ASSERT(code == 0); + syncEntryLog2((char*)"==test4 get-and-del entry 3==", pEntry); + raftCacheLog2((char*)"==test4 after get-and-del entry 3==", pCache); +} + +int main(int argc, char** argv) { + gRaftDetailLog = true; + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE + DEBUG_DEBUG; + + test1(); + test2(); + test3(); + test4(); + + return 0; +} diff --git a/source/libs/sync/test/syncRaftLogTest2.cpp b/source/libs/sync/test/syncRaftLogTest2.cpp index e3ceae0615..9e0c2ecc29 100644 --- a/source/libs/sync/test/syncRaftLogTest2.cpp +++ b/source/libs/sync/test/syncRaftLogTest2.cpp @@ -113,7 +113,7 @@ void test2() { pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); pSyncNode->pLogStore = pLogStore; - //pLogStore->syncLogSetBeginIndex(pLogStore, 5); + // pLogStore->syncLogSetBeginIndex(pLogStore, 5); pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4); logStoreLog2((char*)"\n\n\ntest2 ----- ", pLogStore); @@ -229,7 +229,7 @@ void test4() { assert(pLogStore); pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest4 ----- ", pLogStore); - //pLogStore->syncLogSetBeginIndex(pLogStore, 5); + // pLogStore->syncLogSetBeginIndex(pLogStore, 5); pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4); for (int i = 5; i <= 9; ++i) { @@ -291,7 +291,7 @@ void test5() { assert(pLogStore); pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore); - //pLogStore->syncLogSetBeginIndex(pLogStore, 5); + // pLogStore->syncLogSetBeginIndex(pLogStore, 5); pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4); for (int i = 5; i <= 9; ++i) { @@ -412,26 +412,23 @@ void test6() { do { SyncIndex firstVer = walGetFirstVer(pWal); SyncIndex lastVer = walGetLastVer(pWal); - bool isEmpty = walIsEmpty(pWal); + bool isEmpty = walIsEmpty(pWal); printf("before -------- firstVer:%ld lastVer:%ld isEmpty:%d \n", firstVer, lastVer, isEmpty); } while (0); logStoreDestory(pLogStore); cleanup(); - - // restart init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); pSyncNode->pLogStore = pLogStore; - do { SyncIndex firstVer = walGetFirstVer(pWal); SyncIndex lastVer = walGetLastVer(pWal); - bool isEmpty = walIsEmpty(pWal); + bool isEmpty = walIsEmpty(pWal); printf("after -------- firstVer:%ld lastVer:%ld isEmpty:%d \n", firstVer, lastVer, isEmpty); } while (0); @@ -461,13 +458,13 @@ int main(int argc, char** argv) { } sTrace("gAssert : %d", gAssert); -/* - test1(); - test2(); - test3(); - test4(); - test5(); -*/ + /* + test1(); + test2(); + test3(); + test4(); + test5(); + */ test6(); return 0; diff --git a/source/libs/sync/test/syncRaftLogTest3.cpp b/source/libs/sync/test/syncRaftLogTest3.cpp index 302e29a091..ea1788c545 100644 --- a/source/libs/sync/test/syncRaftLogTest3.cpp +++ b/source/libs/sync/test/syncRaftLogTest3.cpp @@ -312,7 +312,7 @@ void test5() { pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore); - //pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, 6); + // pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, 6); pLogStore->syncLogRestoreFromSnapshot(pSyncNode->pLogStore, 5); for (int i = 6; i <= 10; ++i) { int32_t dataLen = 10; diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index c328629c4b..462debb247 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -52,7 +52,7 @@ typedef struct { char user[TSDB_UNI_LEN]; // meter ID void (*cfp)(void* parent, SRpcMsg*, SEpSet*); - bool (*retry)(int32_t code); + bool (*retry)(int32_t code, tmsg_t msgType); int index; void* parent; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index c960cb4c63..46b6be3986 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1027,7 +1027,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { */ STransConnCtx* pCtx = pMsg->ctx; int32_t code = pResp->code; - if (pTransInst->retry != NULL && pTransInst->retry(code)) { + if (pTransInst->retry != NULL && pTransInst->retry(code, pResp->msgType - 1)) { pMsg->sent = 0; pCtx->retryCnt += 1; if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 0c6cd80f44..291d38bdca 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -157,14 +157,14 @@ int32_t taosRenameFile(const char *oldName, const char *newName) { #ifdef WINDOWS bool code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); if (!code) { - printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); + printf("failed to rename file %s to %s, reason:%s\n", oldName, newName, strerror(errno)); } return !code; #else int32_t code = rename(oldName, newName); if (code < 0) { - printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); + printf("failed to rename file %s to %s, reason:%s\n", oldName, newName, strerror(errno)); } return code; diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index ba07b6c3dd..9c66ac1df8 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -18,6 +18,10 @@ #include "os.h" #if defined(WINDOWS) +BOOL WINAPI CtrlHandler(DWORD fdwCtrlType) { + printf("\n" TAOS_CONSOLE_PROMPT_HEADER); + return TRUE; +} #elif defined(_TD_DARWIN_64) #else #include @@ -124,7 +128,7 @@ int taosSetConsoleEcho(bool on) { void taosSetTerminalMode() { #if defined(WINDOWS) - // assert(0); + SetConsoleCtrlHandler(CtrlHandler, TRUE); #else struct termios newtio; @@ -158,7 +162,6 @@ void taosSetTerminalMode() { int32_t taosGetOldTerminalMode() { #if defined(WINDOWS) - // assert(0); #else /* Make sure stdin is a terminal. */ if (!isatty(STDIN_FILENO)) { @@ -176,7 +179,7 @@ int32_t taosGetOldTerminalMode() { void taosResetTerminalMode() { #if defined(WINDOWS) - // assert(0); + SetConsoleCtrlHandler(CtrlHandler, FALSE); #else if (tcsetattr(0, TCSANOW, &oldtio) != 0) { fprintf(stderr, "Fail to reset the terminal properties!\n"); diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 3f23cd8b5f..f80c2b217f 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -52,7 +52,7 @@ fi docker run \ -v $REP_MOUNT_PARAM \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_TOOLS=true;make -j $THREAD_COUNT" + --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true;make -j $THREAD_COUNT" ret=$? exit $ret diff --git a/tests/pytest/util/constant.py b/tests/pytest/util/constant.py index e5095c74dc..807398f70f 100644 --- a/tests/pytest/util/constant.py +++ b/tests/pytest/util/constant.py @@ -1,5 +1,76 @@ # -*- coding: utf-8 -*- +# basic type +TAOS_DATA_TYPE = [ + "INT", "BIGINT", "SMALLINT", "TINYINT", "INT UNSIGNED", "BIGINT UNSIGNED", "SMALLINT UNSIGNED", "TINYINT UNSIGNED", + "FLOAT", "DOUBLE", + "BOOL", + "BINARY", "NCHAR", "VARCHAR", + "TIMESTAMP", + # "MEDIUMBLOB", "BLOB", # add in 3.x + # "DECIMAL", "NUMERIC", # add in 3.x + "JSON", # only for tag +] + +TAOS_NUM_TYPE = [ + "INT", "BIGINT", "SMALLINT", "TINYINT", "INT UNSIGNED", "BIGINT UNSIGNED", "SMALLINT UNSIGNED", "TINYINT UNSIGNED", "FLOAT", "DOUBLE", + # "DECIMAL", "NUMERIC", # add in 3.x +] +TAOS_CHAR_TYPE = [ + "BINARY", "NCHAR", "VARCHAR", +] +TAOS_BOOL_TYPE = ["BOOL",] +TAOS_TS_TYPE = ["TIMESTAMP",] +TAOS_BIN_TYPE = [ + "MEDIUMBLOB", "BLOB", # add in 3.x +] + +TAOS_TIME_INIT = ["b", "u", "a", "s", "m", "h", "d", "w", "n", "y"] +TAOS_PRECISION = ["ms", "us", "ns"] +PRECISION_DEFAULT = "ms" +PRECISION = PRECISION_DEFAULT + +TAOS_KEYWORDS = [ + "ABORT", "CREATE", "IGNORE", "NULL", "STAR", + "ACCOUNT", "CTIME", "IMMEDIATE", "OF", "STATE", + "ACCOUNTS", "DATABASE", "IMPORT", "OFFSET", "STATEMENT", + "ADD", "DATABASES", "IN", "OR", "STATE_WINDOW", + "AFTER", "DAYS", "INITIALLY", "ORDER", "STORAGE", + "ALL", "DBS", "INSERT", "PARTITIONS", "STREAM", + "ALTER", "DEFERRED", "INSTEAD", "PASS", "STREAMS", + "AND", "DELIMITERS", "INT", "PLUS", "STRING", + "AS", "DESC", "INTEGER", "PPS", "SYNCDB", + "ASC", "DESCRIBE", "INTERVAL", "PRECISION", "TABLE", + "ATTACH", "DETACH", "INTO", "PREV", "TABLES", + "BEFORE", "DISTINCT", "IS", "PRIVILEGE", "TAG", + "BEGIN", "DIVIDE", "ISNULL", "QTIME", "TAGS", + "BETWEEN", "DNODE", "JOIN", "QUERIES", "TBNAME", + "BIGINT", "DNODES", "KEEP", "QUERY", "TIMES", + "BINARY", "DOT", "KEY", "QUORUM", "TIMESTAMP", + "BITAND", "DOUBLE", "KILL", "RAISE", "TINYINT", + "BITNOT", "DROP", "LE", "REM", "TOPIC", + "BITOR", "EACH", "LIKE", "REPLACE", "TOPICS", + "BLOCKS", "END", "LIMIT", "REPLICA", "TRIGGER", + "BOOL", "EQ", "LINEAR", "RESET", "TSERIES", + "BY", "EXISTS", "LOCAL", "RESTRICT", "UMINUS", + "CACHE", "EXPLAIN", "LP", "ROW", "UNION", + "CACHELAST", "FAIL", "LSHIFT", "RP", "UNSIGNED", + "CASCADE", "FILE", "LT", "RSHIFT", "UPDATE", + "CHANGE", "FILL", "MATCH", "SCORES", "UPLUS", + "CLUSTER", "FLOAT", "MAXROWS", "SELECT", "USE", + "COLON", "FOR", "MINROWS", "SEMI", "USER", + "COLUMN", "FROM", "MINUS", "SESSION", "USERS", + "COMMA", "FSYNC", "MNODES", "SET", "USING", + "COMP", "GE", "MODIFY", "SHOW", "VALUES", + "COMPACT", "GLOB", "MODULES", "SLASH", "VARIABLE", + "CONCAT", "GRANTS", "NCHAR", "SLIDING", "VARIABLES", + "CONFLICT", "GROUP", "NE", "SLIMIT", "VGROUPS", + "CONNECTION", "GT", "NONE", "SMALLINT", "VIEW", + "CONNECTIONS", "HAVING", "NOT", "SOFFSET", "VNODES", + "CONNS", "ID", "NOTNULL", "STABLE", "WAL", + "COPY", "IF", "NOW", "STABLES", "WHERE", +] + # basic data type boundary TINYINT_MAX = 127 TINYINT_MIN = -128 @@ -11,7 +82,7 @@ SMALLINT_MAX = 32767 SMALLINT_MIN = -32768 SMALLINT_UN_MAX = 65535 -MALLINT_UN_MIN = 0 +SMALLINT_UN_MIN = 0 INT_MAX = 2147483647 INT_MIN = -2147483648 @@ -33,8 +104,8 @@ DOUBLE_MIN = -1.7E+308 # schema boundary BINARY_LENGTH_MAX = 16374 -NCAHR_LENGTH_MAX_ = 4093 -DBNAME_LENGTH_MAX_ = 64 +NCAHR_LENGTH_MAX = 4093 +DBNAME_LENGTH_MAX = 64 STBNAME_LENGTH_MAX = 192 STBNAME_LENGTH_MIN = 1 @@ -66,4 +137,32 @@ MNODE_SHM_SIZE_DEFAULT = 6292480 VNODE_SHM_SIZE_MAX = 2147483647 VNODE_SHM_SIZE_MIN = 6292480 -VNODE_SHM_SIZE_DEFAULT = 31458304 \ No newline at end of file +VNODE_SHM_SIZE_DEFAULT = 31458304 + +# time_init +TIME_MS = 1 +TIME_US = TIME_MS/1000 +TIME_NS = TIME_US/1000 + +TIME_S = 1000 * TIME_MS +TIME_M = 60 * TIME_S +TIME_H = 60 * TIME_M +TIME_D = 24 * TIME_H +TIME_W = 7 * TIME_D +TIME_N = 30 * TIME_D +TIME_Y = 365 * TIME_D + + +# session parameters +INTERVAL_MIN = 1 * TIME_MS if PRECISION == PRECISION_DEFAULT else 1 * TIME_US + + +# streams and related agg-function +SMA_INDEX_FUNCTIONS = ["MIN", "MAX"] +ROLLUP_FUNCTIONS = ["AVG", "SUM", "MIN", "MAX", "LAST", "FIRST"] +SMA_WATMARK_MAXDELAY_INIT = ['a', "s", "m"] +WATERMARK_MAX = 900000 +WATERMARK_MIN = 0 + +MAX_DELAY_MAX = 900000 +MAX_DELAY_MIN = 1 \ No newline at end of file diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 580fc8ee47..fe802dd9a3 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -21,6 +21,7 @@ import psutil import shutil import pandas as pd from util.log import * +from util.constant import * def _parse_datetime(timestr): try: @@ -117,8 +118,7 @@ class TDSql: col_name_list = [] col_type_list = [] self.cursor.execute(sql) - self.queryCols = self.cursor.description - for query_col in self.queryCols: + for query_col in self.cursor.description: col_name_list.append(query_col[0]) col_type_list.append(query_col[1]) except Exception as e: @@ -301,6 +301,41 @@ class TDSql: args = (caller.filename, caller.lineno, self.sql, elm, expect_elm) tdLog.exit("%s(%d) failed: sql:%s, elm:%s == expect_elm:%s" % args) + def get_times(self, time_str, precision="ms"): + caller = inspect.getframeinfo(inspect.stack()[1][0]) + if time_str[-1] not in TAOS_TIME_INIT: + tdLog.exit(f"{caller.filename}({caller.lineno}) failed: {time_str} not a standard taos time init") + if precision not in TAOS_PRECISION: + tdLog.exit(f"{caller.filename}({caller.lineno}) failed: {precision} not a standard taos time precision") + + if time_str[-1] == TAOS_TIME_INIT[0]: + times = int(time_str[:-1]) * TIME_NS + if time_str[-1] == TAOS_TIME_INIT[1]: + times = int(time_str[:-1]) * TIME_US + if time_str[-1] == TAOS_TIME_INIT[2]: + times = int(time_str[:-1]) * TIME_MS + if time_str[-1] == TAOS_TIME_INIT[3]: + times = int(time_str[:-1]) * TIME_S + if time_str[-1] == TAOS_TIME_INIT[4]: + times = int(time_str[:-1]) * TIME_M + if time_str[-1] == TAOS_TIME_INIT[5]: + times = int(time_str[:-1]) * TIME_H + if time_str[-1] == TAOS_TIME_INIT[6]: + times = int(time_str[:-1]) * TIME_D + if time_str[-1] == TAOS_TIME_INIT[7]: + times = int(time_str[:-1]) * TIME_W + if time_str[-1] == TAOS_TIME_INIT[8]: + times = int(time_str[:-1]) * TIME_N + if time_str[-1] == TAOS_TIME_INIT[9]: + times = int(time_str[:-1]) * TIME_Y + + if precision == "ms": + return int(times) + elif precision == "us": + return int(times*1000) + elif precision == "ns": + return int(times*1000*1000) + def taosdStatus(self, state): tdLog.sleep(5) pstate = 0 diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 1b61bd1890..d3a6a0d102 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -22,15 +22,19 @@ # ---- dnode ./test.sh -f tsim/dnode/balance_replica1.sim +#./test.sh -f tsim/dnode/balance_replica3.sim ./test.sh -f tsim/dnode/create_dnode.sim ./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim ./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim ./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim -#./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim -#./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim -#./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim -#./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim +./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim +./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim +./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim ./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim # ---- insert ./test.sh -f tsim/insert/basic0.sim diff --git a/tests/script/test-all.bat b/tests/script/test-all.bat index 7a1a4bc7fa..81c1f21c04 100644 --- a/tests/script/test-all.bat +++ b/tests/script/test-all.bat @@ -11,14 +11,17 @@ if not "%2" == "" ( ) for /F "usebackq tokens=*" %%i in (!caseFile!) do ( set line=%%i - if "!line:~,9!" == "./test.sh" ( - set /a a+=1 - echo !a! Processing %%i - call :GetTimeSeconds !time! - set time1=!_timeTemp! - echo Start at !time! - call !line:./test.sh=wtest.bat! > result_!a!.txt 2>error_!a!.txt - if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. ) + call :CheckSkipCase %%i + if !skipCase! == false ( + if "!line:~,9!" == "./test.sh" ( + set /a a+=1 + echo !a! Processing %%i + call :GetTimeSeconds !time! + set time1=!_timeTemp! + echo Start at !time! + call !line:./test.sh=wtest.bat! > result_!a!.txt 2>error_!a!.txt + if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. ) + ) ) ) exit !exitNum! @@ -56,3 +59,10 @@ for %%a in (%tt%) do ( ) set /a _timeTemp=(%hh%*60+%mm%)*60+%ss% goto :eof + +:CheckSkipCase +set skipCase=false +if "%*" == "./test.sh -f tsim/query/scalarFunction.sim" ( set skipCase=true ) +if "%*" == "./test.sh -f tsim/stream/distributeInterval0.sim" ( set skipCase=true ) +if "%*" == "./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim" ( set skipCase=true ) +:goto eof \ No newline at end of file diff --git a/tests/script/tsim/dnode/balance_replica1.sim b/tests/script/tsim/dnode/balance_replica1.sim index 14f3f130fb..998d0654ab 100644 --- a/tests/script/tsim/dnode/balance_replica1.sim +++ b/tests/script/tsim/dnode/balance_replica1.sim @@ -117,7 +117,6 @@ if $rows != 6 then return -1 endi -return system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/balance_replica3.sim b/tests/script/tsim/dnode/balance_replica3.sim new file mode 100644 index 0000000000..b8996e259d --- /dev/null +++ b/tests/script/tsim/dnode/balance_replica3.sim @@ -0,0 +1,362 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 +system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode5 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +sql connect + +print =============== step1 create dnode2 +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 +sql create dnode $hostname port 7500 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 +if $rows != 5 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 +endi + +print =============== step2: create db +sql create database d1 vgroups 4 replica 3 + +print =============== step32 wait vgroup2 +$x = 0 +step32: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 +endi +if $data(2)[6] == leader then + $leaderExist = 1 +endi +if $data(2)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step32 +endi + +print =============== step33 wait vgroup3 +$x = 0 +step33: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(3)[4] == leader then + $leaderExist = 1 +endi +if $data(3)[6] == leader then + $leaderExist = 1 +endi +if $data(3)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step33 +endi + +print =============== step34 wait vgroup4 +$x = 0 +step34: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step34 +endi + +print =============== step35 wait vgroup5 +$x = 0 +step35: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step35 +endi + +print =============== step36: create table +sql use d1 +sql create table d1.st (ts timestamp, i int) tags (j int) +sql create table d1.c1 using st tags(1) +sql create table d1.c2 using st tags(1) +sql create table d1.c3 using st tags(1) +sql create table d1.c4 using st tags(1) +sql create table d1.c5 using st tags(1) +sql create table d1.c6 using st tags(1) +sql show d1.tables +if $rows != 6 then + return -1 +endi + +print =============== step4: start dnode5 +system sh/exec.sh -n dnode5 -s start +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 +if $rows != 5 then + return -1 +endi +if $data(1)[4] != ready then + goto step4 +endi +if $data(2)[4] != ready then + goto step4 +endi +if $data(3)[4] != ready then + goto step4 +endi +if $data(4)[4] != ready then + goto step4 +endi +if $data(5)[4] != ready then + goto step4 +endi + +print =============== step5: balance +sql balance vgroup + +print =============== step62 wait vgroup2 +$x = 0 +step62: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 +endi +if $data(2)[6] == leader then + $leaderExist = 1 +endi +if $data(2)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step62 +endi + +print =============== step63 wait vgroup3 +$x = 0 +step63: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(3)[4] == leader then + $leaderExist = 1 +endi +if $data(3)[6] == leader then + $leaderExist = 1 +endi +if $data(3)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step63 +endi + +print =============== step64 wait vgroup4 +$x = 0 +step64: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step64 +endi + +print =============== step65 wait vgroup5 +$x = 0 +step65: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step65 +endi + + + +print =============== step7: select data + +sql show d1.tables +print rows $rows +if $rows != 6 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim b/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim index 35a2466297..e1b7c4a3a4 100644 --- a/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim +++ b/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim @@ -28,7 +28,7 @@ step1: sql show dnodes print ===> $data00 $data01 $data02 $data03 $data04 $data05 print ===> $data10 $data11 $data12 $data13 $data14 $data15 -if $rows != 3 then +if $rows != 5 then return -1 endi if $data(1)[4] != ready then @@ -49,6 +49,128 @@ endi print =============== step3 create database sql create database d1 vgroups 4 replica 3 + +print =============== step32 wait vgroup2 +$x = 0 +step32: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 +endi +if $data(2)[6] == leader then + $leaderExist = 1 +endi +if $data(2)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step32 +endi + +print =============== step33 wait vgroup3 +$x = 0 +step33: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(3)[4] == leader then + $leaderExist = 1 +endi +if $data(3)[6] == leader then + $leaderExist = 1 +endi +if $data(3)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step33 +endi + +print =============== step34 wait vgroup4 +$x = 0 +step34: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step34 +endi + +print =============== step35 wait vgroup5 +$x = 0 +step35: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step35 +endi + +print =============== step36: create table sql use d1 sql create table d1.st (ts timestamp, i int) tags (j int) sql create table d1.c1 using st tags(1) @@ -57,21 +179,6 @@ if $rows != 1 then return -1 endi -sql show d1.vgroups -print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] -if $rows != 1 then - return -1 -endi -if $data(2)[3] != 2 then - return -1 -endi -if $data(2)[5] != 3 then - return -1 -endi -if $data(2)[7] != 4 then - return -1 -endi - print =============== step4: drop dnode 2 system sh/exec.sh -n dnode5 -s start $x = 0 @@ -85,7 +192,7 @@ step4: sql show dnodes print ===> $data00 $data01 $data02 $data03 $data04 $data05 print ===> $data10 $data11 $data12 $data13 $data14 $data15 -if $rows != 3 then +if $rows != 5 then return -1 endi if $data(1)[4] != ready then @@ -115,21 +222,133 @@ if $rows != 4 then return -1 endi -print show d1.vgroups +print =============== step62 wait vgroup2 +$x = 0 +step62: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi sql show d1.vgroups -print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] -if $rows != 1 then +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then return -1 endi +if $data(2)[4] == leader then + $leaderExist = 1 +endi +if $data(2)[6] == leader then + $leaderExist = 1 +endi +if $data(2)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step62 +endi +print =============== step63 wait vgroup3 +$x = 0 +step63: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(3)[4] == leader then + $leaderExist = 1 +endi +if $data(3)[6] == leader then + $leaderExist = 1 +endi +if $data(3)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step63 +endi -print =============== step6: select data +print =============== step64 wait vgroup4 +$x = 0 +step64: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step64 +endi + +print =============== step35 wait vgroup5 +$x = 0 +step65: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +if $rows != 4 then + return -1 +endi +if $data(4)[4] == leader then + $leaderExist = 1 +endi +if $data(4)[6] == leader then + $leaderExist = 1 +endi +if $data(4)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step65 +endi + +print =============== step7: select data +print ===> $data00 $data01 sql show d1.tables if $rows != 1 then return -1 endi -return system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim b/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim index 3ea351f7a7..4855f4dccf 100644 --- a/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim +++ b/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim @@ -49,6 +49,50 @@ endi print =============== step3 create database sql create database d1 vgroups 1 replica 3 +$leaderExist = 0 +$leaderVnode = 0 +$follower1 = 0 +$follower2 = 0 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $rows != 1 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 + $leaderVnode = 2 + $follower1 = 3 + $follower2 = 4 +endi +if $data(2)[6] == leader then + $leaderExist = 1 + $leaderVnode = 3 + $follower1 = 2 + $follower2 = 4 +endi +if $data(2)[8] == leader then + $leaderExist = 1 + $leaderVnode = 4 + $follower1 = 2 + $follower2 = 3 +endi +if $leaderExist != 1 then + goto step3 +endi + +print leader $leaderVnode +print follower1 $follower1 +print follower2 $follower2 + sql use d1 sql create table d1.st (ts timestamp, i int) tags (j int) sql create table d1.c1 using st tags(1) @@ -72,13 +116,6 @@ if $data(2)[7] != 4 then return -1 endi -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT - - -return - - print =============== step4: drop dnode 2 system sh/exec.sh -n dnode5 -s start $x = 0 @@ -92,7 +129,7 @@ step4: sql show dnodes print ===> $data00 $data01 $data02 $data03 $data04 $data05 print ===> $data10 $data11 $data12 $data13 $data14 $data15 -if $rows != 3 then +if $rows != 5 then return -1 endi if $data(1)[4] != ready then @@ -116,8 +153,11 @@ sql drop dnode 2 print show dnodes; sql show dnodes; -print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] -print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 if $rows != 4 then return -1 endi @@ -128,9 +168,9 @@ print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] if $rows != 1 then return -1 endi -#if $data(2)[3] != 3 then -# return -1 -#endi +if $data(2)[3] != 5 then + return -1 +endi print =============== step6: select data sql show d1.tables @@ -138,7 +178,37 @@ if $rows != 1 then return -1 endi -return +$leaderExist = 0 +$leaderVnode = 0 +$follower1 = 0 +$follower2 = 0 + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $rows != 1 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 +endi +if $data(2)[6] == leader then + $leaderExist = 1 +endi +if $data(2)[8] == leader then + $leaderExist = 1 +endi +if $leaderExist != 1 then + goto step6 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica1.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica1.sim new file mode 100644 index 0000000000..d3b5a02a23 --- /dev/null +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica1.sim @@ -0,0 +1,178 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +sql connect +sql create user u1 pass 'taosdata' + +print =============== step1 create dnode2 +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi + +print =============== step2: create db +sql create database d1 vgroups 1 replica 1 + +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 +endi +if $data(1)[4] != ready then + goto step2 +endi +if $data(2)[4] != ready then + goto step2 +endi +if $data(3)[4] != ready then + goto step2 +endi +if $data(4)[4] != ready then + goto step2 +endi + +print =============== step3: create database +sql use d1 +sql create table d1.st (ts timestamp, i int) tags (j int) +sql create table d1.c1 using st tags(1) +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step40: +print redistribute vgroup 2 dnode 3 +sql redistribute vgroup 2 dnode 3 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step41: +print redistribute vgroup 2 dnode 4 +sql redistribute vgroup 2 dnode 4 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step42: +print redistribute vgroup 2 dnode 2 +sql redistribute vgroup 2 dnode 2 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step43: +print redistribute vgroup 2 dnode 3 +sql redistribute vgroup 2 dnode 3 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step44: +print redistribute vgroup 2 dnode 4 +sql redistribute vgroup 2 dnode 4 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step45: +print redistribute vgroup 2 dnode 2 +sql redistribute vgroup 2 dnode 2 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step46: +print redistribute vgroup 2 dnode 3 +sql redistribute vgroup 2 dnode 3 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step47: +print redistribute vgroup 2 dnode 2 +sql redistribute vgroup 2 dnode 2 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim index 9b11e16616..00c6e8a98d 100644 --- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim @@ -65,7 +65,7 @@ sql_error redistribute vgroup 3 dnode 6 dnode 3 dnode 4 # vgroup not exist sql_error redistribute vgroup 3 dnode 5 dnode 3 dnode 4 # un changed -sql_error redistribute vgroup 2 dnode 2 dnode 3 dnode 4 +# sql_error redistribute vgroup 2 dnode 2 dnode 3 dnode 4 # no enought vnodes sql_error redistribute vgroup 2 dnode 1 dnode 3 dnode 4 # offline vnodes @@ -176,8 +176,6 @@ if $rows != 1 then return -1 endi -return - print =============== step32: print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim new file mode 100644 index 0000000000..6614009497 --- /dev/null +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim @@ -0,0 +1,245 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 +system sh/deploy.sh -n dnode6 -i 6 +system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode5 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode6 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +sql connect +sql create user u1 pass 'taosdata' + +print =============== step1 create dnode2 +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 +sql create dnode $hostname port 7500 +sql create dnode $hostname port 7600 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 +if $rows != 6 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 +endi + +print =============== step2: create db +sql create database d1 vgroups 1 replica 3 + +system sh/exec.sh -n dnode5 -s start +system sh/exec.sh -n dnode6 -s start +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 +if $rows != 6 then + return -1 +endi +if $data(1)[4] != ready then + goto step2 +endi +if $data(2)[4] != ready then + goto step2 +endi +if $data(3)[4] != ready then + goto step2 +endi +if $data(4)[4] != ready then + goto step2 +endi +if $data(5)[4] != ready then + goto step2 +endi +if $data(6)[4] != ready then + goto step2 +endi + +print =============== step3: create database +$leaderExist = 0 +$leaderVnode = 0 +$follower1 = 0 +$follower2 = 0 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $rows != 1 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 + $leaderVnode = 2 + $follower1 = 3 + $follower2 = 4 +endi +if $data(2)[6] == leader then + $leaderExist = 1 + $leaderVnode = 3 + $follower1 = 2 + $follower2 = 4 +endi +if $data(2)[8] == leader then + $leaderExist = 1 + $leaderVnode = 4 + $follower1 = 2 + $follower2 = 3 +endi +if $leaderExist != 1 then + goto step3 +endi + +print leader $leaderVnode +print follower1 $follower1 +print follower2 $follower2 + +sql use d1 +sql create table d1.st (ts timestamp, i int) tags (j int) +sql create table d1.c1 using st tags(1) +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step40: +print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode 6 +sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode 6 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step41: +print redistribute vgroup 2 dnode 5 dnode $follower2 dnode $follower1 +sql redistribute vgroup 2 dnode 5 dnode $follower2 dnode $follower1 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step42: +print redistribute vgroup 2 dnode $follower2 dnode 6 dnode $leaderVnode +sql redistribute vgroup 2 dnode $follower2 dnode 6 dnode $leaderVnode +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step43: +print redistribute vgroup 2 dnode $follower1 dnode 5 dnode 6 +sql redistribute vgroup 2 dnode $follower1 dnode 5 dnode 6 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step44: +print redistribute vgroup 2 dnode $follower2 dnode 5 dnode $leaderVnode +sql redistribute vgroup 2 dnode $follower2 dnode 5 dnode $leaderVnode +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step45: +print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 6 +sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 6 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step46: +print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 +sql redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step47: +print redistribute vgroup 2 dnode $follower1 dnode 6 dnode $leaderVnode +sql redistribute vgroup 2 dnode $follower1 dnode 6 dnode $leaderVnode +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim new file mode 100644 index 0000000000..c78d90e352 --- /dev/null +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim @@ -0,0 +1,263 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 +system sh/deploy.sh -n dnode6 -i 6 +system sh/deploy.sh -n dnode7 -i 7 +system sh/deploy.sh -n dnode8 -i 8 +system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode5 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode6 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode7 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode8 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +sql connect +sql create user u1 pass 'taosdata' + +print =============== step1 create dnode2 +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 +sql create dnode $hostname port 7500 +sql create dnode $hostname port 7600 +sql create dnode $hostname port 7700 +sql create dnode $hostname port 7800 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 +if $rows != 8 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 +endi + +print =============== step2: create db +sql create database d1 vgroups 1 replica 3 + +system sh/exec.sh -n dnode5 -s start +system sh/exec.sh -n dnode6 -s start +system sh/exec.sh -n dnode7 -s start +system sh/exec.sh -n dnode8 -s start +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 +if $rows != 8 then + return -1 +endi +if $data(1)[4] != ready then + goto step2 +endi +if $data(2)[4] != ready then + goto step2 +endi +if $data(3)[4] != ready then + goto step2 +endi +if $data(4)[4] != ready then + goto step2 +endi +if $data(5)[4] != ready then + goto step2 +endi +if $data(6)[4] != ready then + goto step2 +endi +if $data(7)[4] != ready then + goto step2 +endi +if $data(8)[4] != ready then + goto step2 +endi + +print =============== step3: create database +$leaderExist = 0 +$leaderVnode = 0 +$follower1 = 0 +$follower2 = 0 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $rows != 1 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 + $leaderVnode = 2 + $follower1 = 3 + $follower2 = 4 +endi +if $data(2)[6] == leader then + $leaderExist = 1 + $leaderVnode = 3 + $follower1 = 2 + $follower2 = 4 +endi +if $data(2)[8] == leader then + $leaderExist = 1 + $leaderVnode = 4 + $follower1 = 2 + $follower2 = 3 +endi +if $leaderExist != 1 then + goto step3 +endi + +print leader $leaderVnode +print follower1 $follower1 +print follower2 $follower2 + +sql use d1 +sql create table d1.st (ts timestamp, i int) tags (j int) +sql create table d1.c1 using st tags(1) +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step40: +print redistribute vgroup 2 dnode 7 dnode 5 dnode 6 +sql redistribute vgroup 2 dnode 7 dnode 5 dnode 6 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step41: +print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode $follower1 +sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode $follower1 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step42: +print redistribute vgroup 2 dnode 7 dnode 6 dnode 8 +sql redistribute vgroup 2 dnode 7 dnode 6 dnode 8 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step43: +print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 +sql redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + + +return + + +print =============== step44: +print redistribute vgroup 2 dnode 6 dnode 7 dnode $leaderVnode +sql redistribute vgroup 2 dnode 6 dnode 7 dnode $leaderVnode +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step45: +print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 8 +sql redistribute vgroup 2dnode $follower1 dnode $follower2 dnode 8 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step46: +print redistribute vgroup 2 dnode $leaderVnode dnode 6 dnode 5 +sql redistribute vgroup 2 dnode $leaderVnode dnode 6 dnode 5 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step47: +print redistribute vgroup 2 dnode $follower1 dnode 7 dnode 8 +sql redistribute vgroup 2 dnode $follower1 dnode 7 dnode 8 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim index 8936655974..a59e989d80 100644 --- a/tests/script/tsim/stream/distributeInterval0.sim +++ b/tests/script/tsim/stream/distributeInterval0.sim @@ -41,7 +41,7 @@ sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); -sql create stream stream_t1 trigger at_once into streamtST1 as select _wstartts, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); +sql create stream stream_t1 trigger at_once watermark 1d into streamtST1 as select _wstartts, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); sleep 1000 diff --git a/tests/script/tsim/stream/distributeIntervalRetrive0.sim b/tests/script/tsim/stream/distributeIntervalRetrive0.sim new file mode 100644 index 0000000000..cde5c7058f --- /dev/null +++ b/tests/script/tsim/stream/distributeIntervalRetrive0.sim @@ -0,0 +1,291 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 + +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +sql create dnode $hostname2 port 7200 + +system sh/exec.sh -n dnode2 -s start + +print ===== step1 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 2 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi + +print ===== step2 + +sql create database test vgroups 4; +sql use test; +sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); +sql create table ts3 using st tags(3,2,2); +sql create table ts4 using st tags(4,2,2); +sql create stream stream_t1 trigger at_once into streamtST1 as select _wstartts, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); + +sleep 1000 + +sql insert into ts1 values(1648791213001,1,12,3,1.0); +sql insert into ts2 values(1648791213001,1,12,3,1.0); +sql insert into ts1 values(1648791213002,NULL,NULL,NULL,NULL); +sql insert into ts2 values(1648791213002,NULL,NULL,NULL,NULL); + +sql insert into ts1 values(1648791223002,2,2,3,1.1); +sql insert into ts1 values(1648791233003,3,2,3,2.1); +sql insert into ts2 values(1648791243004,4,2,43,73.1); + +sql insert into ts1 values(1648791213002,24,22,23,4.1) (1648791243005,4,20,3,3.1); + +sleep 1000 + +sql insert into ts3 values(1648791213001,12,12,13,14.1) (1648791243005,14,30,30,30.1); + +$loop_count = 0 +loop1: +sleep 1000 +sql select * from streamtST1; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data01 != 5 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != 14 then + print =====data02=$data02 + goto loop1 +endi + +# row 1 +if $data11 != 1 then + print =====data11=$data11 + goto loop1 +endi + +if $data12 != 2 then + print =====data12=$data12 + goto loop1 +endi + +#row2 +if $data21 != 1 then + print =====data21=$data21 + goto loop1 +endi + +if $data22 != 3 then + print =====data22=$data22 + goto loop1 +endi + +#row 3 +if $data31 != 3 then + print =====data31=$data31 + goto loop1 +endi + +if $data32 != 22 then + print =====data32=$data32 + goto loop1 +endi + +print loop1 over + +sql insert into ts1 values(1648791223008,4,2,30,3.1) (1648791213009,4,2,3,3.1) (1648791233010,4,2,3,3.1) (1648791243011,4,2,3,3.1)(1648791243012,34,32,33,3.1); + +$loop_count = 0 +loop2: +sleep 1000 +sql select * from streamtST1; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data01 != 6 then + print =====data01=$data01 + goto loop2 +endi + +if $data02 != 18 then + print =====data02=$data02 + goto loop2 +endi + +# row 1 +if $data11 != 2 then + print =====data11=$data11 + goto loop2 +endi + +if $data12 != 6 then + print =====data12=$data12 + goto loop2 +endi + +#row2 +if $data21 != 2 then + print =====data21=$data21 + goto loop2 +endi + +if $data22 != 7 then + print =====data22=$data22 + goto loop2 +endi + +#row 3 +if $data31 != 5 then + print =====data31=$data31 + goto loop2 +endi + +if $data32 != 60 then + print =====data32=$data32 + goto loop2 +endi + +print loop2 over + +sql insert into ts4 values(1648791223008,4,2,30,3.1) (1648791213009,4,2,3,3.1) (1648791233010,4,2,3,3.1); + +$loop_count = 0 +loop3: +sleep 1000 +sql select * from streamtST1; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data01 != 7 then + print =====data01=$data01 + goto loop3 +endi + +if $data02 != 22 then + print =====data02=$data02 + goto loop3 +endi + +# row 1 +if $data11 != 3 then + print =====data11=$data11 + goto loop3 +endi + +if $data12 != 10 then + print =====data12=$data12 + goto loop3 +endi + +#row2 +if $data21 != 3 then + print =====data21=$data21 + goto loop3 +endi + +if $data22 != 11 then + print =====data22=$data22 + goto loop3 +endi + +#row 3 +if $data31 != 5 then + print =====data31=$data31 + goto loop3 +endi + +if $data32 != 60 then + print =====data32=$data32 + goto loop3 +endi + +print loop3 over + +$loop_count = 0 +loop4: +sleep 1000 +sql select * from streamtST1; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data01 != 7 then + print =====data01=$data01 + goto loop4 +endi + +if $data02 != 22 then + print =====data02=$data02 + goto loop4 +endi + +# row 1 +if $data11 != 3 then + print =====data11=$data11 + goto loop4 +endi + +if $data12 != 10 then + print =====data12=$data12 + goto loop4 +endi + +#row2 +if $data21 != 3 then + print =====data21=$data21 + goto loop4 +endi + +if $data22 != 11 then + print =====data22=$data22 + goto loop4 +endi + +#row 3 +if $data31 != 5 then + print =====data31=$data31 + goto loop4 +endi + +if $data32 != 60 then + print =====data32=$data32 + goto loop4 +endi + +print loop4 over + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/distributesession0.sim b/tests/script/tsim/stream/distributeSession0.sim similarity index 100% rename from tests/script/tsim/stream/distributesession0.sim rename to tests/script/tsim/stream/distributeSession0.sim diff --git a/tests/system-test/1-insert/create_retentions.py b/tests/system-test/1-insert/create_retentions.py index 5a0684e2ee..4b37eeb9a5 100644 --- a/tests/system-test/1-insert/create_retentions.py +++ b/tests/system-test/1-insert/create_retentions.py @@ -21,9 +21,9 @@ SINT_UN_COL = "c_sint_un" BINT_UN_COL = "c_bint_un" INT_UN_COL = "c_int_un" -BINARY_COL = "c8" -NCHAR_COL = "c9" -TS_COL = "c10" +BINARY_COL = "c_binary" +NCHAR_COL = "c_nchar" +TS_COL = "c_ts" NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] CHAR_COL = [ BINARY_COL, NCHAR_COL, ] @@ -51,12 +51,28 @@ class DataSet: binary_data : List[str] = None nchar_data : List[str] = None + def __post_init__(self): + self.ts_data = [] + self.int_data = [] + self.bint_data = [] + self.sint_data = [] + self.tint_data = [] + self.int_un_data = [] + self.bint_un_data = [] + self.sint_un_data = [] + self.tint_un_data = [] + self.float_data = [] + self.double_data = [] + self.bool_data = [] + self.binary_data = [] + self.nchar_data = [] + class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), True) + tdSql.init(conn.cursor(), False) @property def create_databases_sql_err(self): @@ -87,28 +103,28 @@ class TDTestCase: @property def create_stable_sql_err(self): return [ - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(ceil) watermark 1s maxdelay 1m", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(ceil) watermark 1s max_delay 1m", f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(count) watermark 1min", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay -1s", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay -1s", f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark -1m", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 1m ", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) maxdelay 1m ", + # f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 1m ", + # f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) max_delay 1m ", f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} binary(16)) tags (tag1 int) rollup(avg) watermark 1s", - f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) tags (tag1 int) rollup(avg) maxdelay 1m", - # f"create table ntb_1 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) rollup(avg) watermark 1s maxdelay 1s", + f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) tags (tag1 int) rollup(avg) max_delay 1m", + # f"create table ntb_1 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) rollup(avg) watermark 1s max_delay 1s", # f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) tags (tag1 int) " , # f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) " , # f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) " , # f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) " , - # watermark, maxdelay: [0, 900000], [ms, s, m, ?] - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 1u", + # watermark, max_delay: [0, 900000], [ms, s, m, ?] + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 1u", f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 1b", f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 900001ms", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 16m", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 901s", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 1h", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 0.2h", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 16m", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 901s", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 1h", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 0.2h", f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 0.002d", ] @@ -117,11 +133,11 @@ class TDTestCase: def create_stable_sql_current(self): return [ f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(avg)", - f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 5s maxdelay 1m", - f"create stable stb3 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(max) watermark 5s maxdelay 1m", - f"create stable stb4 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(sum) watermark 5s maxdelay 1m", - # f"create stable stb5 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(last) watermark 5s maxdelay 1m", - # f"create stable stb6 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s maxdelay 1m", + f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 5s max_delay 1m", + f"create stable stb3 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(max) watermark 5s max_delay 1m", + f"create stable stb4 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(sum) watermark 5s max_delay 1m", + # f"create stable stb5 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(last) watermark 5s max_delay 1m", + # f"create stable stb6 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s max_delay 1m", ] def test_create_stb(self): @@ -135,7 +151,7 @@ class TDTestCase: tdSql.checkRows(len(self.create_stable_sql_current)) # tdSql.execute("use db") # because db is a noraml database, not a rollup database, should not be able to create a rollup database - # tdSql.error(f"create stable nor_db_rollup_stb ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) file_factor 5.0") + # tdSql.error(f"create stable nor_db_rollup_stb ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 5s max_delay 1m") def test_create_databases(self): @@ -177,21 +193,6 @@ class TDTestCase: def __data_set(self, rows): data_set = DataSet() - # neg_data_set = DataSet() - data_set.ts_data = [] - data_set.int_data = [] - data_set.bint_data = [] - data_set.sint_data = [] - data_set.tint_data = [] - data_set.int_un_data = [] - data_set.bint_un_data = [] - data_set.sint_un_data = [] - data_set.tint_un_data = [] - data_set.float_data = [] - data_set.double_data = [] - data_set.bool_data = [] - data_set.binary_data = [] - data_set.nchar_data = [] for i in range(rows): data_set.ts_data.append(NOW + 1 * (rows - i)) @@ -226,6 +227,7 @@ class TDTestCase: return data_set def __insert_data(self): + tdLog.printNoPrefix("==========step: start inser data into tables now.....") data = self.__data_set(rows=self.rows) # now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) @@ -264,10 +266,10 @@ class TDTestCase: def run(self): self.rows = 10 - + tdSql.prepare() tdLog.printNoPrefix("==========step0:all check") - # self.all_test() + self.all_test() tdLog.printNoPrefix("==========step1:create table in normal database") tdSql.prepare() diff --git a/tests/system-test/1-insert/time_range_wise.py b/tests/system-test/1-insert/time_range_wise.py index 5387970d56..d4434987a6 100644 --- a/tests/system-test/1-insert/time_range_wise.py +++ b/tests/system-test/1-insert/time_range_wise.py @@ -17,25 +17,34 @@ TINT_COL = "c_tint" FLOAT_COL = "c_float" DOUBLE_COL = "c_double" BOOL_COL = "c_bool" -TINT_UN_COL = "c_tint_un" -SINT_UN_COL = "c_sint_un" -BINT_UN_COL = "c_bint_un" -INT_UN_COL = "c_int_un" +TINT_UN_COL = "c_utint" +SINT_UN_COL = "c_usint" +BINT_UN_COL = "c_ubint" +INT_UN_COL = "c_uint" BINARY_COL = "c_binary" NCHAR_COL = "c_nchar" TS_COL = "c_ts" - - NUM_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] CHAR_COL = [BINARY_COL, NCHAR_COL, ] BOOLEAN_COL = [BOOL_COL, ] TS_TYPE_COL = [TS_COL, ] +INT_TAG = "t_int" + +ALL_COL = [PRIMARY_COL, INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BINARY_COL, NCHAR_COL, BOOL_COL, TS_COL] +TAG_COL = [INT_TAG] + # insert data args： TIME_STEP = 10000 NOW = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) +# init db/table +DBNAME = "db" +STBNAME = "stb1" +CTBNAME = "ct1" +NTBNAME = "nt1" + @dataclass class DataSet: @@ -73,19 +82,25 @@ class DataSet: @dataclass class SMAschema: - creation : str = "CREATE" - index_name : str = "sma_index_1" - index_flag : str = "SMA INDEX" - operator : str = "ON" - tbname : str = None - watermark : str = None - maxdelay : str = None - func : Tuple[str] = None - interval : Tuple[str] = None - sliding : str = None - other : Any = None - drop : str = "DROP" - drop_flag : str = "INDEX" + creation : str = "CREATE" + index_name : str = "sma_index_1" + index_flag : str = "SMA INDEX" + operator : str = "ON" + tbname : str = None + watermark : str = "5s" + max_delay : str = "6m" + func : Tuple[str] = None + interval : Tuple[str] = ("6m", "10s") + sliding : str = "6m" + other : Any = None + drop : str = "DROP" + drop_flag : str = "INDEX" + querySmaOptimize : int = 1 + show : str = "SHOW" + show_msg : str = "INDEXES" + show_oper : str = "FROM" + dbname : str = None + rollup_db : bool = False def __post_init__(self): if isinstance(self.other, dict): @@ -111,8 +126,8 @@ class SMAschema: self.watermark = v del self.other[k] - if k.lower() == "maxdelay" and isinstance(v, str) and not self.maxdelay: - self.maxdelay = v + if k.lower() == "max_delay" and isinstance(v, str) and not self.max_delay: + self.max_delay = v del self.other[k] if k.lower() == "functions" and isinstance(v, tuple) and not self.func: @@ -131,12 +146,36 @@ class SMAschema: self.drop_flag = v del self.other[k] + if k.lower() == "show_msg" and isinstance(v, str) and not self.show_msg: + self.show_msg = v + del self.other[k] + + if k.lower() == "dbname" and isinstance(v, str) and not self.dbname: + self.dbname = v + del self.other[k] + + if k.lower() == "show_oper" and isinstance(v, str) and not self.show_oper: + self.show_oper = v + del self.other[k] + + if k.lower() == "rollup_db" and isinstance(v, bool) and not self.rollup_db: + self.rollup_db = v + del self.other[k] + + + +# from ...pytest.util.sql import * +# from ...pytest.util.constant import * class TDTestCase: + updatecfgDict = {"querySmaOptimize": 1} def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) + self.precision = "ms" + self.sma_count = 0 + self.sma_created_index = [] """ create sma index : @@ -155,13 +194,17 @@ class TDTestCase: if sma.func: sql += f" function({', '.join(sma.func)})" if sma.interval: - sql += f" interval({', '.join(sma.interval)})" + interval, offset = self.__get_interval_offset(sma.interval) + if offset: + sql += f" interval({interval}, {offset})" + else: + sql += f" interval({interval})" if sma.sliding: sql += f" sliding({sma.sliding})" if sma.watermark: sql += f" watermark {sma.watermark}" - if sma.maxdelay: - sql += f" maxdelay {sma.maxdelay}" + if sma.max_delay: + sql += f" max_delay {sma.max_delay}" if isinstance(sma.other, dict): for k,v in sma.other.items(): if isinstance(v,tuple) or isinstance(v, list): @@ -171,53 +214,305 @@ class TDTestCase: if isinstance(sma.other, tuple) or isinstance(sma.other, list): sql += " ".join(sma.other) if isinstance(sma.other, int) or isinstance(sma.other, float) or isinstance(sma.other, str): - sql += sma.other + sql += f" {sma.other}" return sql - def sma_create_check(self, sma:SMAschema): + def __get_sma_func_col(self, func): + cols = [] + if isinstance(func, str): + cols.append( func.split("(")[-1].split(")")[0] ) + elif isinstance(func, tuple) or isinstance(func, list): + for func_col in func: + cols.append(func_col.split("(")[-1].split(")")[0]) + else: + cols = [] + return cols + + def __check_sma_func(self, func:tuple): + if not isinstance(func, str) and not isinstance(func, tuple) and not isinstance(func, list): + return False + if isinstance(func, str) : + if "(" not in func or ")" not in func: + return False + if func.split("(")[0].upper() not in SMA_INDEX_FUNCTIONS: + return False + if func.split("(")[1].split(")")[0] not in ALL_COL and func.split("(")[1].split(")")[0] not in TAG_COL : + return False + if isinstance(func, tuple) or isinstance(func, list): + for arg in func: + if not isinstance(arg, str): + return False + if "(" not in arg or ")" not in arg: + return False + if arg.split("(")[0].upper() not in SMA_INDEX_FUNCTIONS: + return False + if arg.split("(")[1].split(")")[0] not in ALL_COL and arg.split("(")[1].split(")")[0] not in TAG_COL : + return False + return True + + def __check_sma_watermark(self, arg): + if not arg: + return False + if not isinstance(arg, str): + return False + if arg[-1] not in SMA_WATMARK_MAXDELAY_INIT: + return False + if len(arg) == 1: + return False + if not arg[:-1].isdecimal(): + return False + if tdSql.get_times(arg) > WATERMARK_MAX: + return False + if tdSql.get_times(arg) < WATERMARK_MIN: + return False + + return True + + def __check_sma_max_delay(self, arg): + if not self.__check_sma_watermark(arg): + return False + if tdSql.get_times(arg) < MAX_DELAY_MIN: + return False + + return True + + def __check_sma_sliding(self, arg): + if not isinstance(arg, str): + return False + if arg[-1] not in TAOS_TIME_INIT: + return False + if len(arg) == 1: + return False + if not arg[:-1].isdecimal(): + return False + + return True + + def __get_interval_offset(self, args): + if isinstance(args, str): + interval, offset = args, None + elif isinstance(args,tuple) or isinstance(args, list): + if len(args) == 1: + interval, offset = args[0], None + elif len(args) == 2: + interval, offset = args + else: + interval, offset = False, False + else: + interval, offset = False, False + + return interval, offset + + def __check_sma_interval(self, args): + if not isinstance(args, tuple) and not isinstance(args,str): + return False + interval, offset = self.__get_interval_offset(args) + if not interval: + return False + if not self.__check_sma_sliding(interval): + return False + if tdSql.get_times(interval) < INTERVAL_MIN: + return False + if offset: + if not self.__check_sma_sliding(offset): + return False + if tdSql.get_times(interval) <= tdSql.get_times(offset) : + return False + + return True + + def __sma_create_check(self, sma:SMAschema): + if self.updatecfgDict["querySmaOptimize"] == 0: + return False + # # TODO: if database is a rollup-db, can not create sma index + # tdSql.query("select database()") + # if sma.rollup_db : + # return False tdSql.query("show stables") + if not sma.tbname: + return False stb_in_list = False for row in tdSql.queryResult: if sma.tbname == row[0]: stb_in_list = True - break if not stb_in_list: - tdSql.error(self.__create_sma_index(sma)) - if not sma.creation: - tdSql.error(self.__create_sma_index(sma)) - if not sma.index_flag: - tdSql.error(self.__create_sma_index(sma)) - if not sma.index_name: - tdSql.error(self.__create_sma_index(sma)) - if not sma.operator: - tdSql.error(self.__create_sma_index(sma)) - if not sma.tbname: - tdSql.error(self.__create_sma_index(sma)) - if not sma.func: - tdSql.error(self.__create_sma_index(sma)) - if not sma.interval: - tdSql.error(self.__create_sma_index(sma)) - if not sma.sliding: - tdSql.error(self.__create_sma_index(sma)) + return False + if not sma.creation or not isinstance(sma.creation, str) or sma.creation.upper() != "CREATE": + return False + if not sma.index_flag or not isinstance(sma.index_flag, str) or sma.index_flag.upper() != "SMA INDEX" : + return False + if not sma.index_name or not isinstance(sma.index_name, str) or sma.index_name.upper() in TAOS_KEYWORDS: + return False + if not sma.operator or not isinstance(sma.operator, str) or sma.operator.upper() != "ON": + return False + + if not sma.func or not self.__check_sma_func(sma.func): + return False + tdSql.query(f"desc {sma.tbname}") + _col_list = [] + for col_row in tdSql.queryResult: + _col_list.append(col_row[0]) + _sma_func_cols = self.__get_sma_func_col(sma.func) + for _sma_func_col in _sma_func_cols: + if _sma_func_col not in _col_list: + return False + + if sma.sliding and not self.__check_sma_sliding(sma.sliding): + return False + interval, _ = self.__get_interval_offset(sma.interval) + if not sma.interval or not self.__check_sma_interval(sma.interval) : + return False + if sma.sliding and tdSql.get_times(interval) < tdSql.get_times(sma.sliding): + return False + if sma.watermark and not self.__check_sma_watermark(sma.watermark): + return False + if sma.max_delay and not self.__check_sma_max_delay(sma.max_delay): + return False if sma.other: + return False + + return True + + def sma_create_check(self, sma:SMAschema): + if self.__sma_create_check(sma): + tdSql.query(self.__create_sma_index(sma)) + self.sma_count += 1 + self.sma_created_index.append(sma.index_name) + tdSql.query("show streams") + tdSql.checkRows(self.sma_count) + + else: tdSql.error(self.__create_sma_index(sma)) + def __drop_sma_index(self, sma:SMAschema): + sql = f"{sma.drop} {sma.drop_flag} {sma.index_name}" + return sql + + def __sma_drop_check(self, sma:SMAschema): + if not sma.drop: + return False + if not sma.drop_flag: + return False + if not sma.index_name: + return False + + return True + + def sma_drop_check(self, sma:SMAschema): + if self.__sma_drop_check(sma): + tdSql.query(self.__drop_sma_index(sma)) + print(self.__drop_sma_index(sma)) + self.sma_count -= 1 + self.sma_created_index = list(filter(lambda x: x != sma.index_name, self.sma_created_index)) + tdSql.query("show streams") + tdSql.checkRows(self.sma_count) + + else: + tdSql.error(self.__drop_sma_index(sma)) + + def __show_sma_index(self, sma:SMAschema): + sql = f"{sma.show} {sma.show_msg} {sma.show_oper} {sma.tbname}" + return sql + + def __sma_show_check(self, sma:SMAschema): + if not sma.show: + return False + if not sma.show_msg: + return False + if not sma.show_oper: + return False + if not sma.tbname: + return False + + return True + + def sma_show_check(self, sma:SMAschema): + if self.__sma_show_check(sma): + tdSql.query(self.__show_sma_index(sma)) + tdSql.checkRows(self.sma_count) + else: + tdSql.error(self.__show_sma_index(sma)) + + @property + def __create_sma_sql(self): + err_sqls = [] + cur_sqls = [] + # err_set + # # case 1: required fields check + err_sqls.append( SMAschema(creation="", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(index_name="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(index_flag="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(operator="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(tbname="", func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(func=("",),tbname=STBNAME ) ) + err_sqls.append( SMAschema(interval=(""),tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + + # # case 2: err fields + err_sqls.append( SMAschema(creation="show",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(creation="alter",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(creation="select",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + + err_sqls.append( SMAschema(index_flag="SMA INDEXES", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(index_flag="SMA INDEX ,", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + err_sqls.append( SMAschema(index_name="tbname", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + + + # current_set + + cur_sqls.append( SMAschema(max_delay="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + cur_sqls.append( SMAschema(watermark="",index_name="sma_index_2",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + cur_sqls.append( SMAschema(sliding="",index_name='sma_index_3',tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) + + + return err_sqls, cur_sqls + + def test_create_sma(self): + err_sqls , cur_sqls = self.__create_sma_sql + for err_sql in err_sqls: + self.sma_create_check(err_sql) + for cur_sql in cur_sqls: + self.sma_create_check(cur_sql) + + @property + def __drop_sma_sql(self): + err_sqls = [] + cur_sqls = [] + # err_set + ## case 1: required fields check + err_sqls.append( SMAschema(drop="") ) + err_sqls.append( SMAschema(drop_flag="") ) + err_sqls.append( SMAschema(index_name="") ) + + for index in self.sma_created_index: + cur_sqls.append(SMAschema(index_name=index)) + + return err_sqls, cur_sqls + + def test_drop_sma(self): + err_sqls , cur_sqls = self.__drop_sma_sql + for err_sql in err_sqls: + self.sma_drop_check(err_sql) + # for cur_sql in cur_sqls: + # self.sma_drop_check(cur_sql) def all_test(self): + self.test_create_sma() + self.test_drop_sma() + pass def __create_tb(self): tdLog.printNoPrefix("==========step: create table") - create_stb_sql = f'''create table stb1( + create_stb_sql = f'''create table {STBNAME}( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp, {TINT_UN_COL} tinyint unsigned, {SINT_UN_COL} smallint unsigned, {INT_UN_COL} int unsigned, {BINT_UN_COL} bigint unsigned - ) tags (tag1 int) + ) tags ({INT_TAG} int) ''' - create_ntb_sql = f'''create table t1( + create_ntb_sql = f'''create table {NTBNAME}( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp, @@ -253,6 +548,7 @@ class TDTestCase: return data_set def __insert_data(self): + tdLog.printNoPrefix("==========step: start inser data into tables now.....") data = self.__data_set(rows=self.rows) # now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) @@ -278,7 +574,7 @@ class TDTestCase: tdSql.execute( f"insert into ct4 values ( {NOW - i * int(TIME_STEP * 0.8) }, {row_data} )") tdSql.execute( - f"insert into t1 values ( {NOW - i * int(TIME_STEP * 1.2)}, {row_data} )") + f"insert into {NTBNAME} values ( {NOW - i * int(TIME_STEP * 1.2)}, {row_data} )") tdSql.execute( f"insert into ct2 values ( {NOW + int(TIME_STEP * 0.6)}, {null_data} )") @@ -295,28 +591,31 @@ class TDTestCase: f"insert into ct4 values ( {NOW - self.rows * int(TIME_STEP * 0.39)}, {null_data} )") tdSql.execute( - f"insert into t1 values ( {NOW + int(TIME_STEP * 1.2)}, {null_data} )") + f"insert into {NTBNAME} values ( {NOW + int(TIME_STEP * 1.2)}, {null_data} )") tdSql.execute( - f"insert into t1 values ( {NOW - (self.rows + 1) * int(TIME_STEP * 1.2)}, {null_data} )") + f"insert into {NTBNAME} values ( {NOW - (self.rows + 1) * int(TIME_STEP * 1.2)}, {null_data} )") tdSql.execute( - f"insert into t1 values ( {NOW - self.rows * int(TIME_STEP * 0.59)}, {null_data} )") + f"insert into {NTBNAME} values ( {NOW - self.rows * int(TIME_STEP * 0.59)}, {null_data} )") def run(self): - sma1 = SMAschema(func=("min(c1)","max(c2)")) - sql1 = self.__create_sma_index(sma1) - print("================") - print(sql1) - # a = DataSet() - # return self.rows = 10 tdLog.printNoPrefix("==========step0:all check") - # self.all_test() tdLog.printNoPrefix("==========step1:create table in normal database") tdSql.prepare() self.__create_tb() - self.__insert_data() + # self.__insert_data() + self.all_test() + + # drop databases, create same name db、stb and sma index + # tdSql.prepare() + # self.__create_tb() + # self.__insert_data() + # self.all_test() + + + return tdLog.printNoPrefix("==========step2:create table in rollup database") diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index 244cccb041..961a6446b5 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -23,7 +23,6 @@ class TDTestCase: self.time_step = 1000 def insert_datas_and_check_abs(self ,tbnums , rownums , time_step ): - tdLog.info(" prepare datas for auto check abs function ") tdSql.execute(" create database test ") @@ -36,7 +35,7 @@ class TDTestCase: ts = self.ts for row in range(rownums): - ts += time_step*row + ts = self.ts + time_step*row c1 = random.randint(0,10000) c2 = random.randint(0,100000) c3 = random.randint(0,125) @@ -538,25 +537,41 @@ class TDTestCase: # tdSql.query(" select sum(c1) from stb1 where t1+10 >1; ") #　taosd crash tdSql.query("select c1 ,t1 from stb1 where t1 =0 ") tdSql.checkRows(13) - # tdSql.query("select t1 from stb1 where t1 >0 ") - # tdSql.checkRows(3) + tdSql.query("select t1 from stb1 where t1 >0 ") + tdSql.checkRows(3) + tdSql.query("select t1 from stb1 where t1 =3 ") + tdSql.checkRows(1) # tdSql.query("select sum(t1) from (select c1 ,t1 from stb1)") # tdSql.checkData(0,0,61) # tdSql.query("select distinct(c1) ,t1 from stb1") # tdSql.checkRows(20) - # tdSql.query("select max(t2) , t1 ,c1, t2 from stb1") - # tdSql.checkData(0,3,33333) + tdSql.query("select max(t2) , t1 ,c1, t2 from stb1") + tdSql.checkData(0,3,33333) # tag filter with abs function - # tdSql.query("select t1 from stb1 where abs(t1)=1") - # tdSql.checkRows(1) + tdSql.query("select t1 from stb1 where abs(t1)=1") + tdSql.checkRows(1) tdSql.query("select t1 from stb1 where abs(c1+t1)=1") tdSql.checkRows(1) - # tdSql.query("select t1 from stb1 where abs(t1+c1)=1") - # tdSql.checkRows(1) + tdSql.checkData(0,0,0) + tdSql.query( "select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1") + def support_super_table_test(self): + tdSql.execute(" use testdb ") + self.check_result_auto( " select c1 from stb1 order by ts " , "select abs(c1) from stb1 order by ts" ) + self.check_result_auto( " select c1 from stb1 order by tbname " , "select abs(c1) from stb1 order by tbname" ) + self.check_result_auto( " select c1 from stb1 where c1 > 0 order by tbname " , "select abs(c1) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select c1 from stb1 where c1 > 0 order by tbname " , "select abs(c1) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto( " select t1,c1 from stb1 order by ts " , "select t1, abs(c1) from stb1 order by ts" ) + self.check_result_auto( " select t2,c1 from stb1 order by tbname " , "select t2 ,abs(c1) from stb1 order by tbname" ) + self.check_result_auto( " select t3,c1 from stb1 where c1 > 0 order by tbname " , "select t3 ,abs(c1) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select t4,c1 from stb1 where c1 > 0 order by tbname " , "select t4 , abs(c1) from stb1 where c1 > 0 order by tbname" ) + pass + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -593,6 +608,10 @@ class TDTestCase: self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step) + tdLog.printNoPrefix("==========step8: check abs result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/arccos.py b/tests/system-test/2-query/arccos.py index e15f7675f2..edb9e25c11 100644 --- a/tests/system-test/2-query/arccos.py +++ b/tests/system-test/2-query/arccos.py @@ -479,6 +479,20 @@ class TDTestCase: tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) self.check_result_auto_acos("select num1,num2 from tb3;" , "select acos(num1),acos(num2) from tb3") + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_acos( " select c5 from stb1 order by ts " , "select acos(c5) from stb1 order by ts" ) + self.check_result_auto_acos( " select c5 from stb1 order by tbname " , "select acos(c5) from stb1 order by tbname" ) + self.check_result_auto_acos( " select c5 from stb1 where c1 > 0 order by tbname " , "select acos(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_acos( " select c5 from stb1 where c1 > 0 order by tbname " , "select acos(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_acos( " select t1,c5 from stb1 order by ts " , "select acos(t1), acos(c5) from stb1 order by ts" ) + self.check_result_auto_acos( " select t1,c5 from stb1 order by tbname " , "select acos(t1) ,acos(c5) from stb1 order by tbname" ) + self.check_result_auto_acos( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select acos(t1) ,acos(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_acos( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select acos(t1) , acos(c5) from stb1 where c1 > 0 order by tbname" ) + pass + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -512,6 +526,14 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step7: acos filter query ============") + + self.abs_func_filter() + + tdLog.printNoPrefix("==========step8: check acos result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/arcsin.py b/tests/system-test/2-query/arcsin.py index ee134a56a8..faed5ef3c4 100644 --- a/tests/system-test/2-query/arcsin.py +++ b/tests/system-test/2-query/arcsin.py @@ -479,6 +479,20 @@ class TDTestCase: tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) self.check_result_auto_asin("select num1,num2 from tb3;" , "select asin(num1),asin(num2) from tb3") + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_asin( " select c5 from stb1 order by ts " , "select asin(c5) from stb1 order by ts" ) + self.check_result_auto_asin( " select c5 from stb1 order by tbname " , "select asin(c5) from stb1 order by tbname" ) + self.check_result_auto_asin( " select c5 from stb1 where c1 > 0 order by tbname " , "select asin(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_asin( " select c5 from stb1 where c1 > 0 order by tbname " , "select asin(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_asin( " select t1,c5 from stb1 order by ts " , "select asin(t1), asin(c5) from stb1 order by ts" ) + self.check_result_auto_asin( " select t1,c5 from stb1 order by tbname " , "select asin(t1) ,asin(c5) from stb1 order by tbname" ) + self.check_result_auto_asin( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select asin(t1) ,asin(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_asin( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select asin(t1) , asin(c5) from stb1 where c1 > 0 order by tbname" ) + pass + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -512,6 +526,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step8: check asin result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/arctan.py b/tests/system-test/2-query/arctan.py index 1890c4605c..80d28b5ee5 100644 --- a/tests/system-test/2-query/arctan.py +++ b/tests/system-test/2-query/arctan.py @@ -476,6 +476,20 @@ class TDTestCase: tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) self.check_result_auto_atan("select num1,num2 from tb3;" , "select atan(num1),atan(num2) from tb3") + + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_atan( " select c5 from stb1 order by ts " , "select atan(c5) from stb1 order by ts" ) + self.check_result_auto_atan( " select c5 from stb1 order by tbname " , "select atan(c5) from stb1 order by tbname" ) + self.check_result_auto_atan( " select c5 from stb1 where c1 > 0 order by tbname " , "select atan(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_atan( " select c5 from stb1 where c1 > 0 order by tbname " , "select atan(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_atan( " select t1,c5 from stb1 order by ts " , "select atan(t1), atan(c5) from stb1 order by ts" ) + self.check_result_auto_atan( " select t1,c5 from stb1 order by tbname " , "select atan(t1) ,atan(c5) from stb1 order by tbname" ) + self.check_result_auto_atan( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select atan(t1) ,atan(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_atan( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select atan(t1) , atan(c5) from stb1 where c1 > 0 order by tbname" ) + pass def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -509,6 +523,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step8: check arctan result of stable query ============") + + self.support_super_table_test() + def stop(self): diff --git a/tests/system-test/2-query/ceil.py b/tests/system-test/2-query/ceil.py index 4196a5a8ce..b269b54a17 100644 --- a/tests/system-test/2-query/ceil.py +++ b/tests/system-test/2-query/ceil.py @@ -427,6 +427,18 @@ class TDTestCase: self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select ceil(c1+1) ,ceil(c2) , ceil(c3*1) , ceil(c4/2), ceil(c5)/2, ceil(c6) from sub1_bound ") + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto( " select c5 from stb1 order by ts " , "select ceil(c5) from stb1 order by ts" ) + self.check_result_auto( " select c5 from stb1 order by tbname " , "select ceil(c5) from stb1 order by tbname" ) + self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select ceil(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select ceil(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto( " select t1,c5 from stb1 order by ts " , "select ceil(t1), ceil(c5) from stb1 order by ts" ) + self.check_result_auto( " select t1,c5 from stb1 order by tbname " , "select ceil(t1) ,ceil(c5) from stb1 order by tbname" ) + self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select ceil(t1) ,ceil(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select ceil(t1) , ceil(c5) from stb1 where c1 > 0 order by tbname" ) + pass def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -455,6 +467,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step7: check ceil result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/cos.py b/tests/system-test/2-query/cos.py index 0cb9f66937..1165d8d681 100644 --- a/tests/system-test/2-query/cos.py +++ b/tests/system-test/2-query/cos.py @@ -476,6 +476,19 @@ class TDTestCase: tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) self.check_result_auto_cos("select num1,num2 from tb3;" , "select cos(num1),cos(num2) from tb3") + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_cos( " select c5 from stb1 order by ts " , "select cos(c5) from stb1 order by ts" ) + self.check_result_auto_cos( " select c5 from stb1 order by tbname " , "select cos(c5) from stb1 order by tbname" ) + self.check_result_auto_cos( " select c5 from stb1 where c1 > 0 order by tbname " , "select cos(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_cos( " select c5 from stb1 where c1 > 0 order by tbname " , "select cos(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_cos( " select t1,c5 from stb1 order by ts " , "select cos(t1), cos(c5) from stb1 order by ts" ) + self.check_result_auto_cos( " select t1,c5 from stb1 order by tbname " , "select cos(t1) ,cos(c5) from stb1 order by tbname" ) + self.check_result_auto_cos( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select cos(t1) ,cos(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_cos( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select cos(t1) , cos(c5) from stb1 where c1 > 0 order by tbname" ) + pass def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -509,6 +522,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step8: check cos result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/floor.py b/tests/system-test/2-query/floor.py index c955c24e05..7362191958 100644 --- a/tests/system-test/2-query/floor.py +++ b/tests/system-test/2-query/floor.py @@ -427,6 +427,19 @@ class TDTestCase: self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select floor(c1+1) ,floor(c2) , floor(c3*1) , floor(c4/2), floor(c5)/2, floor(c6) from sub1_bound ") + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto( " select c5 from stb1 order by ts " , "select floor(c5) from stb1 order by ts" ) + self.check_result_auto( " select c5 from stb1 order by tbname " , "select floor(c5) from stb1 order by tbname" ) + self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select floor(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select floor(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto( " select t1,c5 from stb1 order by ts " , "select floor(t1), floor(c5) from stb1 order by ts" ) + self.check_result_auto( " select t1,c5 from stb1 order by tbname " , "select floor(t1) ,floor(c5) from stb1 order by tbname" ) + self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select floor(t1) ,floor(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select floor(t1) , floor(c5) from stb1 where c1 > 0 order by tbname" ) + pass + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -454,6 +467,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step7: check floor result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 5ff11c84dd..df6390f59c 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), True) + tdSql.init(conn.cursor(), False) def __query_condition(self,tbname): query_condition = [] diff --git a/tests/system-test/2-query/join2.py b/tests/system-test/2-query/join2.py index 40da41eee7..5533cb840e 100644 --- a/tests/system-test/2-query/join2.py +++ b/tests/system-test/2-query/join2.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), True) + tdSql.init(conn.cursor(), False) def __query_condition(self,tbname): query_condition = [] diff --git a/tests/system-test/2-query/log.py b/tests/system-test/2-query/log.py index 907ba329ee..f9d6e91199 100644 --- a/tests/system-test/2-query/log.py +++ b/tests/system-test/2-query/log.py @@ -670,7 +670,19 @@ class TDTestCase: tdSql.checkData(0,2,math.log(32767.000000000,2)) tdSql.checkData(0,3,math.log(63.500000000,2)) tdSql.checkData(0,4,63.999401166) - + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_log2( " select c5 from stb1 order by ts " , "select log(c5,2) from stb1 order by ts" ) + self.check_result_auto_log2( " select c5 from stb1 order by tbname " , "select log(c5,2) from stb1 order by tbname" ) + self.check_result_auto_log2( " select c5 from stb1 where c1 > 0 order by tbname " , "select log(c5,2) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_log2( " select c5 from stb1 where c1 > 0 order by tbname " , "select log(c5,2) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_log2( " select t1,c5 from stb1 order by ts " , "select log(t1,2), log(c5,2) from stb1 order by ts" ) + self.check_result_auto_log2( " select t1,c5 from stb1 order by tbname " , "select log(t1,2) ,log(c5,2) from stb1 order by tbname" ) + self.check_result_auto_log2( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select log(t1,2) ,log(c5,2) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_log2( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select log(t1,2) , log(c5,2) from stb1 where c1 > 0 order by tbname" ) + pass def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -707,7 +719,9 @@ class TDTestCase: self.abs_func_filter() - + tdLog.printNoPrefix("==========step9: check log result of stable query ============") + + self.support_super_table_test() def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/pow.py b/tests/system-test/2-query/pow.py index 8b0137b411..c67162961b 100644 --- a/tests/system-test/2-query/pow.py +++ b/tests/system-test/2-query/pow.py @@ -606,7 +606,23 @@ class TDTestCase: tdSql.checkData(0,3,math.pow(63.500000000,2)) tdSql.checkData(0,5,None) + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_pow2( " select c5 from stb1 order by ts " , "select pow(c5,2) from stb1 order by ts" ) + self.check_result_auto_pow2( " select c5 from stb1 order by tbname " , "select pow(c5,2) from stb1 order by tbname" ) + self.check_result_auto_pow2( " select c5 from stb1 where c1 > 0 order by tbname " , "select pow(c5,2) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_pow2( " select c5 from stb1 where c1 > 0 order by tbname " , "select pow(c5,2) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_pow2( " select t1,c5 from stb1 order by ts " , "select pow(t1,2), pow(c5,2) from stb1 order by ts" ) + self.check_result_auto_pow2( " select t1,c5 from stb1 order by tbname " , "select pow(t1,2) ,pow(c5,2) from stb1 order by tbname" ) + self.check_result_auto_pow2( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select pow(t1,2) ,pow(c5,2) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_pow2( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select pow(t1,2) , pow(c5,2) from stb1 where c1 > 0 order by tbname" ) + pass + + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -642,7 +658,9 @@ class TDTestCase: self.abs_func_filter() - + tdLog.printNoPrefix("==========step9: check pow result of stable query ============") + + self.support_super_table_test() def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/round.py b/tests/system-test/2-query/round.py index 223e56bce6..cc272abf42 100644 --- a/tests/system-test/2-query/round.py +++ b/tests/system-test/2-query/round.py @@ -432,6 +432,20 @@ class TDTestCase: self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select round(c1+1) ,round(c2) , round(c3*1) , round(c4/2), round(c5)/2, round(c6) from sub1_bound ") + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto( " select c5 from stb1 order by ts " , "select round(c5) from stb1 order by ts" ) + self.check_result_auto( " select c5 from stb1 order by tbname " , "select round(c5) from stb1 order by tbname" ) + self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select round(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select round(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto( " select t1,c5 from stb1 order by ts " , "select round(t1), round(c5) from stb1 order by ts" ) + self.check_result_auto( " select t1,c5 from stb1 order by tbname " , "select round(t1) ,round(c5) from stb1 order by tbname" ) + self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select round(t1) ,round(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select round(t1) , round(c5) from stb1 where c1 > 0 order by tbname" ) + pass + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -459,6 +473,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step7: check round result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/sin.py b/tests/system-test/2-query/sin.py index 62d42a991e..2c4d90d3e7 100644 --- a/tests/system-test/2-query/sin.py +++ b/tests/system-test/2-query/sin.py @@ -476,6 +476,19 @@ class TDTestCase: tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) self.check_result_auto_sin("select num1,num2 from tb3;" , "select sin(num1),sin(num2) from tb3") + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_sin( " select c5 from stb1 order by ts " , "select sin(c5) from stb1 order by ts" ) + self.check_result_auto_sin( " select c5 from stb1 order by tbname " , "select sin(c5) from stb1 order by tbname" ) + self.check_result_auto_sin( " select c5 from stb1 where c1 > 0 order by tbname " , "select sin(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_sin( " select c5 from stb1 where c1 > 0 order by tbname " , "select sin(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_sin( " select t1,c5 from stb1 order by ts " , "select sin(t1), sin(c5) from stb1 order by ts" ) + self.check_result_auto_sin( " select t1,c5 from stb1 order by tbname " , "select sin(t1) ,sin(c5) from stb1 order by tbname" ) + self.check_result_auto_sin( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select sin(t1) ,sin(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_sin( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select sin(t1) , sin(c5) from stb1 where c1 > 0 order by tbname" ) + pass def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -509,6 +522,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step8: check sin result of stable query ============") + + self.support_super_table_test() + def stop(self): diff --git a/tests/system-test/2-query/sqrt.py b/tests/system-test/2-query/sqrt.py index 28e869e044..772056fd93 100644 --- a/tests/system-test/2-query/sqrt.py +++ b/tests/system-test/2-query/sqrt.py @@ -505,7 +505,19 @@ class TDTestCase: tdSql.checkData(0,2,math.sqrt(32767.000000000)) tdSql.checkData(0,3,math.sqrt(63.500000000)) - + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_sqrt( " select c5 from stb1 order by ts " , "select sqrt(c5) from stb1 order by ts" ) + self.check_result_auto_sqrt( " select c5 from stb1 order by tbname " , "select sqrt(c5) from stb1 order by tbname" ) + self.check_result_auto_sqrt( " select c5 from stb1 where c1 > 0 order by tbname " , "select sqrt(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_sqrt( " select c5 from stb1 where c1 > 0 order by tbname " , "select sqrt(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_sqrt( " select t1,c5 from stb1 order by ts " , "select sqrt(t1), sqrt(c5) from stb1 order by ts" ) + self.check_result_auto_sqrt( " select t1,c5 from stb1 order by tbname " , "select sqrt(t1) ,sqrt(c5) from stb1 order by tbname" ) + self.check_result_auto_sqrt( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select sqrt(t1) ,sqrt(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_sqrt( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select sqrt(t1) , sqrt(c5) from stb1 where c1 > 0 order by tbname" ) + pass + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -541,6 +553,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step9: check sqrt result of stable query ============") + + self.support_super_table_test() + def stop(self): diff --git a/tests/system-test/2-query/substr.py b/tests/system-test/2-query/substr.py index 79b5ac515b..f833a42b57 100644 --- a/tests/system-test/2-query/substr.py +++ b/tests/system-test/2-query/substr.py @@ -31,7 +31,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(),False) def __substr_condition(self): # sourcery skip: extract-method substr_condition = [] diff --git a/tests/system-test/2-query/tail.py b/tests/system-test/2-query/tail.py index 0e2110f912..a4c85cc5ed 100644 --- a/tests/system-test/2-query/tail.py +++ b/tests/system-test/2-query/tail.py @@ -188,8 +188,8 @@ class TDTestCase: def check_tail_table(self , tbname , col_name , tail_rows , offset): tail_sql = f"select tail({col_name} , {tail_rows} , {offset}) from {tbname}" - equal_sql = f"select {col_name} from (select ts , {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}) order by ts" - #equal_sql = f"select {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}" + #equal_sql = f"select {col_name} from (select ts , {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}) order by ts" + equal_sql = f"select {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}" tdSql.query(tail_sql) tail_result = tdSql.queryResult @@ -404,7 +404,7 @@ class TDTestCase: f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) - tdSql.query("select tail(c2,2) from sub1_bound") + tdSql.query("select tail(c2,2) from sub1_bound order by 1 desc") tdSql.checkRows(2) tdSql.checkData(0,0,9223372036854775803) diff --git a/tests/system-test/2-query/tan.py b/tests/system-test/2-query/tan.py index c8914d18c0..9610ffef24 100644 --- a/tests/system-test/2-query/tan.py +++ b/tests/system-test/2-query/tan.py @@ -476,6 +476,20 @@ class TDTestCase: tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) self.check_result_auto_tan("select num1,num2 from tb3;" , "select tan(num1),tan(num2) from tb3") + + def support_super_table_test(self): + tdSql.execute(" use db ") + self.check_result_auto_tan( " select c5 from stb1 order by ts " , "select tan(c5) from stb1 order by ts" ) + self.check_result_auto_tan( " select c5 from stb1 order by tbname " , "select tan(c5) from stb1 order by tbname" ) + self.check_result_auto_tan( " select c5 from stb1 where c1 > 0 order by tbname " , "select tan(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_tan( " select c5 from stb1 where c1 > 0 order by tbname " , "select tan(c5) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto_tan( " select t1,c5 from stb1 order by ts " , "select tan(t1), tan(c5) from stb1 order by ts" ) + self.check_result_auto_tan( " select t1,c5 from stb1 order by tbname " , "select tan(t1) ,tan(c5) from stb1 order by tbname" ) + self.check_result_auto_tan( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select tan(t1) ,tan(c5) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_tan( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select tan(t1) , tan(c5) from stb1 where c1 > 0 order by tbname" ) + pass + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -509,6 +523,10 @@ class TDTestCase: self.abs_func_filter() + tdLog.printNoPrefix("==========step8: check tan result of stable query ============") + + self.support_super_table_test() + def stop(self): tdSql.close() diff --git a/tests/system-test/7-tmq/stbTagFilter.py b/tests/system-test/7-tmq/stbTagFilter.py new file mode 100644 index 0000000000..2a2cb40c09 --- /dev/null +++ b/tests/system-test/7-tmq/stbTagFilter.py @@ -0,0 +1,113 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db2', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + + topicNameList = ['topic1'] + expectRowsList = [] + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdLog.info("create stb") + tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'], ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.asyncInsertData(paraDict) + + tdLog.info("create topics from stb with filter") + # queryString = "select ts, sin(c1), pow(c2,3) from %s.%s where t2 == 'beijing' or t2 == 'changsha'" %(paraDict['dbName'], paraDict['stbName']) + queryString = "select * from %s.%s where t2 == 'beijing' or t2 == 'changsha'" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # start tmq consume processor + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:2000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + # tmqCom.getStartCommitNotifyFromTmqsim() + tmqCom.getStartConsumeNotifyFromTmqsim() + tdLog.info("create some new ctb") + paraDict['ctbStartIdx'] = paraDict['ctbStartIdx'] + paraDict['ctbNum'] + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'], ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data into new ctb") + pThread = tmqCom.asyncInsertData(paraDict) + + pThread.join() + tdLog.info("wait insert end") + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectRowsList[0] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.exit("0 tmq consume rows error!") + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + tdSql.prepare() + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqCheckData1.py b/tests/system-test/7-tmq/tmqCheckData1.py new file mode 100644 index 0000000000..6cf849d1b9 --- /dev/null +++ b/tests/system-test/7-tmq/tmqCheckData1.py @@ -0,0 +1,180 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def checkFileContent(self, consumerId, queryString): + buildPath = tdCom.getBuildPath() + cfgPath = tdCom.getClientCfgPath() + dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) + cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) + tdLog.info(cmdStr) + os.system(cmdStr) + + consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) + tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) + + consumeFile = open(consumeRowsFile, mode='r') + queryFile = open(dstFile, mode='r') + + # skip first line for it is schema + queryFile.readline() + + while True: + dst = queryFile.readline() + src = consumeFile.readline() + + if dst: + if dst != src: + tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) + else: + break + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db1', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 1, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + + topicNameList = ['topic1', 'topic2', 'topic3'] + expectRowsList = [] + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdLog.info("create stb") + tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdLog.info("create ctb") + tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) + tdLog.info("insert data") + tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) + + tdLog.info("create topics from stb with filter") + queryString = "select ts,c1,c2 from %s.%s" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as stable %s.%s" %(topicNameList[0], paraDict["dbName"],paraDict["stbName"]) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectRowsList[0] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.exit("0 tmq consume rows error!") + + self.checkFileContent(consumerId, queryString) + + # reinit consume info, and start tmq_sim, then check consume result + tmqCom.initConsumerTable() + queryString = "select ts, c1, c2 from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as database %s" %(topicNameList[1], paraDict['dbName']) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + consumerId = 1 + topicList = topicNameList[1] + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + if expectRowsList[1] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) + tdLog.exit("1 tmq consume rows error!") + + self.checkFileContent(consumerId, queryString) + + # reinit consume info, and start tmq_sim, then check consume result + tmqCom.initConsumerTable() + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[2], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + consumerId = 2 + topicList = topicNameList[2] + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + if expectRowsList[2] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0])) + tdLog.exit("2 tmq consume rows error!") + + self.checkFileContent(consumerId, queryString) + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + tdSql.prepare() + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 788ae3474c..9254f57c40 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -159,7 +159,7 @@ class TMQCom: tdLog.debug("complete to create %s.%s" %(dbName, stbName)) return - def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1): + def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1,ctbStartIdx=0): tsql.execute("use %s" %dbName) pre_create = "create table" sql = pre_create @@ -168,8 +168,10 @@ class TMQCom: tagValue = 'beijing' if (i % 2 == 0): tagValue = 'shanghai' + elif (i % 3 == 0): + tagValue = 'changsha' - sql += " %s%d using %s tags(%d, '%s')"%(ctbPrefix,i,stbName,i+1, tagValue) + sql += " %s%d using %s tags(%d, '%s')"%(ctbPrefix,i+ctbStartIdx,stbName,i+ctbStartIdx+1, tagValue) if (i > 0) and (i%100 == 0): tsql.execute(sql) sql = pre_create @@ -235,7 +237,7 @@ class TMQCom: tdLog.debug("insert data ............ [OK]") return - def insert_data_2(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs): + def insert_data_2(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs,ctbStartIdx=0): tdLog.debug("start to insert data ............") tsql.execute("use %s" %dbName) pre_insert = "insert into " @@ -245,7 +247,7 @@ class TMQCom: startTs = int(round(t * 1000)) #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) for i in range(ctbNum): - sql += " %s%d values "%(ctbPrefix,i) + sql += " %s%d values "%(ctbPrefix,i+ctbStartIdx) for j in range(rowsPerTbl): if (j % 2 == 0): sql += "(%d, %d, %d, 'tmqrow_%d', now) "%(startTs + j, j, j, j) @@ -254,7 +256,7 @@ class TMQCom: if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): tsql.execute(sql) if j < rowsPerTbl - 1: - sql = "insert into %s%d values " %(ctbPrefix,i) + sql = "insert into %s%d values " %(ctbPrefix,i+ctbStartIdx) else: sql = "insert into " #end sql @@ -354,7 +356,10 @@ class TMQCom: def threadFunctionForInsert(self, **paraDict): # create new connector for new tdSql instance in my thread newTdSql = tdCom.newTdSql() - self.insert_data_2(newTdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) + if 'ctbStartIdx' in paraDict.keys(): + self.insert_data_2(newTdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"],paraDict["ctbStartIdx"]) + else: + self.insert_data_2(newTdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) return def asyncInsertData(self, paraDict): diff --git a/tests/system-test/fulltest.bat b/tests/system-test/fulltest.bat deleted file mode 100644 index 59ddd3fb7d..0000000000 --- a/tests/system-test/fulltest.bat +++ /dev/null @@ -1,102 +0,0 @@ - -python3 .\test.py -f 0-others\taosShell.py -python3 .\test.py -f 0-others\taosShellError.py -python3 .\test.py -f 0-others\taosShellNetChk.py -python3 .\test.py -f 0-others\telemetry.py -python3 .\test.py -f 0-others\taosdMonitor.py -python3 .\test.py -f 0-others\udfTest.py -python3 .\test.py -f 0-others\udf_create.py -python3 .\test.py -f 0-others\udf_restart_taosd.py -python3 .\test.py -f 0-others\cachelast.py - -python3 .\test.py -f 0-others\user_control.py -python3 .\test.py -f 0-others\fsync.py - -python3 .\test.py -f 1-insert\influxdb_line_taosc_insert.py -python3 .\test.py -f 1-insert\opentsdb_telnet_line_taosc_insert.py -python3 .\test.py -f 1-insert\opentsdb_json_taosc_insert.py -@REM #python3 .\test.py -f 1-insert\test_stmt_muti_insert_query.py -python3 .\test.py -f 1-insert\alter_stable.py -python3 .\test.py -f 1-insert\alter_table.py -python3 .\test.py -f 2-query\between.py -python3 .\test.py -f 2-query\distinct.py -python3 .\test.py -f 2-query\varchar.py -python3 .\test.py -f 2-query\ltrim.py -python3 .\test.py -f 2-query\rtrim.py -python3 .\test.py -f 2-query\length.py -python3 .\test.py -f 2-query\char_length.py -python3 .\test.py -f 2-query\upper.py -python3 .\test.py -f 2-query\lower.py -python3 .\test.py -f 2-query\join.py -python3 .\test.py -f 2-query\join2.py -python3 .\test.py -f 2-query\cast.py -python3 .\test.py -f 2-query\union.py -python3 .\test.py -f 2-query\union1.py -python3 .\test.py -f 2-query\concat.py -python3 .\test.py -f 2-query\concat2.py -python3 .\test.py -f 2-query\concat_ws.py -python3 .\test.py -f 2-query\concat_ws2.py -python3 .\test.py -f 2-query\check_tsdb.py -python3 .\test.py -f 2-query\spread.py -python3 .\test.py -f 2-query\hyperloglog.py - -python3 .\test.py -f 2-query\timezone.py -python3 .\test.py -f 2-query\Now.py -python3 .\test.py -f 2-query\Today.py -python3 .\test.py -f 2-query\max.py -python3 .\test.py -f 2-query\min.py -python3 .\test.py -f 2-query\count.py -python3 .\test.py -f 2-query\last.py -python3 .\test.py -f 2-query\first.py -python3 .\test.py -f 2-query\To_iso8601.py -python3 .\test.py -f 2-query\To_unixtimestamp.py -python3 .\test.py -f 2-query\timetruncate.py -python3 .\test.py -f 2-query\diff.py -python3 .\test.py -f 2-query\Timediff.py - -python3 .\test.py -f 2-query\top.py -python3 .\test.py -f 2-query\bottom.py -python3 .\test.py -f 2-query\percentile.py -python3 .\test.py -f 2-query\apercentile.py -python3 .\test.py -f 2-query\abs.py -python3 .\test.py -f 2-query\ceil.py -python3 .\test.py -f 2-query\floor.py -python3 .\test.py -f 2-query\round.py -python3 .\test.py -f 2-query\log.py -python3 .\test.py -f 2-query\pow.py -python3 .\test.py -f 2-query\sqrt.py -python3 .\test.py -f 2-query\sin.py -python3 .\test.py -f 2-query\cos.py -python3 .\test.py -f 2-query\tan.py -python3 .\test.py -f 2-query\arcsin.py -python3 .\test.py -f 2-query\arccos.py -python3 .\test.py -f 2-query\arctan.py -python3 .\test.py -f 2-query\query_cols_tags_and_or.py -@REM # python3 .\test.py -f 2-query\nestedQuery.py -@REM # TD-15983 subquery output duplicate name column. -@REM # Please Xiangyang Guo modify the following script -@REM # python3 .\test.py -f 2-query\nestedQuery_str.py - -python3 .\test.py -f 2-query\avg.py -python3 .\test.py -f 2-query\elapsed.py -python3 .\test.py -f 2-query\csum.py -python3 .\test.py -f 2-query\mavg.py -python3 .\test.py -f 2-query\diff.py -python3 .\test.py -f 2-query\sample.py -python3 .\test.py -f 2-query\function_diff.py -python3 .\test.py -f 2-query\unique.py -python3 .\test.py -f 2-query\stateduration.py -python3 .\test.py -f 2-query\function_stateduration.py -python3 .\test.py -f 2-query\statecount.py - -python3 .\test.py -f 7-tmq\basic5.py -python3 .\test.py -f 7-tmq\subscribeDb.py -python3 .\test.py -f 7-tmq\subscribeDb0.py -python3 .\test.py -f 7-tmq\subscribeDb1.py -python3 .\test.py -f 7-tmq\subscribeStb.py -python3 .\test.py -f 7-tmq\subscribeStb0.py -python3 .\test.py -f 7-tmq\subscribeStb1.py -python3 .\test.py -f 7-tmq\subscribeStb2.py -python3 .\test.py -f 7-tmq\subscribeStb3.py -python3 .\test.py -f 7-tmq\subscribeStb4.py -python3 .\test.py -f 7-tmq\db.py \ No newline at end of file diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 571bb166e1..2e2e390d12 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -141,6 +141,7 @@ python3 ./test.py -f 7-tmq/tmqError.py python3 ./test.py -f 7-tmq/schema.py python3 ./test.py -f 7-tmq/stbFilter.py python3 ./test.py -f 7-tmq/tmqCheckData.py +python3 ./test.py -f 7-tmq/tmqCheckData1.py python3 ./test.py -f 7-tmq/tmqUdf.py #python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5 python3 ./test.py -f 7-tmq/tmqConsumerGroup.py diff --git a/tests/system-test/test-all.bat b/tests/system-test/test-all.bat index adc9e0ce28..d4032cbc0d 100644 --- a/tests/system-test/test-all.bat +++ b/tests/system-test/test-all.bat @@ -6,19 +6,23 @@ if "%1" == "full" ( echo Windows Taosd Full Test set /a exitNum=0 del /Q /F failed.txt - set caseFile="fulltest.bat" + set caseFile="fulltest.sh" if not "%2" == "" ( set caseFile="%2" ) for /F "usebackq tokens=*" %%i in (!caseFile!) do ( - for /f "tokens=1* delims= " %%a in ("%%i") do if not "%%a" == "@REM" ( - set /a a+=1 - echo !a! Processing %%i - call :GetTimeSeconds !time! - set time1=!_timeTemp! - echo Start at !time! - call %%i ARG1 > result_!a!.txt 2>error_!a!.txt - if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. ) + call :CheckSkipCase %%i + if !skipCase! == false ( + set line=%%i + if "!line:~,7!" == "python3" ( + set /a a+=1 + echo !a! Processing %%i + call :GetTimeSeconds !time! + set time1=!_timeTemp! + echo Start at !time! + call %%i ARG1 > result_!a!.txt 2>error_!a!.txt + if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. ) + ) ) ) exit !exitNum! @@ -83,3 +87,8 @@ for %%a in (%tt%) do ( ) set /a _timeTemp=(%hh%*60+%mm%)*60+%ss% goto :eof + +:CheckSkipCase +set skipCase=false +if "%*" == "python3 ./test.py -f 1-insert/insertWithMoreVgroup.py" ( set skipCase=true ) +:goto eof \ No newline at end of file diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index cd6613b17a..2037f67089 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -339,8 +339,8 @@ int32_t shellParseArgs(int32_t argc, char *argv[]) { shell.info.clientVersion = "Welcome to the TDengine shell from %s, Client Version:%s\n" "Copyright (c) 2022 by TAOS Data, Inc. All rights reserved.\n\n"; - shell.info.promptHeader = "taos> "; - shell.info.promptContinue = " -> "; + shell.info.promptHeader = TAOS_CONSOLE_PROMPT_HEADER; + shell.info.promptContinue = TAOS_CONSOLE_PROMPT_CONTINUE; shell.info.promptSize = 6; snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), "version: %s", version); diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index f236c1eb88..9cb3541017 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -411,8 +411,9 @@ char taosGetConsoleChar() { static char mbStr[5]; static unsigned long bufLen = 0; static uint16_t bufIndex = 0, mbStrIndex = 0, mbStrLen = 0; - if (bufLen == 0) { + while (bufLen == 0) { ReadConsoleW(console, buf, SHELL_INPUT_MAX_COMMAND_SIZE, &bufLen, NULL); + if (bufLen > 0 && buf[0] == 0) bufLen = 0; bufIndex = 0; } if (mbStrLen == 0){ @@ -466,6 +467,8 @@ int32_t shellReadCommand(char *command) { } else if (c < '\033') { // Ctrl keys. TODO: Implement ctrl combinations switch (c) { + case 0: + break; case 1: // ctrl A shellPositionCursorHome(&cmd); break;