From 7066274cc741e7bf94f36d8d88bc0576d0c41e40 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 18 Apr 2022 21:50:56 +0800 Subject: [PATCH 01/83] add sync format code --- include/common/tmsgdef.h | 13 + include/libs/sync/sync.h | 108 ++--- include/libs/sync/syncTools.h | 376 ++++++++++++++++ source/libs/sync/CMakeLists.txt | 2 +- source/libs/sync/inc/syncEnv.h | 4 +- source/libs/sync/inc/syncInt.h | 64 ++- source/libs/sync/inc/syncMessage.h | 272 ------------ source/libs/sync/inc/syncRaftCfg.h | 65 +++ source/libs/sync/inc/syncRaftEntry.h | 19 +- source/libs/sync/inc/syncRaftStore.h | 2 +- source/libs/sync/inc/syncRespMgr.h | 55 +++ source/libs/sync/inc/syncSnapshot.h | 1 - source/libs/sync/inc/syncUtil.h | 2 + source/libs/sync/src/syncAppendEntries.c | 53 ++- source/libs/sync/src/syncAppendEntriesReply.c | 9 + source/libs/sync/src/syncCommit.c | 10 +- source/libs/sync/src/syncElection.c | 2 +- source/libs/sync/src/syncEnv.c | 3 + source/libs/sync/src/syncIO.c | 54 ++- source/libs/sync/src/syncMain.c | 419 ++++++++++++++---- source/libs/sync/src/syncMessage.c | 263 +++++++++-- source/libs/sync/src/syncRaftCfg.c | 211 +++++++++ source/libs/sync/src/syncRaftEntry.c | 31 +- source/libs/sync/src/syncRaftLog.c | 42 +- source/libs/sync/src/syncReplication.c | 4 +- source/libs/sync/src/syncRequestVote.c | 2 +- source/libs/sync/src/syncRequestVoteReply.c | 19 +- source/libs/sync/src/syncRespMgr.c | 93 ++++ source/libs/sync/src/syncUtil.c | 14 + source/libs/sync/test/CMakeLists.txt | 108 +++-- .../sync/test/syncAppendEntriesReplyTest.cpp | 29 +- .../libs/sync/test/syncAppendEntriesTest.cpp | 19 +- .../libs/sync/test/syncClientRequestTest.cpp | 18 +- source/libs/sync/test/syncElectTest.cpp | 26 +- source/libs/sync/test/syncElectTest2.cpp | 20 +- source/libs/sync/test/syncElectTest3.cpp | 20 +- source/libs/sync/test/syncEncodeTest.cpp | 18 +- source/libs/sync/test/syncEnqTest.cpp | 4 +- source/libs/sync/test/syncEntryTest.cpp | 6 +- source/libs/sync/test/syncIOClientTest.cpp | 2 +- source/libs/sync/test/syncIOSendMsgTest.cpp | 7 +- source/libs/sync/test/syncIndexMgrTest.cpp | 3 +- source/libs/sync/test/syncInitTest.cpp | 16 +- source/libs/sync/test/syncLogStoreCheck.cpp | 2 +- source/libs/sync/test/syncLogStoreTest.cpp | 123 ++--- source/libs/sync/test/syncPingReplyTest.cpp | 32 +- source/libs/sync/test/syncPingSelfTest.cpp | 2 +- source/libs/sync/test/syncPingTest.cpp | 32 +- source/libs/sync/test/syncPingTimerTest.cpp | 10 +- source/libs/sync/test/syncPingTimerTest2.cpp | 2 +- source/libs/sync/test/syncRaftCfgTest.cpp | 101 +++++ source/libs/sync/test/syncRaftStoreTest.cpp | 23 +- .../libs/sync/test/syncReplicateLoadTest.cpp | 30 +- source/libs/sync/test/syncReplicateTest.cpp | 31 +- source/libs/sync/test/syncReplicateTest2.cpp | 32 +- source/libs/sync/test/syncReplicateTest3.cpp | 32 +- .../sync/test/syncRequestVoteReplyTest.cpp | 22 +- source/libs/sync/test/syncRequestVoteTest.cpp | 21 +- source/libs/sync/test/syncRespMgrTest.cpp | 144 ++++++ source/libs/sync/test/syncRpcMsgTest.cpp | 35 +- source/libs/sync/test/syncTest.cpp | 62 ++- source/libs/sync/test/syncTimeoutTest.cpp | 17 +- source/libs/sync/test/syncUtilTest.cpp | 3 +- .../libs/sync/test/syncVotesGrantedTest.cpp | 5 +- .../libs/sync/test/syncVotesRespondTest.cpp | 5 +- source/libs/sync/test/syncWriteTest.cpp | 37 +- 66 files changed, 2372 insertions(+), 939 deletions(-) create mode 100644 include/libs/sync/syncTools.h create mode 100644 source/libs/sync/inc/syncRaftCfg.h create mode 100644 source/libs/sync/inc/syncRespMgr.h create mode 100644 source/libs/sync/src/syncRaftCfg.c create mode 100644 source/libs/sync/src/syncRespMgr.c create mode 100644 source/libs/sync/test/syncRaftCfgTest.cpp create mode 100644 source/libs/sync/test/syncRespMgrTest.cpp diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index e553dff270..24d259b0f2 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -206,6 +206,19 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SYNC_TIMEOUT, "vnode-sync-timeout", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING, "vnode-sync-ping", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING_REPLY, "vnode-sync-ping-reply", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CLIENT_REQUEST, "vnode-sync-client-request", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CLIENT_REQUEST_REPLY, "vnode-sync-client-request-reply", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SYNC_REQUEST_VOTE, "vnode-sync-request-vote", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SYNC_REQUEST_VOTE_REPLY, "vnode-sync-request-vote-reply", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPEND_ENTRIES, "vnode-sync-append-entries", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, "vnode-sync-append-entries-reply", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SYNC_NOOP, "vnode-sync-noop", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SYNC_UNKNOWN, "vnode-sync-unknown", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SYNC_COMMON_RESPONSE, "vnode-sync-common-response", NULL, NULL) + // Requests handled by QNODE TD_NEW_MSG_SEG(TDMT_QND_MSG) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index a2f88490f0..86666f4c68 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -22,6 +22,7 @@ extern "C" { #include #include +#include "cJSON.h" #include "taosdef.h" #include "trpc.h" #include "wal.h" @@ -38,14 +39,9 @@ typedef enum { TAOS_SYNC_STATE_ERROR = 103, } ESyncState; -typedef struct SSyncBuffer { - void* data; - size_t len; -} SSyncBuffer; - typedef struct SNodeInfo { - uint16_t nodePort; // node sync Port - char nodeFqdn[TSDB_FQDN_LEN]; // node FQDN + uint16_t nodePort; + char nodeFqdn[TSDB_FQDN_LEN]; } SNodeInfo; typedef struct SSyncCfg { @@ -54,40 +50,37 @@ typedef struct SSyncCfg { SNodeInfo nodeInfo[TSDB_MAX_REPLICA]; } SSyncCfg; -typedef struct SNodesRole { - int32_t replicaNum; - SNodeInfo nodeInfo[TSDB_MAX_REPLICA]; - ESyncState role[TSDB_MAX_REPLICA]; -} SNodesRole; +typedef struct SRaftId { + SyncNodeId addr; + SyncGroupId vgId; +} SRaftId; -// abstract definition of snapshot typedef struct SSnapshot { void* data; SyncIndex lastApplyIndex; } SSnapshot; +typedef enum { + TAOS_SYNC_FSM_CB_SUCCESS = 0, + TAOS_SYNC_FSM_CB_OTHER_ERROR, +} ESyncFsmCbCode; + +typedef struct SFsmCbMeta { + SyncIndex index; + bool isWeak; + int32_t code; + ESyncState state; + uint64_t seqNum; +} SFsmCbMeta; + typedef struct SSyncFSM { void* data; - // when value in pMsg finish a raft flow, FpCommitCb is called, code indicates the result - // user can do something according to the code and isWeak. for example, write data into tsdb - void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code, - ESyncState state); + void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta); + void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta); + void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta); - // when value in pMsg has been written into local log store, FpPreCommitCb is called, code indicates the result - // user can do something according to the code and isWeak. for example, write data into tsdb - void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code, - ESyncState state); - - // when log entry is updated by a new one, FpRollBackCb is called - // user can do something to roll back. for example, delete data from tsdb, or just ignore it - void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code, - ESyncState state); - - // user should implement this function, use "data" to take snapshot into "snapshot" int32_t (*FpTakeSnapshot)(SSnapshot* snapshot); - - // user should implement this function, restore "data" from "snapshot" int32_t (*FpRestoreSnapshot)(const SSnapshot* snapshot); } SSyncFSM; @@ -123,21 +116,6 @@ typedef struct SSyncLogStore { } SSyncLogStore; -// raft need to persist two variables in storage: currentTerm, voteFor -typedef struct SStateMgr { - void* data; - - int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm); - int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm); - - int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor); - int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor); - - int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg); - int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg); - -} SStateMgr; - typedef struct SSyncInfo { SyncGroupId vgId; SSyncCfg syncCfg; @@ -152,25 +130,33 @@ typedef struct SSyncInfo { } SSyncInfo; +int32_t syncInit(); +void syncCleanUp(); +int64_t syncOpen(const SSyncInfo* pSyncInfo); +void syncStart(int64_t rid); +void syncStop(int64_t rid); +int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg); +ESyncState syncGetMyRole(int64_t rid); +const char* syncGetMyRoleStr(int64_t rid); +SyncTerm syncGetMyTerm(int64_t rid); + +typedef enum { + TAOS_SYNC_PROPOSE_SUCCESS = 0, + TAOS_SYNC_PROPOSE_NOT_LEADER, + TAOS_SYNC_PROPOSE_OTHER_ERROR, +} ESyncProposeCode; + +int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak); + +extern int32_t sDebugFlag; + +//----------------------------------------- struct SSyncNode; typedef struct SSyncNode SSyncNode; -int32_t syncInit(); -void syncCleanUp(); - -int64_t syncStart(const SSyncInfo* pSyncInfo); -void syncStop(int64_t rid); -int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg); -int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak); -ESyncState syncGetMyRole(int64_t rid); - -// propose with sequence number, to implement linearizable semantics -int32_t syncPropose2(int64_t rid, const SRpcMsg* pMsg, bool isWeak, uint64_t seqNum); - -// for compatibility, the same as syncPropose -int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak); - -extern int32_t sDebugFlag; +struct SSyncBuffer; +typedef struct SSyncBuffer SSyncBuffer; +//----------------------------------------- #ifdef __cplusplus } diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h new file mode 100644 index 0000000000..6d28f54e29 --- /dev/null +++ b/include/libs/sync/syncTools.h @@ -0,0 +1,376 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_TOOLS_H +#define _TD_LIBS_SYNC_TOOLS_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include "cJSON.h" +#include "taosdef.h" +#include "trpc.h" +#include "wal.h" + +// ------------------ control ------------------- +struct SSyncNode; +typedef struct SSyncNode SSyncNode; + +SSyncNode* syncNodeAcquire(int64_t rid); +void syncNodeRelease(SSyncNode* pNode); + +int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg); +int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg); +void syncSetQ(int64_t rid, void* queueHandle); +void syncSetRpc(int64_t rid, void* rpcHandle); +char* sync2SimpleStr(int64_t rid); + +// set timer ms +void setPingTimerMS(int64_t rid, int32_t pingTimerMS); +void setElectTimerMS(int64_t rid, int32_t electTimerMS); +void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS); + +// for compatibility, the same as syncPropose +int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak); + +// utils +const char* syncUtilState2String(ESyncState state); + +// ------------------ for debug ------------------- +void syncRpcMsgPrint(SRpcMsg* pMsg); +void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg); +void syncRpcMsgLog(SRpcMsg* pMsg); +void syncRpcMsgLog2(char* s, SRpcMsg* pMsg); + +// ------------------ for compile ------------------- +typedef struct SSyncBuffer { + void* data; + size_t len; +} SSyncBuffer; + +typedef struct SNodesRole { + int32_t replicaNum; + SNodeInfo nodeInfo[TSDB_MAX_REPLICA]; + ESyncState role[TSDB_MAX_REPLICA]; +} SNodesRole; + +typedef struct SStateMgr { + void* data; + + int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm); + int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm); + + int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor); + int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor); + + int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg); + int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg); + +} SStateMgr; + +// ------------------ for message process ------------------- + +// --------------------------------------------- +typedef struct SyncPing { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + // private data + uint32_t dataLen; + char data[]; +} SyncPing; + +SyncPing* syncPingBuild(uint32_t dataLen); +SyncPing* syncPingBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str); +SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId); +void syncPingDestroy(SyncPing* pMsg); +void syncPingSerialize(const SyncPing* pMsg, char* buf, uint32_t bufLen); +void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg); +char* syncPingSerialize2(const SyncPing* pMsg, uint32_t* len); +SyncPing* syncPingDeserialize2(const char* buf, uint32_t len); +int32_t syncPingSerialize3(const SyncPing* pMsg, char* buf, int32_t bufLen); +SyncPing* syncPingDeserialize3(void* buf, int32_t bufLen); +void syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg); +void syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg); +SyncPing* syncPingFromRpcMsg2(const SRpcMsg* pRpcMsg); +cJSON* syncPing2Json(const SyncPing* pMsg); +char* syncPing2Str(const SyncPing* pMsg); + +// for debug ---------------------- +void syncPingPrint(const SyncPing* pMsg); +void syncPingPrint2(char* s, const SyncPing* pMsg); +void syncPingLog(const SyncPing* pMsg); +void syncPingLog2(char* s, const SyncPing* pMsg); + +// --------------------------------------------- +typedef struct SyncPingReply { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + // private data + uint32_t dataLen; + char data[]; +} SyncPingReply; + +SyncPingReply* syncPingReplyBuild(uint32_t dataLen); +SyncPingReply* syncPingReplyBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str); +SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId); +void syncPingReplyDestroy(SyncPingReply* pMsg); +void syncPingReplySerialize(const SyncPingReply* pMsg, char* buf, uint32_t bufLen); +void syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg); +char* syncPingReplySerialize2(const SyncPingReply* pMsg, uint32_t* len); +SyncPingReply* syncPingReplyDeserialize2(const char* buf, uint32_t len); +int32_t syncPingReplySerialize3(const SyncPingReply* pMsg, char* buf, int32_t bufLen); +SyncPingReply* syncPingReplyDeserialize3(void* buf, int32_t bufLen); +void syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg); +void syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg); +SyncPingReply* syncPingReplyFromRpcMsg2(const SRpcMsg* pRpcMsg); +cJSON* syncPingReply2Json(const SyncPingReply* pMsg); +char* syncPingReply2Str(const SyncPingReply* pMsg); + +// for debug ---------------------- +void syncPingReplyPrint(const SyncPingReply* pMsg); +void syncPingReplyPrint2(char* s, const SyncPingReply* pMsg); +void syncPingReplyLog(const SyncPingReply* pMsg); +void syncPingReplyLog2(char* s, const SyncPingReply* pMsg); + +// --------------------------------------------- +typedef enum ESyncTimeoutType { + SYNC_TIMEOUT_PING = 100, + SYNC_TIMEOUT_ELECTION, + SYNC_TIMEOUT_HEARTBEAT, +} ESyncTimeoutType; + +typedef struct SyncTimeout { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + ESyncTimeoutType timeoutType; + uint64_t logicClock; + int32_t timerMS; + void* data; // need optimized +} SyncTimeout; + +SyncTimeout* syncTimeoutBuild(); +SyncTimeout* syncTimeoutBuild2(ESyncTimeoutType timeoutType, uint64_t logicClock, int32_t timerMS, int32_t vgId, + void* data); +void syncTimeoutDestroy(SyncTimeout* pMsg); +void syncTimeoutSerialize(const SyncTimeout* pMsg, char* buf, uint32_t bufLen); +void syncTimeoutDeserialize(const char* buf, uint32_t len, SyncTimeout* pMsg); +char* syncTimeoutSerialize2(const SyncTimeout* pMsg, uint32_t* len); +SyncTimeout* syncTimeoutDeserialize2(const char* buf, uint32_t len); +void syncTimeout2RpcMsg(const SyncTimeout* pMsg, SRpcMsg* pRpcMsg); +void syncTimeoutFromRpcMsg(const SRpcMsg* pRpcMsg, SyncTimeout* pMsg); +SyncTimeout* syncTimeoutFromRpcMsg2(const SRpcMsg* pRpcMsg); +cJSON* syncTimeout2Json(const SyncTimeout* pMsg); +char* syncTimeout2Str(const SyncTimeout* pMsg); + +// for debug ---------------------- +void syncTimeoutPrint(const SyncTimeout* pMsg); +void syncTimeoutPrint2(char* s, const SyncTimeout* pMsg); +void syncTimeoutLog(const SyncTimeout* pMsg); +void syncTimeoutLog2(char* s, const SyncTimeout* pMsg); + +// --------------------------------------------- +typedef struct SyncClientRequest { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; // SyncClientRequest msgType + uint32_t originalRpcType; // user RpcMsg msgType + uint64_t seqNum; + bool isWeak; + uint32_t dataLen; // user RpcMsg.contLen + char data[]; // user RpcMsg.pCont +} SyncClientRequest; + +SyncClientRequest* syncClientRequestBuild(uint32_t dataLen); +SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak, + int32_t vgId); // step 1 +void syncClientRequestDestroy(SyncClientRequest* pMsg); +void syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen); +void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg); +char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len); +SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len); +void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg); // step 2 +void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg); +SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg); // step 3 +cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg); +char* syncClientRequest2Str(const SyncClientRequest* pMsg); + +// for debug ---------------------- +void syncClientRequestPrint(const SyncClientRequest* pMsg); +void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg); +void syncClientRequestLog(const SyncClientRequest* pMsg); +void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg); + +// --------------------------------------------- +typedef struct SyncClientRequestReply { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + int32_t errCode; + SRaftId leaderHint; +} SyncClientRequestReply; + +// --------------------------------------------- +typedef struct SyncRequestVote { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + // private data + SyncTerm term; + SyncIndex lastLogIndex; + SyncTerm lastLogTerm; +} SyncRequestVote; + +SyncRequestVote* syncRequestVoteBuild(int32_t vgId); +void syncRequestVoteDestroy(SyncRequestVote* pMsg); +void syncRequestVoteSerialize(const SyncRequestVote* pMsg, char* buf, uint32_t bufLen); +void syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* pMsg); +char* syncRequestVoteSerialize2(const SyncRequestVote* pMsg, uint32_t* len); +SyncRequestVote* syncRequestVoteDeserialize2(const char* buf, uint32_t len); +void syncRequestVote2RpcMsg(const SyncRequestVote* pMsg, SRpcMsg* pRpcMsg); +void syncRequestVoteFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVote* pMsg); +SyncRequestVote* syncRequestVoteFromRpcMsg2(const SRpcMsg* pRpcMsg); +cJSON* syncRequestVote2Json(const SyncRequestVote* pMsg); +char* syncRequestVote2Str(const SyncRequestVote* pMsg); + +// for debug ---------------------- +void syncRequestVotePrint(const SyncRequestVote* pMsg); +void syncRequestVotePrint2(char* s, const SyncRequestVote* pMsg); +void syncRequestVoteLog(const SyncRequestVote* pMsg); +void syncRequestVoteLog2(char* s, const SyncRequestVote* pMsg); + +// --------------------------------------------- +typedef struct SyncRequestVoteReply { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + // private data + SyncTerm term; + bool voteGranted; +} SyncRequestVoteReply; + +SyncRequestVoteReply* syncRequestVoteReplyBuild(int32_t vgId); +void syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg); +void syncRequestVoteReplySerialize(const SyncRequestVoteReply* pMsg, char* buf, uint32_t bufLen); +void syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestVoteReply* pMsg); +char* syncRequestVoteReplySerialize2(const SyncRequestVoteReply* pMsg, uint32_t* len); +SyncRequestVoteReply* syncRequestVoteReplyDeserialize2(const char* buf, uint32_t len); +void syncRequestVoteReply2RpcMsg(const SyncRequestVoteReply* pMsg, SRpcMsg* pRpcMsg); +void syncRequestVoteReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVoteReply* pMsg); +SyncRequestVoteReply* syncRequestVoteReplyFromRpcMsg2(const SRpcMsg* pRpcMsg); +cJSON* syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg); +char* syncRequestVoteReply2Str(const SyncRequestVoteReply* pMsg); + +// for debug ---------------------- +void syncRequestVoteReplyPrint(const SyncRequestVoteReply* pMsg); +void syncRequestVoteReplyPrint2(char* s, const SyncRequestVoteReply* pMsg); +void syncRequestVoteReplyLog(const SyncRequestVoteReply* pMsg); +void syncRequestVoteReplyLog2(char* s, const SyncRequestVoteReply* pMsg); + +// --------------------------------------------- +typedef struct SyncAppendEntries { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + // private data + SyncTerm term; + SyncIndex prevLogIndex; + SyncTerm prevLogTerm; + SyncIndex commitIndex; + uint32_t dataLen; + char data[]; +} SyncAppendEntries; + +SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen, int32_t vgId); +void syncAppendEntriesDestroy(SyncAppendEntries* pMsg); +void syncAppendEntriesSerialize(const SyncAppendEntries* pMsg, char* buf, uint32_t bufLen); +void syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntries* pMsg); +char* syncAppendEntriesSerialize2(const SyncAppendEntries* pMsg, uint32_t* len); +SyncAppendEntries* syncAppendEntriesDeserialize2(const char* buf, uint32_t len); +void syncAppendEntries2RpcMsg(const SyncAppendEntries* pMsg, SRpcMsg* pRpcMsg); +void syncAppendEntriesFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntries* pMsg); +SyncAppendEntries* syncAppendEntriesFromRpcMsg2(const SRpcMsg* pRpcMsg); +cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg); +char* syncAppendEntries2Str(const SyncAppendEntries* pMsg); + +// for debug ---------------------- +void syncAppendEntriesPrint(const SyncAppendEntries* pMsg); +void syncAppendEntriesPrint2(char* s, const SyncAppendEntries* pMsg); +void syncAppendEntriesLog(const SyncAppendEntries* pMsg); +void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg); + +// --------------------------------------------- +typedef struct SyncAppendEntriesReply { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + // private data + SyncTerm term; + bool success; + SyncIndex matchIndex; +} SyncAppendEntriesReply; + +SyncAppendEntriesReply* syncAppendEntriesReplyBuild(int32_t vgId); +void syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg); +void syncAppendEntriesReplySerialize(const SyncAppendEntriesReply* pMsg, char* buf, uint32_t bufLen); +void syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppendEntriesReply* pMsg); +char* syncAppendEntriesReplySerialize2(const SyncAppendEntriesReply* pMsg, uint32_t* len); +SyncAppendEntriesReply* syncAppendEntriesReplyDeserialize2(const char* buf, uint32_t len); +void syncAppendEntriesReply2RpcMsg(const SyncAppendEntriesReply* pMsg, SRpcMsg* pRpcMsg); +void syncAppendEntriesReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesReply* pMsg); +SyncAppendEntriesReply* syncAppendEntriesReplyFromRpcMsg2(const SRpcMsg* pRpcMsg); +cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg); +char* syncAppendEntriesReply2Str(const SyncAppendEntriesReply* pMsg); + +// for debug ---------------------- +void syncAppendEntriesReplyPrint(const SyncAppendEntriesReply* pMsg); +void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg); +void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg); +void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg); + +// on message ---------------------- +int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); +int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); +int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg); +int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg); +int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg); +int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg); +int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg); +int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg); + +//--------------------- + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_TOOLS_H*/ diff --git a/source/libs/sync/CMakeLists.txt b/source/libs/sync/CMakeLists.txt index 551849c6f2..cb196acc02 100644 --- a/source/libs/sync/CMakeLists.txt +++ b/source/libs/sync/CMakeLists.txt @@ -11,7 +11,7 @@ target_link_libraries( target_include_directories( sync - PUBLIC "${TD_SOURCE_DIR}/include/libs/sync" + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index 7940d7f436..e9550ff989 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -31,10 +31,10 @@ extern "C" { #define TIMER_MAX_MS 0x7FFFFFFF #define ENV_TICK_TIMER_MS 1000 #define PING_TIMER_MS 1000 -#define ELECT_TIMER_MS_MIN 150 +#define ELECT_TIMER_MS_MIN 500 #define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2) #define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN) -#define HEARTBEAT_TIMER_MS 30 +#define HEARTBEAT_TIMER_MS 100 #define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0}) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index cfa87048a7..cb539e8379 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -25,6 +25,7 @@ extern "C" { #include #include "cJSON.h" #include "sync.h" +#include "syncTools.h" #include "taosdef.h" #include "tglobal.h" #include "tlog.h" @@ -67,6 +68,43 @@ extern "C" { } \ } +#define sFatalLong(...) \ + { \ + if (sDebugFlag & DEBUG_FATAL) { \ + taosPrintLongString("SYN FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \ + } \ + } +#define sErrorLong(...) \ + { \ + if (sDebugFlag & DEBUG_ERROR) { \ + taosPrintLongString("SYN ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \ + } \ + } +#define sWarnLong(...) \ + { \ + if (sDebugFlag & DEBUG_WARN) { \ + taosPrintLongString("SYN WARN ", DEBUG_WARN, 255, __VA_ARGS__); \ + } \ + } +#define sInfoLong(...) \ + { \ + if (sDebugFlag & DEBUG_INFO) { \ + taosPrintLongString("SYN INFO ", DEBUG_INFO, 255, __VA_ARGS__); \ + } \ + } +#define sDebugLong(...) \ + { \ + if (sDebugFlag & DEBUG_DEBUG) { \ + taosPrintLongString("SYN DEBUG ", DEBUG_DEBUG, sDebugFlag, __VA_ARGS__); \ + } \ + } +#define sTraceLong(...) \ + { \ + if (sDebugFlag & DEBUG_TRACE) { \ + taosPrintLongString("SYN TRACE ", DEBUG_TRACE, sDebugFlag, __VA_ARGS__); \ + } \ + } + struct SyncTimeout; typedef struct SyncTimeout SyncTimeout; @@ -106,17 +144,19 @@ typedef struct SVotesRespond SVotesRespond; struct SSyncIndexMgr; typedef struct SSyncIndexMgr SSyncIndexMgr; -typedef struct SRaftId { - SyncNodeId addr; // typedef uint64_t SyncNodeId; - SyncGroupId vgId; // typedef int32_t SyncGroupId; -} SRaftId; +struct SRaftCfg; +typedef struct SRaftCfg SRaftCfg; + +struct SSyncRespMgr; +typedef struct SSyncRespMgr SSyncRespMgr; typedef struct SSyncNode { // init by SSyncInfo SyncGroupId vgId; - SSyncCfg syncCfg; + SRaftCfg* pRaftCfg; char path[TSDB_FILENAME_LEN]; char raftStorePath[TSDB_FILENAME_LEN * 2]; + char configPath[TSDB_FILENAME_LEN * 2]; // sync io SWal* pWal; @@ -199,10 +239,14 @@ typedef struct SSyncNode { int32_t (*FpOnAppendEntriesReply)(SSyncNode* ths, SyncAppendEntriesReply* pMsg); int32_t (*FpOnTimeout)(SSyncNode* pSyncNode, SyncTimeout* pMsg); + // tools + SSyncRespMgr* pSyncRespMgr; + } SSyncNode; // open/close -------------- SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); +void syncNodeStart(SSyncNode* pSyncNode); void syncNodeClose(SSyncNode* pSyncNode); // ping -------------- @@ -222,10 +266,12 @@ int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode); int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode); // utils -------------- -int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); -int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); -cJSON* syncNode2Json(const SSyncNode* pSyncNode); -char* syncNode2Str(const SSyncNode* pSyncNode); +int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); +int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); +cJSON* syncNode2Json(const SSyncNode* pSyncNode); +char* syncNode2Str(const SSyncNode* pSyncNode); +char* syncNode2SimpleStr(const SSyncNode* pSyncNode); + SSyncNode* syncNodeAcquire(int64_t rid); void syncNodeRelease(SSyncNode* pNode); diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index ffb2ae3a26..19079d518d 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -27,22 +27,6 @@ extern "C" { #include "syncInt.h" #include "taosdef.h" -// encode as uint32 -typedef enum ESyncMessageType { - SYNC_UNKNOWN = 9999, - SYNC_TIMEOUT = 99, - SYNC_PING = 101, - SYNC_PING_REPLY = 103, - SYNC_CLIENT_REQUEST = 105, - SYNC_CLIENT_REQUEST_REPLY = 107, - SYNC_REQUEST_VOTE = 109, - SYNC_REQUEST_VOTE_REPLY = 111, - SYNC_APPEND_ENTRIES = 113, - SYNC_APPEND_ENTRIES_REPLY = 115, - SYNC_RESPONSE = 119, - -} ESyncMessageType; - // --------------------------------------------- cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg); cJSON* syncRpcUnknownMsg2Json(); @@ -53,263 +37,7 @@ void syncRpcMsgPrint(SRpcMsg* pMsg); void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg); void syncRpcMsgLog(SRpcMsg* pMsg); void syncRpcMsgLog2(char* s, SRpcMsg* pMsg); - // --------------------------------------------- -typedef enum ESyncTimeoutType { - SYNC_TIMEOUT_PING = 100, - SYNC_TIMEOUT_ELECTION, - SYNC_TIMEOUT_HEARTBEAT, -} ESyncTimeoutType; - -typedef struct SyncTimeout { - uint32_t bytes; - uint32_t msgType; - ESyncTimeoutType timeoutType; - uint64_t logicClock; - int32_t timerMS; - void* data; // need optimized -} SyncTimeout; - -SyncTimeout* syncTimeoutBuild(); -SyncTimeout* syncTimeoutBuild2(ESyncTimeoutType timeoutType, uint64_t logicClock, int32_t timerMS, void* data); -void syncTimeoutDestroy(SyncTimeout* pMsg); -void syncTimeoutSerialize(const SyncTimeout* pMsg, char* buf, uint32_t bufLen); -void syncTimeoutDeserialize(const char* buf, uint32_t len, SyncTimeout* pMsg); -char* syncTimeoutSerialize2(const SyncTimeout* pMsg, uint32_t* len); // -SyncTimeout* syncTimeoutDeserialize2(const char* buf, uint32_t len); // -void syncTimeout2RpcMsg(const SyncTimeout* pMsg, SRpcMsg* pRpcMsg); -void syncTimeoutFromRpcMsg(const SRpcMsg* pRpcMsg, SyncTimeout* pMsg); -SyncTimeout* syncTimeoutFromRpcMsg2(const SRpcMsg* pRpcMsg); // -cJSON* syncTimeout2Json(const SyncTimeout* pMsg); -char* syncTimeout2Str(const SyncTimeout* pMsg); // - -// for debug ---------------------- -void syncTimeoutPrint(const SyncTimeout* pMsg); -void syncTimeoutPrint2(char* s, const SyncTimeout* pMsg); -void syncTimeoutLog(const SyncTimeout* pMsg); -void syncTimeoutLog2(char* s, const SyncTimeout* pMsg); - -// --------------------------------------------- -typedef struct SyncPing { - uint32_t bytes; - uint32_t msgType; - SRaftId srcId; - SRaftId destId; - // private data - uint32_t dataLen; - char data[]; -} SyncPing; - -SyncPing* syncPingBuild(uint32_t dataLen); -SyncPing* syncPingBuild2(const SRaftId* srcId, const SRaftId* destId, const char* str); -SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId); -void syncPingDestroy(SyncPing* pMsg); -void syncPingSerialize(const SyncPing* pMsg, char* buf, uint32_t bufLen); -void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg); -char* syncPingSerialize2(const SyncPing* pMsg, uint32_t* len); -SyncPing* syncPingDeserialize2(const char* buf, uint32_t len); -void syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg); -void syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg); -SyncPing* syncPingFromRpcMsg2(const SRpcMsg* pRpcMsg); -cJSON* syncPing2Json(const SyncPing* pMsg); -char* syncPing2Str(const SyncPing* pMsg); - -// for debug ---------------------- -void syncPingPrint(const SyncPing* pMsg); -void syncPingPrint2(char* s, const SyncPing* pMsg); -void syncPingLog(const SyncPing* pMsg); -void syncPingLog2(char* s, const SyncPing* pMsg); - -// --------------------------------------------- -typedef struct SyncPingReply { - uint32_t bytes; - uint32_t msgType; - SRaftId srcId; - SRaftId destId; - // private data - uint32_t dataLen; - char data[]; -} SyncPingReply; - -SyncPingReply* syncPingReplyBuild(uint32_t dataLen); -SyncPingReply* syncPingReplyBuild2(const SRaftId* srcId, const SRaftId* destId, const char* str); -SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId); -void syncPingReplyDestroy(SyncPingReply* pMsg); -void syncPingReplySerialize(const SyncPingReply* pMsg, char* buf, uint32_t bufLen); -void syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg); -char* syncPingReplySerialize2(const SyncPingReply* pMsg, uint32_t* len); // -SyncPingReply* syncPingReplyDeserialize2(const char* buf, uint32_t len); // -void syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg); -void syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg); -SyncPingReply* syncPingReplyFromRpcMsg2(const SRpcMsg* pRpcMsg); // -cJSON* syncPingReply2Json(const SyncPingReply* pMsg); -char* syncPingReply2Str(const SyncPingReply* pMsg); // - -// for debug ---------------------- -void syncPingReplyPrint(const SyncPingReply* pMsg); -void syncPingReplyPrint2(char* s, const SyncPingReply* pMsg); -void syncPingReplyLog(const SyncPingReply* pMsg); -void syncPingReplyLog2(char* s, const SyncPingReply* pMsg); - -// --------------------------------------------- -typedef struct SyncClientRequest { - uint32_t bytes; - uint32_t msgType; - uint32_t originalRpcType; - uint64_t seqNum; - bool isWeak; - uint32_t dataLen; - char data[]; -} SyncClientRequest; - -SyncClientRequest* syncClientRequestBuild(uint32_t dataLen); -SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak); // step 1 -void syncClientRequestDestroy(SyncClientRequest* pMsg); -void syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen); -void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg); -char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len); -SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len); -void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg); // step 2 -void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg); -SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg); // step 3 -cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg); -char* syncClientRequest2Str(const SyncClientRequest* pMsg); - -// for debug ---------------------- -void syncClientRequestPrint(const SyncClientRequest* pMsg); -void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg); -void syncClientRequestLog(const SyncClientRequest* pMsg); -void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg); - -// --------------------------------------------- -typedef struct SyncClientRequestReply { - uint32_t bytes; - uint32_t msgType; - int32_t errCode; - SRaftId leaderHint; -} SyncClientRequestReply; - -// --------------------------------------------- -typedef struct SyncRequestVote { - uint32_t bytes; - uint32_t msgType; - SRaftId srcId; - SRaftId destId; - // private data - SyncTerm term; - SyncIndex lastLogIndex; - SyncTerm lastLogTerm; -} SyncRequestVote; - -SyncRequestVote* syncRequestVoteBuild(); -void syncRequestVoteDestroy(SyncRequestVote* pMsg); -void syncRequestVoteSerialize(const SyncRequestVote* pMsg, char* buf, uint32_t bufLen); -void syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* pMsg); -char* syncRequestVoteSerialize2(const SyncRequestVote* pMsg, uint32_t* len); -SyncRequestVote* syncRequestVoteDeserialize2(const char* buf, uint32_t len); -void syncRequestVote2RpcMsg(const SyncRequestVote* pMsg, SRpcMsg* pRpcMsg); -void syncRequestVoteFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVote* pMsg); -SyncRequestVote* syncRequestVoteFromRpcMsg2(const SRpcMsg* pRpcMsg); -cJSON* syncRequestVote2Json(const SyncRequestVote* pMsg); -char* syncRequestVote2Str(const SyncRequestVote* pMsg); - -// for debug ---------------------- -void syncRequestVotePrint(const SyncRequestVote* pMsg); -void syncRequestVotePrint2(char* s, const SyncRequestVote* pMsg); -void syncRequestVoteLog(const SyncRequestVote* pMsg); -void syncRequestVoteLog2(char* s, const SyncRequestVote* pMsg); - -// --------------------------------------------- -typedef struct SyncRequestVoteReply { - uint32_t bytes; - uint32_t msgType; - SRaftId srcId; - SRaftId destId; - // private data - SyncTerm term; - bool voteGranted; -} SyncRequestVoteReply; - -SyncRequestVoteReply* syncRequestVoteReplyBuild(); -void syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg); -void syncRequestVoteReplySerialize(const SyncRequestVoteReply* pMsg, char* buf, uint32_t bufLen); -void syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestVoteReply* pMsg); -char* syncRequestVoteReplySerialize2(const SyncRequestVoteReply* pMsg, uint32_t* len); -SyncRequestVoteReply* syncRequestVoteReplyDeserialize2(const char* buf, uint32_t len); -void syncRequestVoteReply2RpcMsg(const SyncRequestVoteReply* pMsg, SRpcMsg* pRpcMsg); -void syncRequestVoteReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVoteReply* pMsg); -SyncRequestVoteReply* syncRequestVoteReplyFromRpcMsg2(const SRpcMsg* pRpcMsg); -cJSON* syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg); -char* syncRequestVoteReply2Str(const SyncRequestVoteReply* pMsg); - -// for debug ---------------------- -void syncRequestVoteReplyPrint(const SyncRequestVoteReply* pMsg); -void syncRequestVoteReplyPrint2(char* s, const SyncRequestVoteReply* pMsg); -void syncRequestVoteReplyLog(const SyncRequestVoteReply* pMsg); -void syncRequestVoteReplyLog2(char* s, const SyncRequestVoteReply* pMsg); - -// --------------------------------------------- -typedef struct SyncAppendEntries { - uint32_t bytes; - uint32_t msgType; - SRaftId srcId; - SRaftId destId; - // private data - SyncTerm term; - SyncIndex prevLogIndex; - SyncTerm prevLogTerm; - SyncIndex commitIndex; - uint32_t dataLen; - char data[]; -} SyncAppendEntries; - -SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen); -void syncAppendEntriesDestroy(SyncAppendEntries* pMsg); -void syncAppendEntriesSerialize(const SyncAppendEntries* pMsg, char* buf, uint32_t bufLen); -void syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntries* pMsg); -char* syncAppendEntriesSerialize2(const SyncAppendEntries* pMsg, uint32_t* len); -SyncAppendEntries* syncAppendEntriesDeserialize2(const char* buf, uint32_t len); -void syncAppendEntries2RpcMsg(const SyncAppendEntries* pMsg, SRpcMsg* pRpcMsg); -void syncAppendEntriesFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntries* pMsg); -SyncAppendEntries* syncAppendEntriesFromRpcMsg2(const SRpcMsg* pRpcMsg); -cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg); -char* syncAppendEntries2Str(const SyncAppendEntries* pMsg); - -// for debug ---------------------- -void syncAppendEntriesPrint(const SyncAppendEntries* pMsg); -void syncAppendEntriesPrint2(char* s, const SyncAppendEntries* pMsg); -void syncAppendEntriesLog(const SyncAppendEntries* pMsg); -void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg); - -// --------------------------------------------- -typedef struct SyncAppendEntriesReply { - uint32_t bytes; - uint32_t msgType; - SRaftId srcId; - SRaftId destId; - // private data - SyncTerm term; - bool success; - SyncIndex matchIndex; -} SyncAppendEntriesReply; - -SyncAppendEntriesReply* syncAppendEntriesReplyBuild(); -void syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg); -void syncAppendEntriesReplySerialize(const SyncAppendEntriesReply* pMsg, char* buf, uint32_t bufLen); -void syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppendEntriesReply* pMsg); -char* syncAppendEntriesReplySerialize2(const SyncAppendEntriesReply* pMsg, uint32_t* len); -SyncAppendEntriesReply* syncAppendEntriesReplyDeserialize2(const char* buf, uint32_t len); -void syncAppendEntriesReply2RpcMsg(const SyncAppendEntriesReply* pMsg, SRpcMsg* pRpcMsg); -void syncAppendEntriesReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesReply* pMsg); -SyncAppendEntriesReply* syncAppendEntriesReplyFromRpcMsg2(const SRpcMsg* pRpcMsg); -cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg); -char* syncAppendEntriesReply2Str(const SyncAppendEntriesReply* pMsg); - -// for debug ---------------------- -void syncAppendEntriesReplyPrint(const SyncAppendEntriesReply* pMsg); -void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg); -void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg); -void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncRaftCfg.h b/source/libs/sync/inc/syncRaftCfg.h new file mode 100644 index 0000000000..bfc64cb7b6 --- /dev/null +++ b/source/libs/sync/inc/syncRaftCfg.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_RAFT_CFG_H +#define _TD_LIBS_SYNC_RAFT_CFG_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "cJSON.h" +#include "syncInt.h" +#include "taosdef.h" + +typedef struct SRaftCfg { + SSyncCfg cfg; + TdFilePtr pFile; + char path[TSDB_FILENAME_LEN * 2]; +} SRaftCfg; + +SRaftCfg *raftCfgOpen(const char *path); +int32_t raftCfgClose(SRaftCfg *pRaftCfg); +int32_t raftCfgPersist(SRaftCfg *pRaftCfg); + +cJSON * syncCfg2Json(SSyncCfg *pSyncCfg); +char * syncCfg2Str(SSyncCfg *pSyncCfg); +int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg); +int32_t syncCfgFromStr(const char *s, SSyncCfg *pSyncCfg); + +cJSON *raftCfg2Json(SRaftCfg *pRaftCfg); +char * raftCfg2Str(SRaftCfg *pRaftCfg); + +int32_t syncCfgCreateFile(SSyncCfg *pCfg, const char *path); + +// for debug ---------------------- +void syncCfgPrint(SSyncCfg *pCfg); +void syncCfgPrint2(char *s, SSyncCfg *pCfg); +void syncCfgLog(SSyncCfg *pCfg); +void syncCfgLog2(char *s, SSyncCfg *pCfg); + +void raftCfgPrint(SRaftCfg *pCfg); +void raftCfgPrint2(char *s, SRaftCfg *pCfg); +void raftCfgLog(SRaftCfg *pCfg); +void raftCfgLog2(char *s, SRaftCfg *pCfg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_RAFT_CFG_H*/ diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index d4e63e9676..b130186516 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -27,29 +27,22 @@ extern "C" { #include "syncMessage.h" #include "taosdef.h" -typedef enum EntryType { - SYNC_RAFT_ENTRY_NOOP = 0, - SYNC_RAFT_ENTRY_DATA = 1, - SYNC_RAFT_ENTRY_CONFIG = 2, -} EntryType; - typedef struct SSyncRaftEntry { uint32_t bytes; - uint32_t msgType; - uint32_t originalRpcType; + uint32_t msgType; // SyncClientRequest msgType + uint32_t originalRpcType; // user RpcMsg msgType uint64_t seqNum; bool isWeak; SyncTerm term; SyncIndex index; - EntryType entryType; - uint32_t dataLen; - char data[]; + uint32_t dataLen; // user RpcMsg.contLen + char data[]; // user RpcMsg.pCont } SSyncRaftEntry; SSyncRaftEntry* syncEntryBuild(uint32_t dataLen); SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); // step 4 -SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index, EntryType entryType); -SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index); +SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); +SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId); void syncEntryDestory(SSyncRaftEntry* pEntry); char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len); // step 5 SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len); // step 6 diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index 355a08ac84..e0cbcf0744 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -28,7 +28,7 @@ extern "C" { #include "taosdef.h" #define RAFT_STORE_BLOCK_SIZE 512 -#define RAFT_STORE_PATH_LEN 128 +#define RAFT_STORE_PATH_LEN (TSDB_FILENAME_LEN * 2) typedef struct SRaftStore { SyncTerm currentTerm; diff --git a/source/libs/sync/inc/syncRespMgr.h b/source/libs/sync/inc/syncRespMgr.h new file mode 100644 index 0000000000..e37c0bb625 --- /dev/null +++ b/source/libs/sync/inc/syncRespMgr.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_RESP_MGR_H +#define _TD_LIBS_SYNC_RESP_MGR_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "syncInt.h" +#include "taosdef.h" + +typedef struct SRespStub { + SRpcMsg rpcMsg; + int64_t createTime; +} SRespStub; + +typedef struct SSyncRespMgr { + SHashObj * pRespHash; + int64_t ttl; + void * data; + TdThreadMutex mutex; + uint64_t seqNum; +} SSyncRespMgr; + +SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl); +void syncRespMgrDestroy(SSyncRespMgr *pObj); +int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub); +int32_t syncRespMgrDel(SSyncRespMgr *pObj, uint64_t index); +int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub); +int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub); +void syncRespClean(SSyncRespMgr *pObj); +void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_RESP_MGR_H*/ diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 611f33a0f2..fd2119ce65 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -27,7 +27,6 @@ extern "C" { #include "taosdef.h" int32_t takeSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot); - int32_t restoreSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot); #ifdef __cplusplus diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index a4bb11afc6..83f31c5dac 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -55,6 +55,8 @@ char* syncUtilprintBin(char* ptr, uint32_t len); char* syncUtilprintBin2(char* ptr, uint32_t len); SyncIndex syncUtilMinIndex(SyncIndex a, SyncIndex b); SyncIndex syncUtilMaxIndex(SyncIndex a, SyncIndex b); +void syncUtilMsgHtoN(void* msg); +void syncUtilMsgNtoH(void* msg); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 2e9cd63de6..6623ed1caa 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -125,7 +125,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { "logOK:%d", pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK); - SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); pReply->srcId = ths->myRaftId; pReply->destId = pMsg->srcId; pReply->term = ths->pRaftStore->currentTerm; @@ -199,13 +199,20 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SSyncRaftEntry* pRollBackEntry = logStoreGetEntry(ths->pLogStore, index); assert(pRollBackEntry != NULL); - // maybe is a NOOP ENTRY - // assert(pRollBackEntry->entryType == SYNC_RAFT_ENTRY_DATA); + if (pRollBackEntry->msgType != TDMT_VND_SYNC_NOOP) { + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); + + SFsmCbMeta cbMeta; + cbMeta.index = pRollBackEntry->index; + cbMeta.isWeak = pRollBackEntry->isWeak; + cbMeta.code = 0; + cbMeta.state = ths->state; + cbMeta.seqNum = pRollBackEntry->seqNum; + ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, cbMeta); + rpcFreeCont(rpcMsg.pCont); + } - SRpcMsg rpcMsg; - syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); - ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, pRollBackEntry->index, pRollBackEntry->isWeak, 0, ths->state); - rpcFreeCont(rpcMsg.pCont); syncEntryDestory(pRollBackEntry); } } @@ -220,8 +227,14 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); if (ths->pFsm != NULL) { - if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->entryType == SYNC_RAFT_ENTRY_DATA) { - ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 2, ths->state); + if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + SFsmCbMeta cbMeta; + cbMeta.index = pAppendEntry->index; + cbMeta.isWeak = pAppendEntry->isWeak; + cbMeta.code = 2; + cbMeta.state = ths->state; + cbMeta.seqNum = pAppendEntry->seqNum; + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta); } } rpcFreeCont(rpcMsg.pCont); @@ -245,8 +258,14 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); if (ths->pFsm != NULL) { - if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->entryType == SYNC_RAFT_ENTRY_DATA) { - ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 3, ths->state); + if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + SFsmCbMeta cbMeta; + cbMeta.index = pAppendEntry->index; + cbMeta.isWeak = pAppendEntry->isWeak; + cbMeta.code = 3; + cbMeta.state = ths->state; + cbMeta.seqNum = pAppendEntry->seqNum; + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta); } } rpcFreeCont(rpcMsg.pCont); @@ -261,7 +280,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { assert(0); } - SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); pReply->srcId = ths->myRaftId; pReply->destId = pMsg->srcId; pReply->term = ths->pRaftStore->currentTerm; @@ -301,8 +320,14 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pEntry, &rpcMsg); - if (ths->pFsm->FpCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) { - ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, ths->state); + if (ths->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + SFsmCbMeta cbMeta; + cbMeta.index = pEntry->index; + cbMeta.isWeak = pEntry->isWeak; + cbMeta.code = 0; + cbMeta.state = ths->state; + cbMeta.seqNum = pEntry->seqNum; + ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, cbMeta); } rpcFreeCont(rpcMsg.pCont); diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 790ac7f8e1..77d85e2915 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -56,6 +56,15 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p // syncNodeUpdateTerm(ths, pMsg->term); // } + if (pMsg->term > ths->pRaftStore->currentTerm) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "syncNodeOnAppendEntriesReplyCb error term, receive:%lu current:%lu", pMsg->term, + ths->pRaftStore->currentTerm); + syncNodeLog2(logBuf, ths); + sError("%s", logBuf); + return ret; + } + assert(pMsg->term == ths->pRaftStore->currentTerm); if (pMsg->success) { diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 6023c00afa..8b54b0a090 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -97,8 +97,14 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pEntry, &rpcMsg); - if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) { - pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, pSyncNode->state); + if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + SFsmCbMeta cbMeta; + cbMeta.index = pEntry->index; + cbMeta.isWeak = pEntry->isWeak; + cbMeta.code = 0; + cbMeta.state = pSyncNode->state; + cbMeta.seqNum = pEntry->seqNum; + pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, cbMeta); } rpcFreeCont(rpcMsg.pCont); diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 6ae70689ef..5101344b84 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -35,7 +35,7 @@ int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) { int32_t ret = 0; for (int i = 0; i < pSyncNode->peersNum; ++i) { - SyncRequestVote* pMsg = syncRequestVoteBuild(); + SyncRequestVote* pMsg = syncRequestVoteBuild(pSyncNode->vgId); pMsg->srcId = pSyncNode->myRaftId; pMsg->destId = pSyncNode->peersId[i]; pMsg->term = pSyncNode->pRaftStore->currentTerm; diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index 726ebdae07..4213d7b45d 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -27,10 +27,13 @@ static void syncEnvTick(void *param, void *tmrId); // -------------------------------- int32_t syncEnvStart() { + sDebugFlag = 207; + int32_t ret = 0; taosSeedRand(taosGetTimestampSec()); gSyncEnv = doSyncEnvStart(gSyncEnv); assert(gSyncEnv != NULL); + sTrace("syncEnvStart ok!"); return ret; } diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 717b1cc756..b1d7af3254 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -29,7 +29,7 @@ static int32_t syncIODestroy(SSyncIO *io); static int32_t syncIOStartInternal(SSyncIO *io); static int32_t syncIOStopInternal(SSyncIO *io); -static void *syncIOConsumerFunc(void *param); +static void * syncIOConsumerFunc(void *param); static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); @@ -70,9 +70,15 @@ int32_t syncIOSendMsg(void *clientRpc, const SEpSet *pEpSet, SRpcMsg *pMsg) { assert(pEpSet->numOfEps == 1); int32_t ret = 0; - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), "==syncIOSendMsg== %s:%d", pEpSet->eps[0].fqdn, pEpSet->eps[0].port); - syncRpcMsgPrint2(logBuf, pMsg); + { + syncUtilMsgNtoH(pMsg->pCont); + + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==syncIOSendMsg== %s:%d", pEpSet->eps[0].fqdn, pEpSet->eps[0].port); + syncRpcMsgLog2(logBuf, pMsg); + + syncUtilMsgHtoN(pMsg->pCont); + } pMsg->handle = NULL; pMsg->noResp = 1; @@ -83,7 +89,7 @@ int32_t syncIOSendMsg(void *clientRpc, const SEpSet *pEpSet, SRpcMsg *pMsg) { int32_t syncIOEqMsg(void *queue, SRpcMsg *pMsg) { int32_t ret = 0; char logBuf[128]; - syncRpcMsgPrint2((char *)"==syncIOEqMsg==", pMsg); + syncRpcMsgLog2((char *)"==syncIOEqMsg==", pMsg); SRpcMsg *pTemp; pTemp = taosAllocateQitem(sizeof(SRpcMsg)); @@ -235,9 +241,9 @@ static int32_t syncIOStopInternal(SSyncIO *io) { } static void *syncIOConsumerFunc(void *param) { - SSyncIO *io = param; + SSyncIO * io = param; STaosQall *qall; - SRpcMsg *pRpcMsg, rpcMsg; + SRpcMsg * pRpcMsg, rpcMsg; qall = taosAllocateQall(); while (1) { @@ -252,7 +258,7 @@ static void *syncIOConsumerFunc(void *param) { syncRpcMsgLog2((char *)"==syncIOConsumerFunc==", pRpcMsg); // use switch case instead of if else - if (pRpcMsg->msgType == SYNC_PING) { + if (pRpcMsg->msgType == TDMT_VND_SYNC_PING) { if (io->FpOnSyncPing != NULL) { SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); @@ -260,7 +266,7 @@ static void *syncIOConsumerFunc(void *param) { syncPingDestroy(pSyncMsg); } - } else if (pRpcMsg->msgType == SYNC_PING_REPLY) { + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING_REPLY) { if (io->FpOnSyncPingReply != NULL) { SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); @@ -268,7 +274,7 @@ static void *syncIOConsumerFunc(void *param) { syncPingReplyDestroy(pSyncMsg); } - } else if (pRpcMsg->msgType == SYNC_CLIENT_REQUEST) { + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) { if (io->FpOnSyncClientRequest != NULL) { SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); @@ -276,7 +282,7 @@ static void *syncIOConsumerFunc(void *param) { syncClientRequestDestroy(pSyncMsg); } - } else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE) { + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) { if (io->FpOnSyncRequestVote != NULL) { SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); @@ -284,7 +290,7 @@ static void *syncIOConsumerFunc(void *param) { syncRequestVoteDestroy(pSyncMsg); } - } else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE_REPLY) { + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) { if (io->FpOnSyncRequestVoteReply != NULL) { SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); @@ -292,7 +298,7 @@ static void *syncIOConsumerFunc(void *param) { syncRequestVoteReplyDestroy(pSyncMsg); } - } else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES) { + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) { if (io->FpOnSyncAppendEntries != NULL) { SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); @@ -300,7 +306,7 @@ static void *syncIOConsumerFunc(void *param) { syncAppendEntriesDestroy(pSyncMsg); } - } else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES_REPLY) { + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) { if (io->FpOnSyncAppendEntriesReply != NULL) { SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); @@ -308,7 +314,7 @@ static void *syncIOConsumerFunc(void *param) { syncAppendEntriesReplyDestroy(pSyncMsg); } - } else if (pRpcMsg->msgType == SYNC_TIMEOUT) { + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_TIMEOUT) { if (io->FpOnSyncTimeout != NULL) { SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); @@ -336,7 +342,7 @@ static void *syncIOConsumerFunc(void *param) { rpcMsg.handle = pRpcMsg->handle; rpcMsg.code = 0; - syncRpcMsgPrint2((char *)"syncIOConsumerFunc rpcSendResponse --> ", &rpcMsg); + syncRpcMsgLog2((char *)"syncIOConsumerFunc rpcSendResponse --> ", &rpcMsg); rpcSendResponse(&rpcMsg); } */ @@ -350,7 +356,9 @@ static void *syncIOConsumerFunc(void *param) { } static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { - syncRpcMsgPrint2((char *)"==syncIOProcessRequest==", pMsg); + syncUtilMsgNtoH(pMsg->pCont); + + syncRpcMsgLog2((char *)"==syncIOProcessRequest==", pMsg); SSyncIO *io = pParent; SRpcMsg *pTemp; pTemp = taosAllocateQitem(sizeof(SRpcMsg)); @@ -359,10 +367,10 @@ static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { } static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { - if (pMsg->msgType == SYNC_RESPONSE) { + if (pMsg->msgType == TDMT_VND_SYNC_COMMON_RESPONSE) { sTrace("==syncIOProcessReply=="); } else { - syncRpcMsgPrint2((char *)"==syncIOProcessReply==", pMsg); + syncRpcMsgLog2((char *)"==syncIOProcessReply==", pMsg); } rpcFreeCont(pMsg->pCont); } @@ -408,14 +416,14 @@ static void syncIOTickQ(void *param, void *tmrId) { srcId.vgId = -1; destId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port); destId.vgId = -1; - SyncPingReply *pMsg = syncPingReplyBuild2(&srcId, &destId, "syncIOTickQ"); + SyncPingReply *pMsg = syncPingReplyBuild2(&srcId, &destId, -1, "syncIOTickQ"); SRpcMsg rpcMsg; syncPingReply2RpcMsg(pMsg, &rpcMsg); SRpcMsg *pTemp; pTemp = taosAllocateQitem(sizeof(SRpcMsg)); memcpy(pTemp, &rpcMsg, sizeof(SRpcMsg)); - syncRpcMsgPrint2((char *)"==syncIOTickQ==", &rpcMsg); + syncRpcMsgLog2((char *)"==syncIOTickQ==", &rpcMsg); taosWriteQitem(io->pMsgQ, pTemp); syncPingReplyDestroy(pMsg); @@ -430,12 +438,12 @@ static void syncIOTickPing(void *param, void *tmrId) { srcId.vgId = -1; destId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port); destId.vgId = -1; - SyncPing *pMsg = syncPingBuild2(&srcId, &destId, "syncIOTickPing"); + SyncPing *pMsg = syncPingBuild2(&srcId, &destId, -1, "syncIOTickPing"); // SyncPing *pMsg = syncPingBuild3(&srcId, &destId); SRpcMsg rpcMsg; syncPing2RpcMsg(pMsg, &rpcMsg); - syncRpcMsgPrint2((char *)"==syncIOTickPing==", &rpcMsg); + syncRpcMsgLog2((char *)"==syncIOTickPing==", &rpcMsg); rpcSendRequest(io->clientRpc, &io->myAddr, &rpcMsg, NULL); syncPingDestroy(pMsg); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 78e454309a..6d51cee14e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -23,11 +23,13 @@ #include "syncIndexMgr.h" #include "syncInt.h" #include "syncMessage.h" +#include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncReplication.h" #include "syncRequestVote.h" #include "syncRequestVoteReply.h" +#include "syncRespMgr.h" #include "syncTimeout.h" #include "syncUtil.h" #include "syncVoteMgr.h" @@ -44,9 +46,9 @@ static int32_t syncNodeEqNoop(SSyncNode* ths); static int32_t syncNodeAppendNoop(SSyncNode* ths); // process message ---- -static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); -static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); -static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg); +int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); +int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); +int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg); // life cycle static void syncFreeNode(void* param); @@ -76,10 +78,12 @@ void syncCleanUp() { } } -int64_t syncStart(const SSyncInfo* pSyncInfo) { +int64_t syncOpen(const SSyncInfo* pSyncInfo) { SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo); assert(pSyncNode != NULL); + syncNodeLog2("syncNodeOpen open success", pSyncNode); + pSyncNode->rid = taosAddRef(tsNodeRefId, pSyncNode); if (pSyncNode->rid < 0) { syncFreeNode(pSyncNode); @@ -89,6 +93,16 @@ int64_t syncStart(const SSyncInfo* pSyncInfo) { return pSyncNode->rid; } +void syncStart(int64_t rid) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return; + } + syncNodeStart(pSyncNode); + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); +} + void syncStop(int64_t rid) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { @@ -105,11 +119,6 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { return ret; } -int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { - int32_t ret = syncPropose2(rid, pMsg, isWeak, 0); - return ret; -} - int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { int32_t ret = syncPropose(rid, pMsg, isWeak); return ret; @@ -121,28 +130,163 @@ ESyncState syncGetMyRole(int64_t rid) { return TAOS_SYNC_STATE_ERROR; } assert(rid == pSyncNode->rid); - return pSyncNode->state; + ESyncState state = pSyncNode->state; + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return state; } -int32_t syncPropose2(int64_t rid, const SRpcMsg* pMsg, bool isWeak, uint64_t seqNum) { - int32_t ret = 0; +const char* syncGetMyRoleStr(int64_t rid) { + const char* s = syncUtilState2String(syncGetMyRole(rid)); + return s; +} + +SyncTerm syncGetMyTerm(int64_t rid) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { - return -1; + return TAOS_SYNC_STATE_ERROR; + } + assert(rid == pSyncNode->rid); + SyncTerm term = pSyncNode->pRaftStore->currentTerm; + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return term; +} + +int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return TAOS_SYNC_STATE_ERROR; + } + assert(rid == pSyncNode->rid); + + SRespStub stub; + int32_t ret = syncRespMgrGet(pSyncNode->pSyncRespMgr, index, &stub); + if (ret == 1) { + memcpy(msg, &(stub.rpcMsg), sizeof(SRpcMsg)); + } + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return ret; +} + +int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return TAOS_SYNC_STATE_ERROR; + } + assert(rid == pSyncNode->rid); + + SRespStub stub; + int32_t ret = syncRespMgrGetAndDel(pSyncNode->pSyncRespMgr, index, &stub); + if (ret == 1) { + memcpy(msg, &(stub.rpcMsg), sizeof(SRpcMsg)); + } + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return ret; +} + +void syncSetQ(int64_t rid, void* queue) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + sTrace("syncSetQ get pSyncNode is NULL, rid:%ld", rid); + return; + } + assert(rid == pSyncNode->rid); + pSyncNode->queue = queue; + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); +} + +void syncSetRpc(int64_t rid, void* rpcHandle) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + sTrace("syncSetRpc get pSyncNode is NULL, rid:%ld", rid); + return; + } + assert(rid == pSyncNode->rid); + pSyncNode->rpcClient = rpcHandle; + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); +} + +char* sync2SimpleStr(int64_t rid) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + sTrace("syncSetRpc get pSyncNode is NULL, rid:%ld", rid); + return NULL; + } + assert(rid == pSyncNode->rid); + char* s = syncNode2SimpleStr(pSyncNode); + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + + return s; +} + +void setPingTimerMS(int64_t rid, int32_t pingTimerMS) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return; + } + assert(rid == pSyncNode->rid); + pSyncNode->pingBaseLine = pingTimerMS; + pSyncNode->pingTimerMS = pingTimerMS; + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); +} + +void setElectTimerMS(int64_t rid, int32_t electTimerMS) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return; + } + assert(rid == pSyncNode->rid); + pSyncNode->electBaseLine = electTimerMS; + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); +} + +void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return; + } + assert(rid == pSyncNode->rid); + pSyncNode->hbBaseLine = hbTimerMS; + pSyncNode->heartbeatTimerMS = hbTimerMS; + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); +} + +int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { + int32_t ret = TAOS_SYNC_PROPOSE_SUCCESS; + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return TAOS_SYNC_PROPOSE_OTHER_ERROR; } assert(rid == pSyncNode->rid); if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { - SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, seqNum, isWeak); + SRespStub stub; + stub.createTime = taosGetTimestampMs(); + stub.rpcMsg = *pMsg; + uint64_t seqNum = syncRespMgrAdd(pSyncNode->pSyncRespMgr, &stub); + + SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, seqNum, isWeak, pSyncNode->vgId); SRpcMsg rpcMsg; syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); - pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); + if (pSyncNode->FpEqMsg != NULL) { + pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); + } else { + sTrace("syncPropose pSyncNode->FpEqMsg is NULL"); + } syncClientRequestDestroy(pSyncMsg); - ret = 0; + ret = TAOS_SYNC_PROPOSE_SUCCESS; } else { sTrace("syncPropose not leader, %s", syncUtilState2String(pSyncNode->state)); - ret = -1; // todo : need define err code !! + ret = TAOS_SYNC_PROPOSE_NOT_LEADER; } taosReleaseRef(tsNodeRefId, pSyncNode->rid); @@ -155,49 +299,62 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { assert(pSyncNode != NULL); memset(pSyncNode, 0, sizeof(SSyncNode)); - if (taosMkDir(pSyncInfo->path) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - sError("failed to create dir:%s since %s", pSyncInfo->path, terrstr()); - return NULL; + int32_t ret = 0; + if (!taosDirExist((char*)(pSyncInfo->path))) { + if (taosMkDir(pSyncInfo->path) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + sError("failed to create dir:%s since %s", pSyncInfo->path, terrstr()); + return NULL; + } + + // create raft config file + snprintf(pSyncNode->configPath, sizeof(pSyncNode->configPath), "%s/raft_config.json", pSyncInfo->path); + ret = syncCfgCreateFile((SSyncCfg*)&(pSyncInfo->syncCfg), pSyncNode->configPath); + assert(ret == 0); } // init by SSyncInfo pSyncNode->vgId = pSyncInfo->vgId; - pSyncNode->syncCfg = pSyncInfo->syncCfg; memcpy(pSyncNode->path, pSyncInfo->path, sizeof(pSyncNode->path)); snprintf(pSyncNode->raftStorePath, sizeof(pSyncNode->raftStorePath), "%s/raft_store.json", pSyncInfo->path); + snprintf(pSyncNode->configPath, sizeof(pSyncNode->configPath), "%s/raft_config.json", pSyncInfo->path); + pSyncNode->pWal = pSyncInfo->pWal; pSyncNode->rpcClient = pSyncInfo->rpcClient; pSyncNode->FpSendMsg = pSyncInfo->FpSendMsg; pSyncNode->queue = pSyncInfo->queue; pSyncNode->FpEqMsg = pSyncInfo->FpEqMsg; + // init raft config + pSyncNode->pRaftCfg = raftCfgOpen(pSyncNode->configPath); + assert(pSyncNode->pRaftCfg != NULL); + // init internal - pSyncNode->myNodeInfo = pSyncInfo->syncCfg.nodeInfo[pSyncInfo->syncCfg.myIndex]; - syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncInfo->vgId, &pSyncNode->myRaftId); + pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex]; + syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId); // init peersNum, peers, peersId - pSyncNode->peersNum = pSyncInfo->syncCfg.replicaNum - 1; + pSyncNode->peersNum = pSyncNode->pRaftCfg->cfg.replicaNum - 1; int j = 0; - for (int i = 0; i < pSyncInfo->syncCfg.replicaNum; ++i) { - if (i != pSyncInfo->syncCfg.myIndex) { - pSyncNode->peersNodeInfo[j] = pSyncInfo->syncCfg.nodeInfo[i]; + for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { + if (i != pSyncNode->pRaftCfg->cfg.myIndex) { + pSyncNode->peersNodeInfo[j] = pSyncNode->pRaftCfg->cfg.nodeInfo[i]; j++; } } for (int i = 0; i < pSyncNode->peersNum; ++i) { - syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncInfo->vgId, &pSyncNode->peersId[i]); + syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i]); } // init replicaNum, replicasId - pSyncNode->replicaNum = pSyncInfo->syncCfg.replicaNum; - for (int i = 0; i < pSyncInfo->syncCfg.replicaNum; ++i) { - syncUtilnodeInfo2raftId(&pSyncInfo->syncCfg.nodeInfo[i], pSyncInfo->vgId, &pSyncNode->replicasId[i]); + pSyncNode->replicaNum = pSyncNode->pRaftCfg->cfg.replicaNum; + for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { + syncUtilnodeInfo2raftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]); } // init raft algorithm pSyncNode->pFsm = pSyncInfo->pFsm; - pSyncNode->quorum = syncUtilQuorum(pSyncInfo->syncCfg.replicaNum); + pSyncNode->quorum = syncUtilQuorum(pSyncNode->pRaftCfg->cfg.replicaNum); pSyncNode->leaderCache = EMPTY_RAFT_ID; // init life cycle outside @@ -287,10 +444,25 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { pSyncNode->FpOnAppendEntriesReply = syncNodeOnAppendEntriesReplyCb; pSyncNode->FpOnTimeout = syncNodeOnTimeoutCb; + // tools + pSyncNode->pSyncRespMgr = syncRespMgrCreate(NULL, 0); + assert(pSyncNode->pSyncRespMgr != NULL); + + // start in syncNodeStart + // start raft + // syncNodeBecomeFollower(pSyncNode); + + return pSyncNode; +} + +void syncNodeStart(SSyncNode* pSyncNode) { // start raft syncNodeBecomeFollower(pSyncNode); - return pSyncNode; + // for test + int32_t ret = 0; + // ret = syncNodeStartPingTimer(pSyncNode); + assert(ret == 0); } void syncNodeClose(SSyncNode* pSyncNode) { @@ -305,12 +477,18 @@ void syncNodeClose(SSyncNode* pSyncNode) { syncIndexMgrDestroy(pSyncNode->pNextIndex); syncIndexMgrDestroy(pSyncNode->pMatchIndex); logStoreDestory(pSyncNode->pLogStore); + raftCfgClose(pSyncNode->pRaftCfg); syncNodeStopPingTimer(pSyncNode); syncNodeStopElectTimer(pSyncNode); syncNodeStopHeartbeatTimer(pSyncNode); - taosMemoryFree(pSyncNode); + if (pSyncNode->pFsm != NULL) { + taosMemoryFree(pSyncNode->pFsm); + } + + // free memory in syncFreeNode + // taosMemoryFree(pSyncNode); } // ping -------------- @@ -328,7 +506,7 @@ int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* int32_t syncNodePingSelf(SSyncNode* pSyncNode) { int32_t ret = 0; - SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId); + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId, pSyncNode->vgId); ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg); assert(ret == 0); @@ -339,10 +517,9 @@ int32_t syncNodePingSelf(SSyncNode* pSyncNode) { int32_t syncNodePingPeers(SSyncNode* pSyncNode) { int32_t ret = 0; for (int i = 0; i < pSyncNode->peersNum; ++i) { - SRaftId destId; - syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &destId); - SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId); - ret = syncNodePing(pSyncNode, &destId, pMsg); + SRaftId* destId = &(pSyncNode->peersId[i]); + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, destId, pSyncNode->vgId); + ret = syncNodePing(pSyncNode, destId, pMsg); assert(ret == 0); syncPingDestroy(pMsg); } @@ -351,11 +528,10 @@ int32_t syncNodePingPeers(SSyncNode* pSyncNode) { int32_t syncNodePingAll(SSyncNode* pSyncNode) { int32_t ret = 0; - for (int i = 0; i < pSyncNode->syncCfg.replicaNum; ++i) { - SRaftId destId; - syncUtilnodeInfo2raftId(&pSyncNode->syncCfg.nodeInfo[i], pSyncNode->vgId, &destId); - SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId); - ret = syncNodePing(pSyncNode, &destId, pMsg); + for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { + SRaftId* destId = &(pSyncNode->replicasId[i]); + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, destId, pSyncNode->vgId); + ret = syncNodePing(pSyncNode, destId, pMsg); assert(ret == 0); syncPingDestroy(pMsg); } @@ -430,14 +606,30 @@ int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode) { int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { SEpSet epSet; syncUtilraftId2EpSet(destRaftId, &epSet); - pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg); + if (pSyncNode->FpSendMsg != NULL) { + pMsg->noResp = 1; + // htonl + syncUtilMsgHtoN(pMsg->pCont); + + pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg); + } else { + sTrace("syncNodeSendMsgById pSyncNode->FpSendMsg is NULL"); + } return 0; } int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg) { SEpSet epSet; syncUtilnodeInfo2EpSet(nodeInfo, &epSet); - pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg); + if (pSyncNode->FpSendMsg != NULL) { + pMsg->noResp = 1; + // htonl + syncUtilMsgHtoN(pMsg->pCont); + + pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg); + } else { + sTrace("syncNodeSendMsgByInfo pSyncNode->FpSendMsg is NULL"); + } return 0; } @@ -448,7 +640,11 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { if (pSyncNode != NULL) { // init by SSyncInfo cJSON_AddNumberToObject(pRoot, "vgId", pSyncNode->vgId); + cJSON_AddItemToObject(pRoot, "SRaftCfg", raftCfg2Json(pSyncNode->pRaftCfg)); cJSON_AddStringToObject(pRoot, "path", pSyncNode->path); + cJSON_AddStringToObject(pRoot, "raftStorePath", pSyncNode->raftStorePath); + cJSON_AddStringToObject(pRoot, "configPath", pSyncNode->configPath); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pWal); cJSON_AddStringToObject(pRoot, "pWal", u64buf); @@ -501,9 +697,7 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { // tla+ server vars cJSON_AddNumberToObject(pRoot, "state", pSyncNode->state); cJSON_AddStringToObject(pRoot, "state_str", syncUtilState2String(pSyncNode->state)); - char tmpBuf[RAFT_STORE_BLOCK_SIZE]; - raftStoreSerialize(pSyncNode->pRaftStore, tmpBuf, sizeof(tmpBuf)); - cJSON_AddStringToObject(pRoot, "pRaftStore", tmpBuf); + cJSON_AddItemToObject(pRoot, "pRaftStore", raftStore2Json(pSyncNode->pRaftStore)); // tla+ candidate vars cJSON_AddItemToObject(pRoot, "pVotesGranted", voteGranted2Json(pSyncNode->pVotesGranted)); @@ -518,6 +712,11 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { snprintf(u64buf, sizeof(u64buf), "%" PRId64 "", pSyncNode->commitIndex); cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); + // timer ms init + cJSON_AddNumberToObject(pRoot, "pingBaseLine", pSyncNode->pingBaseLine); + cJSON_AddNumberToObject(pRoot, "electBaseLine", pSyncNode->electBaseLine); + cJSON_AddNumberToObject(pRoot, "hbBaseLine", pSyncNode->hbBaseLine); + // ping timer snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pPingTimer); cJSON_AddStringToObject(pRoot, "pPingTimer", u64buf); @@ -586,6 +785,19 @@ char* syncNode2Str(const SSyncNode* pSyncNode) { return serialized; } +char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { + int len = 256; + char* s = (char*)taosMemoryMalloc(len); + snprintf(s, len, + "syncNode2SimpleStr vgId:%d currentTerm:%lu, commitIndex:%ld, state:%d %s, electTimerLogicClock:%lu, " + "electTimerLogicClockUser:%lu, " + "electTimerMS:%d", + pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, pSyncNode->state, + syncUtilState2String(pSyncNode->state), pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, + pSyncNode->electTimerMS); + return s; +} + SSyncNode* syncNodeAcquire(int64_t rid) { SSyncNode* pNode = taosAcquireRef(tsNodeRefId, rid); if (pNode == NULL) { @@ -672,24 +884,36 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { assert(voteGrantedMajority(pSyncNode->pVotesGranted)); syncNodeBecomeLeader(pSyncNode); + syncNodeLog2("==state change syncNodeCandidate2Leader==", pSyncNode); + // Raft 3.6.2 Committing entries from previous terms + + // use this now syncNodeAppendNoop(pSyncNode); + + // do not use this // syncNodeEqNoop(pSyncNode); } void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { assert(pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER); pSyncNode->state = TAOS_SYNC_STATE_CANDIDATE; + + syncNodeLog2("==state change syncNodeFollower2Candidate==", pSyncNode); } void syncNodeLeader2Follower(SSyncNode* pSyncNode) { assert(pSyncNode->state == TAOS_SYNC_STATE_LEADER); syncNodeBecomeFollower(pSyncNode); + + syncNodeLog2("==state change syncNodeLeader2Follower==", pSyncNode); } void syncNodeCandidate2Follower(SSyncNode* pSyncNode) { assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); syncNodeBecomeFollower(pSyncNode); + + syncNodeLog2("==state change syncNodeCandidate2Follower==", pSyncNode); } // raft vote -------------- @@ -707,7 +931,7 @@ void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId) void syncNodeVoteForSelf(SSyncNode* pSyncNode) { syncNodeVoteForTerm(pSyncNode, pSyncNode->pRaftStore->currentTerm, &(pSyncNode->myRaftId)); - SyncRequestVoteReply* pMsg = syncRequestVoteReplyBuild(); + SyncRequestVoteReply* pMsg = syncRequestVoteReplyBuild(pSyncNode->vgId); pMsg->srcId = pSyncNode->myRaftId; pMsg->destId = pSyncNode->myRaftId; pMsg->term = pSyncNode->pRaftStore->currentTerm; @@ -735,13 +959,13 @@ void syncNodePrint2(char* s, SSyncNode* pObj) { void syncNodeLog(SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); - sTrace("syncNodeLog | len:%lu | %s", strlen(serialized), serialized); + sTraceLong("syncNodeLog | len:%lu | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void syncNodeLog2(char* s, SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); - sTrace("syncNodeLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTraceLong("syncNodeLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } @@ -751,11 +975,15 @@ static void syncNodeEqPingTimer(void* param, void* tmrId) { SSyncNode* pSyncNode = (SSyncNode*)param; if (atomic_load_64(&pSyncNode->pingTimerLogicClockUser) <= atomic_load_64(&pSyncNode->pingTimerLogicClock)) { SyncTimeout* pSyncMsg = syncTimeoutBuild2(SYNC_TIMEOUT_PING, atomic_load_64(&pSyncNode->pingTimerLogicClock), - pSyncNode->pingTimerMS, pSyncNode); + pSyncNode->pingTimerMS, pSyncNode->vgId, pSyncNode); SRpcMsg rpcMsg; syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); syncRpcMsgLog2((char*)"==syncNodeEqPingTimer==", &rpcMsg); - pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); + if (pSyncNode->FpEqMsg != NULL) { + pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); + } else { + sTrace("syncNodeEqPingTimer pSyncNode->FpEqMsg is NULL"); + } syncTimeoutDestroy(pSyncMsg); taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, @@ -770,17 +998,21 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { SSyncNode* pSyncNode = (SSyncNode*)param; if (atomic_load_64(&pSyncNode->electTimerLogicClockUser) <= atomic_load_64(&pSyncNode->electTimerLogicClock)) { SyncTimeout* pSyncMsg = syncTimeoutBuild2(SYNC_TIMEOUT_ELECTION, atomic_load_64(&pSyncNode->electTimerLogicClock), - pSyncNode->electTimerMS, pSyncNode); + pSyncNode->electTimerMS, pSyncNode->vgId, pSyncNode); SRpcMsg rpcMsg; syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); syncRpcMsgLog2((char*)"==syncNodeEqElectTimer==", &rpcMsg); - pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); + if (pSyncNode->FpEqMsg != NULL) { + pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); + } else { + sTrace("syncNodeEqElectTimer pSyncNode->FpEqMsg is NULL"); + } syncTimeoutDestroy(pSyncMsg); // reset timer ms pSyncNode->electTimerMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine); - taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, - &pSyncNode->pPingTimer); + taosTmrReset(syncNodeEqElectTimer, pSyncNode->electTimerMS, pSyncNode, gSyncEnv->pTimerManager, + &pSyncNode->pElectTimer); } else { sTrace("==syncNodeEqElectTimer== electTimerLogicClock:%" PRIu64 ", electTimerLogicClockUser:%" PRIu64 "", pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser); @@ -793,11 +1025,15 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { atomic_load_64(&pSyncNode->heartbeatTimerLogicClock)) { SyncTimeout* pSyncMsg = syncTimeoutBuild2(SYNC_TIMEOUT_HEARTBEAT, atomic_load_64(&pSyncNode->heartbeatTimerLogicClock), - pSyncNode->heartbeatTimerMS, pSyncNode); + pSyncNode->heartbeatTimerMS, pSyncNode->vgId, pSyncNode); SRpcMsg rpcMsg; syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); syncRpcMsgLog2((char*)"==syncNodeEqHeartbeatTimer==", &rpcMsg); - pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); + if (pSyncNode->FpEqMsg != NULL) { + pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); + } else { + sTrace("syncNodeEqHeartbeatTimer pSyncNode->FpEqMsg is NULL"); + } syncTimeoutDestroy(pSyncMsg); taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, @@ -815,7 +1051,7 @@ static int32_t syncNodeEqNoop(SSyncNode* ths) { SyncIndex index = ths->pLogStore->getLastIndex(ths->pLogStore) + 1; SyncTerm term = ths->pRaftStore->currentTerm; - SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index); + SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, ths->vgId); assert(pEntry != NULL); uint32_t entryLen; @@ -826,7 +1062,11 @@ static int32_t syncNodeEqNoop(SSyncNode* ths) { SRpcMsg rpcMsg; syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); - ths->FpEqMsg(ths->queue, &rpcMsg); + if (ths->FpEqMsg != NULL) { + ths->FpEqMsg(ths->queue, &rpcMsg); + } else { + sTrace("syncNodeEqNoop pSyncNode->FpEqMsg is NULL"); + } taosMemoryFree(serialized); syncClientRequestDestroy(pSyncMsg); @@ -839,7 +1079,7 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) { SyncIndex index = ths->pLogStore->getLastIndex(ths->pLogStore) + 1; SyncTerm term = ths->pRaftStore->currentTerm; - SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index); + SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, ths->vgId); assert(pEntry != NULL); if (ths->state == TAOS_SYNC_STATE_LEADER) { @@ -851,18 +1091,34 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) { } // on message ---- -static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) { +int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) { + // log state + char logBuf[1024]; + snprintf(logBuf, sizeof(logBuf), + "==syncNodeOnPingCb== vgId:%d, state: %d, %s, term:%lu electTimerLogicClock:%lu, " + "electTimerLogicClockUser:%lu, electTimerMS:%d", + ths->vgId, ths->state, syncUtilState2String(ths->state), ths->pRaftStore->currentTerm, + ths->electTimerLogicClock, ths->electTimerLogicClockUser, ths->electTimerMS); + int32_t ret = 0; - syncPingLog2("==syncNodeOnPingCb==", pMsg); - SyncPingReply* pMsgReply = syncPingReplyBuild3(&ths->myRaftId, &pMsg->srcId); + syncPingLog2(logBuf, pMsg); + SyncPingReply* pMsgReply = syncPingReplyBuild3(&ths->myRaftId, &pMsg->srcId, ths->vgId); SRpcMsg rpcMsg; syncPingReply2RpcMsg(pMsgReply, &rpcMsg); + + /* + // htonl + SMsgHead* pHead = rpcMsg.pCont; + pHead->contLen = htonl(pHead->contLen); + pHead->vgId = htonl(pHead->vgId); + */ + syncNodeSendMsgById(&pMsgReply->destId, ths, &rpcMsg); return ret; } -static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { +int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { int32_t ret = 0; syncPingReplyLog2("==syncNodeOnPingReplyCb==", pMsg); return ret; @@ -878,7 +1134,7 @@ static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { // /\ UNCHANGED <> // -static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) { +int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) { int32_t ret = 0; syncClientRequestLog2("==syncNodeOnClientRequestCb==", pMsg); @@ -898,8 +1154,14 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg syncEntry2OriginalRpc(pEntry, &rpcMsg); if (ths->pFsm != NULL) { - if (ths->pFsm->FpPreCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) { - ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, ths->state); + if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + SFsmCbMeta cbMeta; + cbMeta.index = pEntry->index; + cbMeta.isWeak = pEntry->isWeak; + cbMeta.code = 0; + cbMeta.state = ths->state; + cbMeta.seqNum = pEntry->seqNum; + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta); } } rpcFreeCont(rpcMsg.pCont); @@ -913,8 +1175,14 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg syncEntry2OriginalRpc(pEntry, &rpcMsg); if (ths->pFsm != NULL) { - if (ths->pFsm->FpPreCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) { - ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 1, ths->state); + if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + SFsmCbMeta cbMeta; + cbMeta.index = pEntry->index; + cbMeta.isWeak = pEntry->isWeak; + cbMeta.code = 1; + cbMeta.state = ths->state; + cbMeta.seqNum = pEntry->seqNum; + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta); } } rpcFreeCont(rpcMsg.pCont); @@ -926,7 +1194,8 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg static void syncFreeNode(void* param) { SSyncNode* pNode = param; - syncNodePrint2((char*)"==syncFreeNode==", pNode); + // inner object already free + // syncNodePrint2((char*)"==syncFreeNode==", pNode); taosMemoryFree(pNode); -} \ No newline at end of file +} diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 5dca165455..76c68569c3 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -22,50 +22,50 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { cJSON* pRoot; // in compiler optimization, switch case = if else constants - if (pRpcMsg->msgType == SYNC_TIMEOUT) { + if (pRpcMsg->msgType == TDMT_VND_SYNC_TIMEOUT) { SyncTimeout* pSyncMsg = syncTimeoutDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); pRoot = syncTimeout2Json(pSyncMsg); syncTimeoutDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == SYNC_PING) { + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING) { SyncPing* pSyncMsg = syncPingDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); pRoot = syncPing2Json(pSyncMsg); syncPingDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == SYNC_PING_REPLY) { + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING_REPLY) { SyncPingReply* pSyncMsg = syncPingReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); pRoot = syncPingReply2Json(pSyncMsg); syncPingReplyDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == SYNC_CLIENT_REQUEST) { + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) { SyncClientRequest* pSyncMsg = syncClientRequestDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); pRoot = syncClientRequest2Json(pSyncMsg); syncClientRequestDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == SYNC_CLIENT_REQUEST_REPLY) { + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST_REPLY) { pRoot = syncRpcUnknownMsg2Json(); - } else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE) { + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) { SyncRequestVote* pSyncMsg = syncRequestVoteDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); pRoot = syncRequestVote2Json(pSyncMsg); syncRequestVoteDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE_REPLY) { + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) { SyncRequestVoteReply* pSyncMsg = syncRequestVoteReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); pRoot = syncRequestVoteReply2Json(pSyncMsg); syncRequestVoteReplyDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES) { + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) { SyncAppendEntries* pSyncMsg = syncAppendEntriesDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); pRoot = syncAppendEntries2Json(pSyncMsg); syncAppendEntriesDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES_REPLY) { + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) { SyncAppendEntriesReply* pSyncMsg = syncAppendEntriesReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); pRoot = syncAppendEntriesReply2Json(pSyncMsg); syncAppendEntriesReplyDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == SYNC_RESPONSE) { + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_COMMON_RESPONSE) { pRoot = cJSON_CreateObject(); char* s; s = syncUtilprintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); @@ -98,7 +98,7 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { cJSON* syncRpcUnknownMsg2Json() { cJSON* pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "msgType", SYNC_UNKNOWN); + cJSON_AddNumberToObject(pRoot, "msgType", TDMT_VND_SYNC_UNKNOWN); cJSON_AddStringToObject(pRoot, "data", "unknown message"); cJSON* pJson = cJSON_CreateObject(); @@ -146,12 +146,14 @@ SyncTimeout* syncTimeoutBuild() { SyncTimeout* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; - pMsg->msgType = SYNC_TIMEOUT; + pMsg->msgType = TDMT_VND_SYNC_TIMEOUT; return pMsg; } -SyncTimeout* syncTimeoutBuild2(ESyncTimeoutType timeoutType, uint64_t logicClock, int32_t timerMS, void* data) { +SyncTimeout* syncTimeoutBuild2(ESyncTimeoutType timeoutType, uint64_t logicClock, int32_t timerMS, int32_t vgId, + void* data) { SyncTimeout* pMsg = syncTimeoutBuild(); + pMsg->vgId = vgId; pMsg->timeoutType = timeoutType; pMsg->logicClock = logicClock; pMsg->timerMS = timerMS; @@ -217,6 +219,7 @@ cJSON* syncTimeout2Json(const SyncTimeout* pMsg) { if (pMsg != NULL) { cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON_AddNumberToObject(pRoot, "timeoutType", pMsg->timeoutType); snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->logicClock); @@ -271,22 +274,23 @@ SyncPing* syncPingBuild(uint32_t dataLen) { SyncPing* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; - pMsg->msgType = SYNC_PING; + pMsg->msgType = TDMT_VND_SYNC_PING; pMsg->dataLen = dataLen; return pMsg; } -SyncPing* syncPingBuild2(const SRaftId* srcId, const SRaftId* destId, const char* str) { +SyncPing* syncPingBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str) { uint32_t dataLen = strlen(str) + 1; SyncPing* pMsg = syncPingBuild(dataLen); + pMsg->vgId = vgId; pMsg->srcId = *srcId; pMsg->destId = *destId; snprintf(pMsg->data, pMsg->dataLen, "%s", str); return pMsg; } -SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId) { - SyncPing* pMsg = syncPingBuild2(srcId, destId, "ping"); +SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId) { + SyncPing* pMsg = syncPingBuild2(srcId, destId, vgId, "ping"); return pMsg; } @@ -326,6 +330,98 @@ SyncPing* syncPingDeserialize2(const char* buf, uint32_t len) { return pMsg; } +int32_t syncPingSerialize3(const SyncPing* pMsg, char* buf, int32_t bufLen) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + if (tStartEncode(&encoder) < 0) { + return -1; + } + + if (tEncodeU32(&encoder, pMsg->bytes) < 0) { + return -1; + } + if (tEncodeI32(&encoder, pMsg->vgId) < 0) { + return -1; + } + if (tEncodeU32(&encoder, pMsg->msgType) < 0) { + return -1; + } + if (tEncodeU64(&encoder, pMsg->srcId.addr) < 0) { + return -1; + } + if (tEncodeI32(&encoder, pMsg->srcId.vgId) < 0) { + return -1; + } + if (tEncodeU64(&encoder, pMsg->destId.addr) < 0) { + return -1; + } + if (tEncodeI32(&encoder, pMsg->destId.vgId) < 0) { + return -1; + } + if (tEncodeU32(&encoder, pMsg->dataLen) < 0) { + return -1; + } + if (tEncodeBinary(&encoder, pMsg->data, pMsg->dataLen)) { + return -1; + } + + tEndEncode(&encoder); + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +SyncPing* syncPingDeserialize3(void* buf, int32_t bufLen) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + if (tStartDecode(&decoder) < 0) { + return NULL; + } + + SyncPing* pMsg = NULL; + uint32_t bytes; + if (tDecodeU32(&decoder, &bytes) < 0) { + return NULL; + } + + pMsg = taosMemoryMalloc(bytes); + assert(pMsg != NULL); + pMsg->bytes = bytes; + + if (tDecodeI32(&decoder, &pMsg->vgId) < 0) { + return NULL; + } + if (tDecodeU32(&decoder, &pMsg->msgType) < 0) { + return NULL; + } + if (tDecodeU64(&decoder, &pMsg->srcId.addr) < 0) { + return NULL; + } + if (tDecodeI32(&decoder, &pMsg->srcId.vgId) < 0) { + return NULL; + } + if (tDecodeU64(&decoder, &pMsg->destId.addr) < 0) { + return NULL; + } + if (tDecodeI32(&decoder, &pMsg->destId.vgId) < 0) { + return NULL; + } + if (tDecodeU32(&decoder, &pMsg->dataLen) < 0) { + return NULL; + } + uint64_t len; + char* data = NULL; + if (tDecodeBinary(&decoder, (const void**)(&data), &len) < 0) { + return NULL; + } + assert(len = pMsg->dataLen); + memcpy(pMsg->data, data, len); + + tEndDecode(&decoder); + tCoderClear(&decoder); + return pMsg; +} + void syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg) { memset(pRpcMsg, 0, sizeof(*pRpcMsg)); pRpcMsg->msgType = pMsg->msgType; @@ -349,6 +445,7 @@ cJSON* syncPing2Json(const SyncPing* pMsg) { if (pMsg != NULL) { cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON* pSrcId = cJSON_CreateObject(); @@ -436,22 +533,23 @@ SyncPingReply* syncPingReplyBuild(uint32_t dataLen) { SyncPingReply* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; - pMsg->msgType = SYNC_PING_REPLY; + pMsg->msgType = TDMT_VND_SYNC_PING_REPLY; pMsg->dataLen = dataLen; return pMsg; } -SyncPingReply* syncPingReplyBuild2(const SRaftId* srcId, const SRaftId* destId, const char* str) { +SyncPingReply* syncPingReplyBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str) { uint32_t dataLen = strlen(str) + 1; SyncPingReply* pMsg = syncPingReplyBuild(dataLen); + pMsg->vgId = vgId; pMsg->srcId = *srcId; pMsg->destId = *destId; snprintf(pMsg->data, pMsg->dataLen, "%s", str); return pMsg; } -SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId) { - SyncPingReply* pMsg = syncPingReplyBuild2(srcId, destId, "pang"); +SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId) { + SyncPingReply* pMsg = syncPingReplyBuild2(srcId, destId, vgId, "pang"); return pMsg; } @@ -491,6 +589,98 @@ SyncPingReply* syncPingReplyDeserialize2(const char* buf, uint32_t len) { return pMsg; } +int32_t syncPingReplySerialize3(const SyncPingReply* pMsg, char* buf, int32_t bufLen) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + if (tStartEncode(&encoder) < 0) { + return -1; + } + + if (tEncodeU32(&encoder, pMsg->bytes) < 0) { + return -1; + } + if (tEncodeI32(&encoder, pMsg->vgId) < 0) { + return -1; + } + if (tEncodeU32(&encoder, pMsg->msgType) < 0) { + return -1; + } + if (tEncodeU64(&encoder, pMsg->srcId.addr) < 0) { + return -1; + } + if (tEncodeI32(&encoder, pMsg->srcId.vgId) < 0) { + return -1; + } + if (tEncodeU64(&encoder, pMsg->destId.addr) < 0) { + return -1; + } + if (tEncodeI32(&encoder, pMsg->destId.vgId) < 0) { + return -1; + } + if (tEncodeU32(&encoder, pMsg->dataLen) < 0) { + return -1; + } + if (tEncodeBinary(&encoder, pMsg->data, pMsg->dataLen)) { + return -1; + } + + tEndEncode(&encoder); + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +SyncPingReply* syncPingReplyDeserialize3(void* buf, int32_t bufLen) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + if (tStartDecode(&decoder) < 0) { + return NULL; + } + + SyncPingReply* pMsg = NULL; + uint32_t bytes; + if (tDecodeU32(&decoder, &bytes) < 0) { + return NULL; + } + + pMsg = taosMemoryMalloc(bytes); + assert(pMsg != NULL); + pMsg->bytes = bytes; + + if (tDecodeI32(&decoder, &pMsg->vgId) < 0) { + return NULL; + } + if (tDecodeU32(&decoder, &pMsg->msgType) < 0) { + return NULL; + } + if (tDecodeU64(&decoder, &pMsg->srcId.addr) < 0) { + return NULL; + } + if (tDecodeI32(&decoder, &pMsg->srcId.vgId) < 0) { + return NULL; + } + if (tDecodeU64(&decoder, &pMsg->destId.addr) < 0) { + return NULL; + } + if (tDecodeI32(&decoder, &pMsg->destId.vgId) < 0) { + return NULL; + } + if (tDecodeU32(&decoder, &pMsg->dataLen) < 0) { + return NULL; + } + uint64_t len; + char* data = NULL; + if (tDecodeBinary(&decoder, (const void**)(&data), &len) < 0) { + return NULL; + } + assert(len = pMsg->dataLen); + memcpy(pMsg->data, data, len); + + tEndDecode(&decoder); + tCoderClear(&decoder); + return pMsg; +} + void syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg) { memset(pRpcMsg, 0, sizeof(*pRpcMsg)); pRpcMsg->msgType = pMsg->msgType; @@ -514,6 +704,7 @@ cJSON* syncPingReply2Json(const SyncPingReply* pMsg) { if (pMsg != NULL) { cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON* pSrcId = cJSON_CreateObject(); @@ -601,7 +792,7 @@ SyncClientRequest* syncClientRequestBuild(uint32_t dataLen) { SyncClientRequest* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; - pMsg->msgType = SYNC_CLIENT_REQUEST; + pMsg->msgType = TDMT_VND_SYNC_CLIENT_REQUEST; pMsg->seqNum = 0; pMsg->isWeak = false; pMsg->dataLen = dataLen; @@ -609,8 +800,9 @@ SyncClientRequest* syncClientRequestBuild(uint32_t dataLen) { } // step 1. original SRpcMsg => SyncClientRequest, add seqNum, isWeak -SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak) { +SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak, int32_t vgId) { SyncClientRequest* pMsg = syncClientRequestBuild(pOriginalRpcMsg->contLen); + pMsg->vgId = vgId; pMsg->originalRpcType = pOriginalRpcMsg->msgType; pMsg->seqNum = seqNum; pMsg->isWeak = isWeak; @@ -678,6 +870,7 @@ cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg) { if (pMsg != NULL) { cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON_AddNumberToObject(pRoot, "originalRpcType", pMsg->originalRpcType); snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->seqNum); @@ -734,12 +927,13 @@ void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg) { } // ---- message process SyncRequestVote---- -SyncRequestVote* syncRequestVoteBuild() { +SyncRequestVote* syncRequestVoteBuild(int32_t vgId) { uint32_t bytes = sizeof(SyncRequestVote); SyncRequestVote* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; - pMsg->msgType = SYNC_REQUEST_VOTE; + pMsg->vgId = vgId; + pMsg->msgType = TDMT_VND_SYNC_REQUEST_VOTE; return pMsg; } @@ -801,6 +995,7 @@ cJSON* syncRequestVote2Json(const SyncRequestVote* pMsg) { if (pMsg != NULL) { cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON* pSrcId = cJSON_CreateObject(); @@ -880,12 +1075,13 @@ void syncRequestVoteLog2(char* s, const SyncRequestVote* pMsg) { } // ---- message process SyncRequestVoteReply---- -SyncRequestVoteReply* syncRequestVoteReplyBuild() { +SyncRequestVoteReply* syncRequestVoteReplyBuild(int32_t vgId) { uint32_t bytes = sizeof(SyncRequestVoteReply); SyncRequestVoteReply* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; - pMsg->msgType = SYNC_REQUEST_VOTE_REPLY; + pMsg->vgId = vgId; + pMsg->msgType = TDMT_VND_SYNC_REQUEST_VOTE_REPLY; return pMsg; } @@ -947,6 +1143,7 @@ cJSON* syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg) { if (pMsg != NULL) { cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON* pSrcId = cJSON_CreateObject(); @@ -1023,12 +1220,13 @@ void syncRequestVoteReplyLog2(char* s, const SyncRequestVoteReply* pMsg) { } // ---- message process SyncAppendEntries---- -SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen) { +SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen, int32_t vgId) { uint32_t bytes = sizeof(SyncAppendEntries) + dataLen; SyncAppendEntries* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; - pMsg->msgType = SYNC_APPEND_ENTRIES; + pMsg->vgId = vgId; + pMsg->msgType = TDMT_VND_SYNC_APPEND_ENTRIES; pMsg->dataLen = dataLen; return pMsg; } @@ -1092,6 +1290,7 @@ cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg) { if (pMsg != NULL) { cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON* pSrcId = cJSON_CreateObject(); @@ -1186,12 +1385,13 @@ void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg) { } // ---- message process SyncAppendEntriesReply---- -SyncAppendEntriesReply* syncAppendEntriesReplyBuild() { +SyncAppendEntriesReply* syncAppendEntriesReplyBuild(int32_t vgId) { uint32_t bytes = sizeof(SyncAppendEntriesReply); SyncAppendEntriesReply* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; - pMsg->msgType = SYNC_APPEND_ENTRIES_REPLY; + pMsg->vgId = vgId; + pMsg->msgType = TDMT_VND_SYNC_APPEND_ENTRIES_REPLY; return pMsg; } @@ -1253,6 +1453,7 @@ cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg) { if (pMsg != NULL) { cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON* pSrcId = cJSON_CreateObject(); diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c new file mode 100644 index 0000000000..dc540424ec --- /dev/null +++ b/source/libs/sync/src/syncRaftCfg.c @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncRaftCfg.h" +#include "cJSON.h" +#include "syncEnv.h" +#include "syncUtil.h" + +// file must already exist! +SRaftCfg *raftCfgOpen(const char *path) { + SRaftCfg *pCfg = taosMemoryMalloc(sizeof(SRaftCfg)); + snprintf(pCfg->path, sizeof(pCfg->path), "%s", path); + + pCfg->pFile = taosOpenFile(pCfg->path, TD_FILE_READ | TD_FILE_WRITE); + assert(pCfg->pFile != NULL); + + taosLSeekFile(pCfg->pFile, 0, SEEK_SET); + + char buf[1024]; + int len = taosReadFile(pCfg->pFile, buf, sizeof(buf)); + assert(len > 0); + + int32_t ret = syncCfgFromStr(buf, &(pCfg->cfg)); + assert(ret == 0); + + return pCfg; +} + +int32_t raftCfgClose(SRaftCfg *pRaftCfg) { + int64_t ret = taosCloseFile(&(pRaftCfg->pFile)); + assert(ret == 0); + taosMemoryFree(pRaftCfg); + return 0; +} + +int32_t raftCfgPersist(SRaftCfg *pRaftCfg) { + assert(pRaftCfg != NULL); + + char *s = syncCfg2Str(&(pRaftCfg->cfg)); + taosLSeekFile(pRaftCfg->pFile, 0, SEEK_SET); + int64_t ret = taosWriteFile(pRaftCfg->pFile, s, strlen(s) + 1); + assert(ret == strlen(s) + 1); + taosMemoryFree(s); + + taosFsyncFile(pRaftCfg->pFile); + return 0; +} + +cJSON *syncCfg2Json(SSyncCfg *pSyncCfg) { + char u64buf[128]; + cJSON *pRoot = cJSON_CreateObject(); + + if (pSyncCfg != NULL) { + cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncCfg->replicaNum); + cJSON_AddNumberToObject(pRoot, "myIndex", pSyncCfg->myIndex); + + cJSON *pNodeInfoArr = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "nodeInfo", pNodeInfoArr); + for (int i = 0; i < pSyncCfg->replicaNum; ++i) { + cJSON *pNodeInfo = cJSON_CreateObject(); + cJSON_AddNumberToObject(pNodeInfo, "nodePort", ((pSyncCfg->nodeInfo)[i]).nodePort); + cJSON_AddStringToObject(pNodeInfo, "nodeFqdn", ((pSyncCfg->nodeInfo)[i]).nodeFqdn); + cJSON_AddItemToArray(pNodeInfoArr, pNodeInfo); + } + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncCfg", pRoot); + return pJson; +} + +char *syncCfg2Str(SSyncCfg *pSyncCfg) { + cJSON *pJson = syncCfg2Json(pSyncCfg); + char * serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg) { + memset(pSyncCfg, 0, sizeof(SSyncCfg)); + cJSON *pJson = cJSON_GetObjectItem(pRoot, "SSyncCfg"); + + cJSON *pReplicaNum = cJSON_GetObjectItem(pJson, "replicaNum"); + assert(cJSON_IsNumber(pReplicaNum)); + pSyncCfg->replicaNum = cJSON_GetNumberValue(pReplicaNum); + + cJSON *pMyIndex = cJSON_GetObjectItem(pJson, "myIndex"); + assert(cJSON_IsNumber(pMyIndex)); + pSyncCfg->myIndex = cJSON_GetNumberValue(pMyIndex); + + cJSON *pNodeInfoArr = cJSON_GetObjectItem(pJson, "nodeInfo"); + int arraySize = cJSON_GetArraySize(pNodeInfoArr); + assert(arraySize == pSyncCfg->replicaNum); + + for (int i = 0; i < arraySize; ++i) { + cJSON *pNodeInfo = cJSON_GetArrayItem(pNodeInfoArr, i); + assert(pNodeInfo != NULL); + + cJSON *pNodePort = cJSON_GetObjectItem(pNodeInfo, "nodePort"); + assert(cJSON_IsNumber(pNodePort)); + ((pSyncCfg->nodeInfo)[i]).nodePort = cJSON_GetNumberValue(pNodePort); + + cJSON *pNodeFqdn = cJSON_GetObjectItem(pNodeInfo, "nodeFqdn"); + assert(cJSON_IsString(pNodeFqdn)); + snprintf(((pSyncCfg->nodeInfo)[i]).nodeFqdn, sizeof(((pSyncCfg->nodeInfo)[i]).nodeFqdn), "%s", + pNodeFqdn->valuestring); + } + + return 0; +} + +int32_t syncCfgFromStr(const char *s, SSyncCfg *pSyncCfg) { + cJSON *pRoot = cJSON_Parse(s); + assert(pRoot != NULL); + + int32_t ret = syncCfgFromJson(pRoot, pSyncCfg); + assert(ret == 0); + + cJSON_Delete(pRoot); + return 0; +} + +cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) { + cJSON *pJson = syncCfg2Json(&(pRaftCfg->cfg)); + return pJson; +} + +char *raftCfg2Str(SRaftCfg *pRaftCfg) { + char *s = syncCfg2Str(&(pRaftCfg->cfg)); + return s; +} + +int32_t syncCfgCreateFile(SSyncCfg *pCfg, const char *path) { + assert(pCfg != NULL); + + TdFilePtr pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE); + assert(pFile != NULL); + + char * s = syncCfg2Str(pCfg); + int64_t ret = taosWriteFile(pFile, s, strlen(s) + 1); + assert(ret == strlen(s) + 1); + + taosMemoryFree(s); + taosCloseFile(&pFile); + return 0; +} + +// for debug ---------------------- +void syncCfgPrint(SSyncCfg *pCfg) { + char *serialized = syncCfg2Str(pCfg); + printf("syncCfgPrint | len:%lu | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncCfgPrint2(char *s, SSyncCfg *pCfg) { + char *serialized = syncCfg2Str(pCfg); + printf("syncCfgPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncCfgLog(SSyncCfg *pCfg) { + char *serialized = syncCfg2Str(pCfg); + sTrace("syncCfgLog | len:%lu | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncCfgLog2(char *s, SSyncCfg *pCfg) { + char *serialized = syncCfg2Str(pCfg); + sTrace("syncCfgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); +} + +void raftCfgPrint(SRaftCfg *pCfg) { + char *serialized = raftCfg2Str(pCfg); + printf("raftCfgPrint | len:%lu | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftCfgPrint2(char *s, SRaftCfg *pCfg) { + char *serialized = raftCfg2Str(pCfg); + printf("raftCfgPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftCfgLog(SRaftCfg *pCfg) { + char *serialized = raftCfg2Str(pCfg); + sTrace("raftCfgLog | len:%lu | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void raftCfgLog2(char *s, SRaftCfg *pCfg) { + char *serialized = raftCfg2Str(pCfg); + sTrace("raftCfgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); +} diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index 3de39a6aa8..21ee35eaf9 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -28,13 +28,13 @@ SSyncRaftEntry* syncEntryBuild(uint32_t dataLen) { // step 4. SyncClientRequest => SSyncRaftEntry, add term, index SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) { - SSyncRaftEntry* pEntry = syncEntryBuild3(pMsg, term, index, SYNC_RAFT_ENTRY_DATA); + SSyncRaftEntry* pEntry = syncEntryBuild3(pMsg, term, index); assert(pEntry != NULL); return pEntry; } -SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index, EntryType entryType) { +SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) { SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->dataLen); assert(pEntry != NULL); @@ -44,19 +44,37 @@ SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncInde pEntry->isWeak = pMsg->isWeak; pEntry->term = term; pEntry->index = index; - pEntry->entryType = entryType; pEntry->dataLen = pMsg->dataLen; memcpy(pEntry->data, pMsg->data, pMsg->dataLen); return pEntry; } -SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index) { - SSyncRaftEntry* pEntry = syncEntryBuild(0); +SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId) { + // init rpcMsg + SMsgHead head; + head.vgId = vgId; + head.contLen = sizeof(SMsgHead); + SRpcMsg rpcMsg; + memset(&rpcMsg, 0, sizeof(SRpcMsg)); + rpcMsg.contLen = head.contLen; + rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); + rpcMsg.msgType = TDMT_VND_SYNC_NOOP; + memcpy(rpcMsg.pCont, &head, sizeof(head)); + + SSyncRaftEntry* pEntry = syncEntryBuild(rpcMsg.contLen); assert(pEntry != NULL); + + pEntry->msgType = TDMT_VND_SYNC_CLIENT_REQUEST; + pEntry->originalRpcType = TDMT_VND_SYNC_NOOP; + pEntry->seqNum = 0; + pEntry->isWeak = 0; pEntry->term = term; pEntry->index = index; - pEntry->entryType = SYNC_RAFT_ENTRY_NOOP; + + assert(pEntry->dataLen == rpcMsg.contLen); + memcpy(pEntry->data, rpcMsg.pCont, rpcMsg.contLen); + rpcFreeCont(rpcMsg.pCont); return pEntry; } @@ -103,7 +121,6 @@ cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry) { cJSON_AddStringToObject(pRoot, "term", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pEntry->index); cJSON_AddStringToObject(pRoot, "index", u64buf); - cJSON_AddNumberToObject(pRoot, "entryType", pEntry->entryType); cJSON_AddNumberToObject(pRoot, "dataLen", pEntry->dataLen); char* s; diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 63c6265d10..dc903ad7b2 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -50,38 +50,46 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { SyncIndex lastIndex = logStoreLastIndex(pLogStore); assert(pEntry->index == lastIndex + 1); - uint32_t len; - char* serialized = syncEntrySerialize(pEntry, &len); - assert(serialized != NULL); - int code = 0; - /* - code = walWrite(pWal, pEntry->index, pEntry->entryType, serialized, len); - assert(code == 0); - */ - assert(walWrite(pWal, pEntry->index, pEntry->entryType, serialized, len) == 0); + int code = 0; + SSyncLogMeta syncMeta; + syncMeta.isWeek = pEntry->isWeak; + syncMeta.seqNum = pEntry->seqNum; + syncMeta.term = pEntry->term; + code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen); + assert(code == 0); walFsync(pWal, true); - taosMemoryFree(serialized); return code; } SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - SSyncRaftEntry* pEntry = NULL; if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) { SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); assert(walReadWithHandle(pWalHandle, index) == 0); - pEntry = syncEntryDeserialize(pWalHandle->pHead->head.body, pWalHandle->pHead->head.len); + + SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.len); assert(pEntry != NULL); + pEntry->msgType = TDMT_VND_SYNC_CLIENT_REQUEST; + pEntry->originalRpcType = pWalHandle->pHead->head.msgType; + pEntry->seqNum = pWalHandle->pHead->head.syncMeta.seqNum; + pEntry->isWeak = pWalHandle->pHead->head.syncMeta.isWeek; + pEntry->term = pWalHandle->pHead->head.syncMeta.term; + pEntry->index = index; + assert(pEntry->dataLen == pWalHandle->pHead->head.len); + memcpy(pEntry->data, pWalHandle->pHead->head.body, pWalHandle->pHead->head.len); + // need to hold, do not new every time!! walCloseReadHandle(pWalHandle); - } + return pEntry; - return pEntry; + } else { + return NULL; + } } int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { @@ -207,20 +215,20 @@ void logStorePrint(SSyncLogStore* pLogStore) { void logStorePrint2(char* s, SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); - printf("logStorePrint | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("logStorePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void logStoreLog(SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); - sTrace("logStorePrint | len:%lu | %s", strlen(serialized), serialized); + sTraceLong("logStoreLog | len:%lu | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void logStoreLog2(char* s, SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); - sTrace("logStorePrint | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTraceLong("logStoreLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 3ced7a18bc..2fdb8a0e17 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -77,7 +77,7 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { SyncAppendEntries* pMsg = NULL; SSyncRaftEntry* pEntry = logStoreGetEntry(pSyncNode->pLogStore, nextIndex); if (pEntry != NULL) { - pMsg = syncAppendEntriesBuild(pEntry->bytes); + pMsg = syncAppendEntriesBuild(pEntry->bytes, pSyncNode->vgId); assert(pMsg != NULL); // add pEntry into msg @@ -91,7 +91,7 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { } else { // maybe overflow, send empty record - pMsg = syncAppendEntriesBuild(0); + pMsg = syncAppendEntriesBuild(0, pSyncNode->vgId); assert(pMsg != NULL); } diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index e23748a81e..619a1546a9 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -64,7 +64,7 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { raftStoreVote(ths->pRaftStore, &(pMsg->srcId)); } - SyncRequestVoteReply* pReply = syncRequestVoteReplyBuild(); + SyncRequestVoteReply* pReply = syncRequestVoteReplyBuild(ths->vgId); pReply->srcId = ths->myRaftId; pReply->destId = pMsg->srcId; pReply->term = ths->pRaftStore->currentTerm; diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index 1531f701ff..a6348dff50 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -49,11 +49,20 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) return ret; } - assert(!(pMsg->term > ths->pRaftStore->currentTerm)); - // no need this code, because if I receive reply.term, then I must have sent for that term. - // if (pMsg->term > ths->pRaftStore->currentTerm) { - // syncNodeUpdateTerm(ths, pMsg->term); - // } + // assert(!(pMsg->term > ths->pRaftStore->currentTerm)); + // no need this code, because if I receive reply.term, then I must have sent for that term. + // if (pMsg->term > ths->pRaftStore->currentTerm) { + // syncNodeUpdateTerm(ths, pMsg->term); + // } + + if (pMsg->term > ths->pRaftStore->currentTerm) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "syncNodeOnRequestVoteReplyCb error term, receive:%lu current:%lu", pMsg->term, + ths->pRaftStore->currentTerm); + syncNodePrint2(logBuf, ths); + sError("%s", logBuf); + return ret; + } assert(pMsg->term == ths->pRaftStore->currentTerm); diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c new file mode 100644 index 0000000000..47c73745fd --- /dev/null +++ b/source/libs/sync/src/syncRespMgr.c @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncRespMgr.h" + +SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) { + SSyncRespMgr *pObj = (SSyncRespMgr *)taosMemoryMalloc(sizeof(SSyncRespMgr)); + memset(pObj, 0, sizeof(SSyncRespMgr)); + + pObj->pRespHash = + taosHashInit(sizeof(uint64_t), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + assert(pObj->pRespHash != NULL); + pObj->ttl = ttl; + pObj->data = data; + pObj->seqNum = 0; + taosThreadMutexInit(&(pObj->mutex), NULL); + + return pObj; +} + +void syncRespMgrDestroy(SSyncRespMgr *pObj) { + taosThreadMutexLock(&(pObj->mutex)); + taosHashCleanup(pObj->pRespHash); + taosThreadMutexUnlock(&(pObj->mutex)); + taosThreadMutexDestroy(&(pObj->mutex)); + taosMemoryFree(pObj); +} + +int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub) { + taosThreadMutexLock(&(pObj->mutex)); + + uint64_t keyCode = ++(pObj->seqNum); + taosHashPut(pObj->pRespHash, &keyCode, sizeof(keyCode), pStub, sizeof(SRespStub)); + + taosThreadMutexUnlock(&(pObj->mutex)); + return keyCode; +} + +int32_t syncRespMgrDel(SSyncRespMgr *pObj, uint64_t index) { + taosThreadMutexLock(&(pObj->mutex)); + + taosHashRemove(pObj->pRespHash, &index, sizeof(index)); + + taosThreadMutexUnlock(&(pObj->mutex)); + return 0; +} + +int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub) { + taosThreadMutexLock(&(pObj->mutex)); + + void *pTmp = taosHashGet(pObj->pRespHash, &index, sizeof(index)); + if (pTmp != NULL) { + memcpy(pStub, pTmp, sizeof(SRespStub)); + taosThreadMutexUnlock(&(pObj->mutex)); + return 1; // get one object + } + taosThreadMutexUnlock(&(pObj->mutex)); + return 0; // get none object +} + +int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub) { + taosThreadMutexLock(&(pObj->mutex)); + + void *pTmp = taosHashGet(pObj->pRespHash, &index, sizeof(index)); + if (pTmp != NULL) { + memcpy(pStub, pTmp, sizeof(SRespStub)); + taosThreadMutexUnlock(&(pObj->mutex)); + taosHashRemove(pObj->pRespHash, &index, sizeof(index)); + return 1; // get one object + } + taosThreadMutexUnlock(&(pObj->mutex)); + return 0; // get none object +} + +void syncRespClean(SSyncRespMgr *pObj) { + taosThreadMutexLock(&(pObj->mutex)); + syncRespCleanByTTL(pObj, pObj->ttl); + taosThreadMutexUnlock(&(pObj->mutex)); +} + +void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl) {} \ No newline at end of file diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 287500a587..3b20dab0db 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -197,3 +197,17 @@ SyncIndex syncUtilMaxIndex(SyncIndex a, SyncIndex b) { SyncIndex r = a > b ? a : b; return r; } + +void syncUtilMsgHtoN(void* msg) { + // htonl + SMsgHead* pHead = msg; + pHead->contLen = htonl(pHead->contLen); + pHead->vgId = htonl(pHead->vgId); +} + +void syncUtilMsgNtoH(void* msg) { + // ntohl + SMsgHead* pHead = msg; + pHead->contLen = ntohl(pHead->contLen); + pHead->vgId = ntohl(pHead->vgId); +} \ No newline at end of file diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 58b01e4ca2..85fe285661 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -38,6 +38,8 @@ add_executable(syncReplicateTest3 "") add_executable(syncReplicateLoadTest "") add_executable(syncRefTest "") add_executable(syncLogStoreCheck "") +add_executable(syncRaftCfgTest "") +add_executable(syncRespMgrTest "") target_sources(syncTest @@ -200,206 +202,224 @@ target_sources(syncLogStoreCheck PRIVATE "syncLogStoreCheck.cpp" ) +target_sources(syncRaftCfgTest + PRIVATE + "syncRaftCfgTest.cpp" +) +target_sources(syncRespMgrTest + PRIVATE + "syncRespMgrTest.cpp" +) target_include_directories(syncTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncEnvTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncPingTimerTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncIOTickQTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncIOTickPingTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncIOSendMsgTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncIOClientTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncIOServerTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncRaftStoreTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncEnqTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncIndexTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncInitTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncUtilTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncVotesGrantedTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncVotesRespondTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncIndexMgrTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncLogStoreTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncEntryTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncRequestVoteTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncRequestVoteReplyTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncAppendEntriesTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncAppendEntriesReplyTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncClientRequestTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncTimeoutTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncPingTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncPingReplyTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncRpcMsgTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncPingTimerTest2 PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncPingSelfTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncElectTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncElectTest2 PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncElectTest3 PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncEncodeTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncWriteTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncReplicateTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncReplicateTest2 PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncReplicateTest3 PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncReplicateLoadTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncRefTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncLogStoreCheck PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncRaftCfgTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncRespMgrTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) @@ -564,6 +584,14 @@ target_link_libraries(syncLogStoreCheck sync gtest_main ) +target_link_libraries(syncRaftCfgTest + sync + gtest_main +) +target_link_libraries(syncRespMgrTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp index 72aeb155e7..a90259bc3a 100644 --- a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp @@ -15,7 +15,7 @@ void logTest() { } SyncAppendEntriesReply *createMsg() { - SyncAppendEntriesReply *pMsg = syncAppendEntriesReplyBuild(); + SyncAppendEntriesReply *pMsg = syncAppendEntriesReplyBuild(1000); pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); pMsg->srcId.vgId = 100; pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); @@ -27,7 +27,7 @@ SyncAppendEntriesReply *createMsg() { void test1() { SyncAppendEntriesReply *pMsg = createMsg(); - syncAppendEntriesReplyPrint2((char *)"test1:", pMsg); + syncAppendEntriesReplyLog2((char *)"test1:", pMsg); syncAppendEntriesReplyDestroy(pMsg); } @@ -36,10 +36,10 @@ void test2() { uint32_t len = pMsg->bytes; char * serialized = (char *)taosMemoryMalloc(len); syncAppendEntriesReplySerialize(pMsg, serialized, len); - SyncAppendEntriesReply *pMsg2 = syncAppendEntriesReplyBuild(); + SyncAppendEntriesReply *pMsg2 = syncAppendEntriesReplyBuild(1000); syncAppendEntriesReplyDeserialize(serialized, len, pMsg2); - syncAppendEntriesReplyPrint2((char *)"test2: syncAppendEntriesReplySerialize -> syncAppendEntriesReplyDeserialize ", - pMsg2); + syncAppendEntriesReplyLog2((char *)"test2: syncAppendEntriesReplySerialize -> syncAppendEntriesReplyDeserialize ", + pMsg2); taosMemoryFree(serialized); syncAppendEntriesReplyDestroy(pMsg); @@ -51,8 +51,8 @@ void test3() { uint32_t len; char * serialized = syncAppendEntriesReplySerialize2(pMsg, &len); SyncAppendEntriesReply *pMsg2 = syncAppendEntriesReplyDeserialize2(serialized, len); - syncAppendEntriesReplyPrint2((char *)"test3: syncAppendEntriesReplySerialize3 -> syncAppendEntriesReplyDeserialize2 ", - pMsg2); + syncAppendEntriesReplyLog2((char *)"test3: syncAppendEntriesReplySerialize3 -> syncAppendEntriesReplyDeserialize2 ", + pMsg2); taosMemoryFree(serialized); syncAppendEntriesReplyDestroy(pMsg); @@ -63,11 +63,12 @@ void test4() { SyncAppendEntriesReply *pMsg = createMsg(); SRpcMsg rpcMsg; syncAppendEntriesReply2RpcMsg(pMsg, &rpcMsg); - SyncAppendEntriesReply *pMsg2 = syncAppendEntriesReplyBuild(); + SyncAppendEntriesReply *pMsg2 = syncAppendEntriesReplyBuild(1000); syncAppendEntriesReplyFromRpcMsg(&rpcMsg, pMsg2); - syncAppendEntriesReplyPrint2((char *)"test4: syncAppendEntriesReply2RpcMsg -> syncAppendEntriesReplyFromRpcMsg ", - pMsg2); + syncAppendEntriesReplyLog2((char *)"test4: syncAppendEntriesReply2RpcMsg -> syncAppendEntriesReplyFromRpcMsg ", + pMsg2); + rpcFreeCont(rpcMsg.pCont); syncAppendEntriesReplyDestroy(pMsg); syncAppendEntriesReplyDestroy(pMsg2); } @@ -77,17 +78,17 @@ void test5() { SRpcMsg rpcMsg; syncAppendEntriesReply2RpcMsg(pMsg, &rpcMsg); SyncAppendEntriesReply *pMsg2 = syncAppendEntriesReplyFromRpcMsg2(&rpcMsg); - syncAppendEntriesReplyPrint2((char *)"test5: syncAppendEntriesReply2RpcMsg -> syncAppendEntriesReplyFromRpcMsg2 ", - pMsg2); + syncAppendEntriesReplyLog2((char *)"test5: syncAppendEntriesReply2RpcMsg -> syncAppendEntriesReplyFromRpcMsg2 ", + pMsg2); + rpcFreeCont(rpcMsg.pCont); syncAppendEntriesReplyDestroy(pMsg); syncAppendEntriesReplyDestroy(pMsg2); } int main() { - // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; - sDebugFlag = 143 + 64; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; logTest(); test1(); diff --git a/source/libs/sync/test/syncAppendEntriesTest.cpp b/source/libs/sync/test/syncAppendEntriesTest.cpp index 69a0aee9a8..bb9f306a1c 100644 --- a/source/libs/sync/test/syncAppendEntriesTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesTest.cpp @@ -15,7 +15,7 @@ void logTest() { } SyncAppendEntries *createMsg() { - SyncAppendEntries *pMsg = syncAppendEntriesBuild(20); + SyncAppendEntries *pMsg = syncAppendEntriesBuild(20, 1000); pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); pMsg->srcId.vgId = 100; pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); @@ -29,7 +29,7 @@ SyncAppendEntries *createMsg() { void test1() { SyncAppendEntries *pMsg = createMsg(); - syncAppendEntriesPrint2((char *)"test1:", pMsg); + syncAppendEntriesLog2((char *)"test1:", pMsg); syncAppendEntriesDestroy(pMsg); } @@ -38,9 +38,9 @@ void test2() { uint32_t len = pMsg->bytes; char * serialized = (char *)taosMemoryMalloc(len); syncAppendEntriesSerialize(pMsg, serialized, len); - SyncAppendEntries *pMsg2 = syncAppendEntriesBuild(pMsg->dataLen); + SyncAppendEntries *pMsg2 = syncAppendEntriesBuild(pMsg->dataLen, 1000); syncAppendEntriesDeserialize(serialized, len, pMsg2); - syncAppendEntriesPrint2((char *)"test2: syncAppendEntriesSerialize -> syncAppendEntriesDeserialize ", pMsg2); + syncAppendEntriesLog2((char *)"test2: syncAppendEntriesSerialize -> syncAppendEntriesDeserialize ", pMsg2); taosMemoryFree(serialized); syncAppendEntriesDestroy(pMsg); @@ -52,7 +52,7 @@ void test3() { uint32_t len; char * serialized = syncAppendEntriesSerialize2(pMsg, &len); SyncAppendEntries *pMsg2 = syncAppendEntriesDeserialize2(serialized, len); - syncAppendEntriesPrint2((char *)"test3: syncAppendEntriesSerialize3 -> syncAppendEntriesDeserialize2 ", pMsg2); + syncAppendEntriesLog2((char *)"test3: syncAppendEntriesSerialize3 -> syncAppendEntriesDeserialize2 ", pMsg2); taosMemoryFree(serialized); syncAppendEntriesDestroy(pMsg); @@ -65,8 +65,9 @@ void test4() { syncAppendEntries2RpcMsg(pMsg, &rpcMsg); SyncAppendEntries *pMsg2 = (SyncAppendEntries *)taosMemoryMalloc(rpcMsg.contLen); syncAppendEntriesFromRpcMsg(&rpcMsg, pMsg2); - syncAppendEntriesPrint2((char *)"test4: syncAppendEntries2RpcMsg -> syncAppendEntriesFromRpcMsg ", pMsg2); + syncAppendEntriesLog2((char *)"test4: syncAppendEntries2RpcMsg -> syncAppendEntriesFromRpcMsg ", pMsg2); + rpcFreeCont(rpcMsg.pCont); syncAppendEntriesDestroy(pMsg); syncAppendEntriesDestroy(pMsg2); } @@ -76,16 +77,16 @@ void test5() { SRpcMsg rpcMsg; syncAppendEntries2RpcMsg(pMsg, &rpcMsg); SyncAppendEntries *pMsg2 = syncAppendEntriesFromRpcMsg2(&rpcMsg); - syncAppendEntriesPrint2((char *)"test5: syncAppendEntries2RpcMsg -> syncAppendEntriesFromRpcMsg2 ", pMsg2); + syncAppendEntriesLog2((char *)"test5: syncAppendEntries2RpcMsg -> syncAppendEntriesFromRpcMsg2 ", pMsg2); + rpcFreeCont(rpcMsg.pCont); syncAppendEntriesDestroy(pMsg); syncAppendEntriesDestroy(pMsg2); } int main() { - // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; - sDebugFlag = 143 + 64; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; logTest(); test1(); diff --git a/source/libs/sync/test/syncClientRequestTest.cpp b/source/libs/sync/test/syncClientRequestTest.cpp index f22478d538..9376aca91f 100644 --- a/source/libs/sync/test/syncClientRequestTest.cpp +++ b/source/libs/sync/test/syncClientRequestTest.cpp @@ -21,13 +21,14 @@ SyncClientRequest *createMsg() { rpcMsg.contLen = 20; rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); strcpy((char *)rpcMsg.pCont, "hello rpc"); - SyncClientRequest *pMsg = syncClientRequestBuild2(&rpcMsg, 123, true); + SyncClientRequest *pMsg = syncClientRequestBuild2(&rpcMsg, 123, true, 1000); + rpcFreeCont(rpcMsg.pCont); return pMsg; } void test1() { SyncClientRequest *pMsg = createMsg(); - syncClientRequestPrint2((char *)"test1:", pMsg); + syncClientRequestLog2((char *)"test1:", pMsg); syncClientRequestDestroy(pMsg); } @@ -38,7 +39,7 @@ void test2() { syncClientRequestSerialize(pMsg, serialized, len); SyncClientRequest *pMsg2 = syncClientRequestBuild(pMsg->dataLen); syncClientRequestDeserialize(serialized, len, pMsg2); - syncClientRequestPrint2((char *)"test2: syncClientRequestSerialize -> syncClientRequestDeserialize ", pMsg2); + syncClientRequestLog2((char *)"test2: syncClientRequestSerialize -> syncClientRequestDeserialize ", pMsg2); taosMemoryFree(serialized); syncClientRequestDestroy(pMsg); @@ -50,7 +51,7 @@ void test3() { uint32_t len; char * serialized = syncClientRequestSerialize2(pMsg, &len); SyncClientRequest *pMsg2 = syncClientRequestDeserialize2(serialized, len); - syncClientRequestPrint2((char *)"test3: syncClientRequestSerialize3 -> syncClientRequestDeserialize2 ", pMsg2); + syncClientRequestLog2((char *)"test3: syncClientRequestSerialize3 -> syncClientRequestDeserialize2 ", pMsg2); taosMemoryFree(serialized); syncClientRequestDestroy(pMsg); @@ -63,8 +64,9 @@ void test4() { syncClientRequest2RpcMsg(pMsg, &rpcMsg); SyncClientRequest *pMsg2 = (SyncClientRequest *)taosMemoryMalloc(rpcMsg.contLen); syncClientRequestFromRpcMsg(&rpcMsg, pMsg2); - syncClientRequestPrint2((char *)"test4: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg ", pMsg2); + syncClientRequestLog2((char *)"test4: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg ", pMsg2); + rpcFreeCont(rpcMsg.pCont); syncClientRequestDestroy(pMsg); syncClientRequestDestroy(pMsg2); } @@ -74,16 +76,16 @@ void test5() { SRpcMsg rpcMsg; syncClientRequest2RpcMsg(pMsg, &rpcMsg); SyncClientRequest *pMsg2 = syncClientRequestFromRpcMsg2(&rpcMsg); - syncClientRequestPrint2((char *)"test5: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg2 ", pMsg2); + syncClientRequestLog2((char *)"test5: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg2 ", pMsg2); + rpcFreeCont(rpcMsg.pCont); syncClientRequestDestroy(pMsg); syncClientRequestDestroy(pMsg2); } int main() { - // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; - sDebugFlag = 143 + 64; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; logTest(); test1(); diff --git a/source/libs/sync/test/syncElectTest.cpp b/source/libs/sync/test/syncElectTest.cpp index 251a4b538f..39b5eba1aa 100644 --- a/source/libs/sync/test/syncElectTest.cpp +++ b/source/libs/sync/test/syncElectTest.cpp @@ -20,11 +20,12 @@ uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; int32_t replicaNum = 3; int32_t myIndex = 0; -SRaftId ids[TSDB_MAX_REPLICA]; -SSyncInfo syncInfo; -SSyncFSM* pFsm; -SWal* pWal; -SSyncNode* gSyncNode; +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SWal* pWal; +SSyncNode* gSyncNode; +const char* pDir = "./syncElectTest"; SSyncNode* syncNodeInit() { syncInfo.vgId = 1234; @@ -33,7 +34,7 @@ SSyncNode* syncNodeInit() { syncInfo.queue = gSyncIO->pMsgQ; syncInfo.FpEqMsg = syncIOEqMsg; syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "./elect_test_%d", myIndex); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s_sync_%d", pDir, myIndex); int code = walInit(); assert(code == 0); @@ -48,7 +49,7 @@ SSyncNode* syncNodeInit() { walCfg.level = TAOS_WAL_FSYNC; char tmpdir[128]; - snprintf(tmpdir, sizeof(tmpdir), "./elect_test_wal_%d", myIndex); + snprintf(tmpdir, sizeof(tmpdir), "%s_wal_%d", pDir, myIndex); pWal = walOpen(tmpdir, &walCfg); assert(pWal != NULL); @@ -78,6 +79,8 @@ SSyncNode* syncNodeInit() { gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; gSyncIO->pSyncNode = pSyncNode; + syncNodeStart(pSyncNode); + return pSyncNode; } @@ -110,16 +113,15 @@ int main(int argc, char** argv) { gSyncNode = syncInitTest(); assert(gSyncNode != NULL); - syncNodePrint2((char*)"", gSyncNode); + syncNodeLog2((char*)"", gSyncNode); initRaftId(gSyncNode); //--------------------------- while (1) { - sTrace( - "elect sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d", - gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->pRaftStore->currentTerm, - gSyncNode->electTimerLogicClock, gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS); + char* s = syncNode2SimpleStr(gSyncNode); + sTrace("%s", s); + taosMemoryFree(s); taosMsleep(1000); } diff --git a/source/libs/sync/test/syncElectTest2.cpp b/source/libs/sync/test/syncElectTest2.cpp index 709d9d8580..d569b47fb1 100644 --- a/source/libs/sync/test/syncElectTest2.cpp +++ b/source/libs/sync/test/syncElectTest2.cpp @@ -21,10 +21,11 @@ uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; int32_t replicaNum = 3; int32_t myIndex = 0; -SRaftId ids[TSDB_MAX_REPLICA]; -SSyncInfo syncInfo; -SSyncFSM* pFsm; -SWal* pWal; +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SWal* pWal; +const char* pDir = "./syncElectTest2"; int64_t syncNodeInit() { syncInfo.vgId = 1234; @@ -33,7 +34,7 @@ int64_t syncNodeInit() { syncInfo.queue = gSyncIO->pMsgQ; syncInfo.FpEqMsg = syncIOEqMsg; syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "./elect2_test_%d", myIndex); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s_sync_%d", pDir, myIndex); int code = walInit(); assert(code == 0); @@ -48,7 +49,7 @@ int64_t syncNodeInit() { walCfg.level = TAOS_WAL_FSYNC; char tmpdir[128]; - snprintf(tmpdir, sizeof(tmpdir), "./elect2_test_wal_%d", myIndex); + snprintf(tmpdir, sizeof(tmpdir), "%s_wal_%d", pDir, myIndex); pWal = walOpen(tmpdir, &walCfg); assert(pWal != NULL); @@ -64,8 +65,9 @@ int64_t syncNodeInit() { // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); } - int64_t rid = syncStart(&syncInfo); + int64_t rid = syncOpen(&syncInfo); assert(rid > 0); + syncStart(rid); SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); assert(pSyncNode != NULL); @@ -81,6 +83,8 @@ int64_t syncNodeInit() { gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; gSyncIO->pSyncNode = pSyncNode; + syncNodeStart(pSyncNode); + syncNodeRelease(pSyncNode); return rid; @@ -117,7 +121,7 @@ int main(int argc, char** argv) { SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); assert(pSyncNode != NULL); - syncNodePrint2((char*)"", pSyncNode); + syncNodeLog2((char*)"", pSyncNode); initRaftId(pSyncNode); //--------------------------- diff --git a/source/libs/sync/test/syncElectTest3.cpp b/source/libs/sync/test/syncElectTest3.cpp index e36a2d0fb1..59e1b993d8 100644 --- a/source/libs/sync/test/syncElectTest3.cpp +++ b/source/libs/sync/test/syncElectTest3.cpp @@ -21,10 +21,11 @@ uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; int32_t replicaNum = 3; int32_t myIndex = 0; -SRaftId ids[TSDB_MAX_REPLICA]; -SSyncInfo syncInfo; -SSyncFSM* pFsm; -SWal* pWal; +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SWal* pWal; +const char* pDir = "./syncElectTest3"; int64_t syncNodeInit() { syncInfo.vgId = 1234; @@ -33,7 +34,7 @@ int64_t syncNodeInit() { syncInfo.queue = gSyncIO->pMsgQ; syncInfo.FpEqMsg = syncIOEqMsg; syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "./elect3_test_%d", myIndex); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s_sync_%d", pDir, myIndex); int code = walInit(); assert(code == 0); @@ -48,7 +49,7 @@ int64_t syncNodeInit() { walCfg.level = TAOS_WAL_FSYNC; char tmpdir[128]; - snprintf(tmpdir, sizeof(tmpdir), "./elect3_test_wal_%d", myIndex); + snprintf(tmpdir, sizeof(tmpdir), "%s_wal_%d", pDir, myIndex); pWal = walOpen(tmpdir, &walCfg); assert(pWal != NULL); @@ -64,8 +65,9 @@ int64_t syncNodeInit() { // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); } - int64_t rid = syncStart(&syncInfo); + int64_t rid = syncOpen(&syncInfo); assert(rid > 0); + syncStart(rid); SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); assert(pSyncNode != NULL); @@ -84,6 +86,8 @@ int64_t syncNodeInit() { gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; gSyncIO->pSyncNode = pSyncNode; + syncNodeStart(pSyncNode); + syncNodeRelease(pSyncNode); return rid; @@ -120,7 +124,7 @@ int main(int argc, char** argv) { SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); assert(pSyncNode != NULL); - syncNodePrint2((char*)"", pSyncNode); + syncNodeLog2((char*)"", pSyncNode); initRaftId(pSyncNode); //--------------------------- diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp index b1ba95dc3e..f052d08a1e 100644 --- a/source/libs/sync/test/syncEncodeTest.cpp +++ b/source/libs/sync/test/syncEncodeTest.cpp @@ -102,7 +102,7 @@ SRpcMsg *step0() { } SyncClientRequest *step1(const SRpcMsg *pMsg) { - SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true); + SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true, 1000); return pRetMsg; } @@ -159,23 +159,23 @@ int main(int argc, char **argv) { // step0 SRpcMsg *pMsg0 = step0(); - syncRpcMsgPrint2((char *)"==step0==", pMsg0); + syncRpcMsgLog2((char *)"==step0==", pMsg0); // step1 SyncClientRequest *pMsg1 = step1(pMsg0); - syncClientRequestPrint2((char *)"==step1==", pMsg1); + syncClientRequestLog2((char *)"==step1==", pMsg1); // step2 SRpcMsg *pMsg2 = step2(pMsg1); - syncRpcMsgPrint2((char *)"==step2==", pMsg2); + syncRpcMsgLog2((char *)"==step2==", pMsg2); // step3 SyncClientRequest *pMsg3 = step3(pMsg2); - syncClientRequestPrint2((char *)"==step3==", pMsg3); + syncClientRequestLog2((char *)"==step3==", pMsg3); // step4 SSyncRaftEntry *pMsg4 = step4(pMsg3); - syncEntryPrint2((char *)"==step4==", pMsg4); + syncEntryLog2((char *)"==step4==", pMsg4); // log, relog SSyncNode *pSyncNode = syncNodeInit(); @@ -183,7 +183,7 @@ int main(int argc, char **argv) { SSyncRaftEntry *pEntry = pMsg4; pSyncNode->pLogStore->appendEntry(pSyncNode->pLogStore, pEntry); SSyncRaftEntry *pEntry2 = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, pEntry->index); - syncEntryPrint2((char *)"==pEntry2==", pEntry2); + syncEntryLog2((char *)"==pEntry2==", pEntry2); // step5 uint32_t len; @@ -194,11 +194,11 @@ int main(int argc, char **argv) { // step6 SSyncRaftEntry *pMsg6 = step6(pMsg5, len); - syncEntryPrint2((char *)"==step6==", pMsg6); + syncEntryLog2((char *)"==step6==", pMsg6); // step7 SRpcMsg *pMsg7 = step7(pMsg6); - syncRpcMsgPrint2((char *)"==step7==", pMsg7); + syncRpcMsgLog2((char *)"==step7==", pMsg7); return 0; } diff --git a/source/libs/sync/test/syncEnqTest.cpp b/source/libs/sync/test/syncEnqTest.cpp index 48cc7caf2f..6f83ede5a0 100644 --- a/source/libs/sync/test/syncEnqTest.cpp +++ b/source/libs/sync/test/syncEnqTest.cpp @@ -89,14 +89,14 @@ int main(int argc, char** argv) { SSyncNode* pSyncNode = syncInitTest(); assert(pSyncNode != NULL); - syncNodePrint2((char*)"syncInitTest", pSyncNode); + syncNodeLog2((char*)"syncInitTest", pSyncNode); initRaftId(pSyncNode); //-------------------------------------------------------------- for (int i = 0; i < 10; ++i) { - SyncPingReply* pSyncMsg = syncPingReplyBuild2(&pSyncNode->myRaftId, &pSyncNode->myRaftId, "syncEnqTest"); + SyncPingReply* pSyncMsg = syncPingReplyBuild2(&pSyncNode->myRaftId, &pSyncNode->myRaftId, 1000, "syncEnqTest"); SRpcMsg rpcMsg; syncPingReply2RpcMsg(pSyncMsg, &rpcMsg); pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); diff --git a/source/libs/sync/test/syncEntryTest.cpp b/source/libs/sync/test/syncEntryTest.cpp index e8427e8168..53ae91fc60 100644 --- a/source/libs/sync/test/syncEntryTest.cpp +++ b/source/libs/sync/test/syncEntryTest.cpp @@ -52,7 +52,7 @@ void test3() { pSyncMsg->isWeak = 1; strcpy(pSyncMsg->data, "test3"); - SSyncRaftEntry* pEntry = syncEntryBuild3(pSyncMsg, 100, 200, SYNC_RAFT_ENTRY_NOOP); + SSyncRaftEntry* pEntry = syncEntryBuild3(pSyncMsg, 100, 200); syncEntryPrint(pEntry); syncClientRequestDestroy(pSyncMsg); @@ -68,7 +68,6 @@ void test4() { pEntry->isWeak = true; pEntry->term = 44; pEntry->index = 55; - pEntry->entryType = SYNC_RAFT_ENTRY_CONFIG; strcpy(pEntry->data, "test4"); syncEntryPrint(pEntry); @@ -84,9 +83,8 @@ void test4() { } int main(int argc, char** argv) { - // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; - sDebugFlag = 143 + 64; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; test1(); test2(); diff --git a/source/libs/sync/test/syncIOClientTest.cpp b/source/libs/sync/test/syncIOClientTest.cpp index dffa8b5cb9..c146ff4e92 100644 --- a/source/libs/sync/test/syncIOClientTest.cpp +++ b/source/libs/sync/test/syncIOClientTest.cpp @@ -38,7 +38,7 @@ int main() { destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); destId.vgId = 100; - SyncPingReply* pSyncMsg = syncPingReplyBuild2(&srcId, &destId, "syncIOClientTest"); + SyncPingReply* pSyncMsg = syncPingReplyBuild2(&srcId, &destId, 1000, "syncIOClientTest"); SRpcMsg rpcMsg; syncPingReply2RpcMsg(pSyncMsg, &rpcMsg); diff --git a/source/libs/sync/test/syncIOSendMsgTest.cpp b/source/libs/sync/test/syncIOSendMsgTest.cpp index 913e57e59a..03d308ea28 100644 --- a/source/libs/sync/test/syncIOSendMsgTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgTest.cpp @@ -89,15 +89,16 @@ int main(int argc, char** argv) { SSyncNode* pSyncNode = syncInitTest(); assert(pSyncNode != NULL); - syncNodePrint2((char*)"syncInitTest", pSyncNode); + syncNodeLog2((char*)"syncInitTest", pSyncNode); initRaftId(pSyncNode); //-------------------------------------------------------------- for (int i = 0; i < 10; ++i) { - SyncPingReply* pSyncMsg = syncPingReplyBuild2(&pSyncNode->myRaftId, &pSyncNode->myRaftId, "syncIOSendMsgTest"); - SRpcMsg rpcMsg; + SyncPingReply* pSyncMsg = + syncPingReplyBuild2(&pSyncNode->myRaftId, &pSyncNode->myRaftId, 1000, "syncIOSendMsgTest"); + SRpcMsg rpcMsg; syncPingReply2RpcMsg(pSyncMsg, &rpcMsg); SEpSet epSet; diff --git a/source/libs/sync/test/syncIndexMgrTest.cpp b/source/libs/sync/test/syncIndexMgrTest.cpp index 319ea3e15a..ea5d5f6b6f 100644 --- a/source/libs/sync/test/syncIndexMgrTest.cpp +++ b/source/libs/sync/test/syncIndexMgrTest.cpp @@ -73,9 +73,8 @@ void initRaftId(SSyncNode* pSyncNode) { } int main(int argc, char** argv) { - // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; - sDebugFlag = 143 + 64; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; myIndex = 0; if (argc >= 2) { diff --git a/source/libs/sync/test/syncInitTest.cpp b/source/libs/sync/test/syncInitTest.cpp index 48b5488e41..ca0657c74d 100644 --- a/source/libs/sync/test/syncInitTest.cpp +++ b/source/libs/sync/test/syncInitTest.cpp @@ -30,7 +30,7 @@ SSyncNode* syncNodeInit() { syncInfo.queue = gSyncIO->pMsgQ; syncInfo.FpEqMsg = syncIOEqMsg; syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./"); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./sync_init_test"); SSyncCfg* pCfg = &syncInfo.syncCfg; pCfg->myIndex = myIndex; @@ -64,15 +64,15 @@ void initRaftId(SSyncNode* pSyncNode) { for (int i = 0; i < replicaNum; ++i) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); - printf("raftId[%d] : %s\n", i, s); + sTrace("raftId[%d] : %s\n", i, s); taosMemoryFree(s); } } int main(int argc, char** argv) { - // taosInitLog((char *)"syncTest.log", 100000, 10); + // taosInitLog((char *)"tmp/syncInitTest.log", 100); tsAsyncLog = 0; - sDebugFlag = 143 + 64; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; myIndex = 0; if (argc >= 2) { @@ -88,11 +88,13 @@ int main(int argc, char** argv) { SSyncNode* pSyncNode = syncInitTest(); assert(pSyncNode != NULL); - syncNodePrint2((char*)"syncInitTest", pSyncNode); - + syncNodeLog2((char*)"syncInitTest", pSyncNode); initRaftId(pSyncNode); - //-------------------------------------------------------------- + syncNodeClose(pSyncNode); + syncEnvStop(); + // syncIOStop(); + // taosCloseLog(); return 0; } \ No newline at end of file diff --git a/source/libs/sync/test/syncLogStoreCheck.cpp b/source/libs/sync/test/syncLogStoreCheck.cpp index 85e39a61c5..b77a744c30 100644 --- a/source/libs/sync/test/syncLogStoreCheck.cpp +++ b/source/libs/sync/test/syncLogStoreCheck.cpp @@ -99,7 +99,7 @@ int main(int argc, char** argv) { pSyncNode = logStoreCheck(argv[1]); assert(pSyncNode != NULL); - logStorePrint2((char*)"logStoreCheck", pSyncNode->pLogStore); + logStoreLog2((char*)"logStoreCheck", pSyncNode->pLogStore); return 0; } diff --git a/source/libs/sync/test/syncLogStoreTest.cpp b/source/libs/sync/test/syncLogStoreTest.cpp index a5adba9a88..c4a5f6cae4 100644 --- a/source/libs/sync/test/syncLogStoreTest.cpp +++ b/source/libs/sync/test/syncLogStoreTest.cpp @@ -16,74 +16,44 @@ void logTest() { sFatal("--- sync log test: fatal"); } -uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; -int32_t replicaNum = 1; -int32_t myIndex = 0; +SSyncNode* pSyncNode; +SWal* pWal; +SSyncLogStore* pLogStore; +const char* pWalPath = "./syncLogStoreTest_wal"; -SRaftId ids[TSDB_MAX_REPLICA]; -SSyncInfo syncInfo; -SSyncFSM* pFsm; -SWal* pWal; -SSyncNode* pSyncNode; +void init() { + walInit(); + taosRemoveDir(pWalPath); -SSyncNode* syncNodeInit() { - syncInfo.vgId = 1234; - syncInfo.rpcClient = gSyncIO->clientRpc; - syncInfo.FpSendMsg = syncIOSendMsg; - syncInfo.queue = gSyncIO->pMsgQ; - syncInfo.FpEqMsg = syncIOEqMsg; - syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./"); - - int code = walInit(); - assert(code == 0); SWalCfg walCfg; memset(&walCfg, 0, sizeof(SWalCfg)); - walCfg.vgId = syncInfo.vgId; + walCfg.vgId = 1000; walCfg.fsyncPeriod = 1000; walCfg.retentionPeriod = 1000; walCfg.rollPeriod = 1000; walCfg.retentionSize = 1000; walCfg.segSize = 1000; walCfg.level = TAOS_WAL_FSYNC; - pWal = walOpen("./wal_test", &walCfg); + pWal = walOpen(pWalPath, &walCfg); assert(pWal != NULL); - syncInfo.pWal = pWal; - - SSyncCfg* pCfg = &syncInfo.syncCfg; - pCfg->myIndex = myIndex; - pCfg->replicaNum = replicaNum; - - for (int i = 0; i < replicaNum; ++i) { - pCfg->nodeInfo[i].nodePort = ports[i]; - snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); - } - - pSyncNode = syncNodeOpen(&syncInfo); - assert(pSyncNode != NULL); - - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; - gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; - gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; - gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; - gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; - gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; - gSyncIO->pSyncNode = pSyncNode; - - return pSyncNode; + pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); + memset(pSyncNode, 0, sizeof(SSyncNode)); + pSyncNode->pWal = pWal; } -SSyncNode* syncInitTest() { return syncNodeInit(); } +void cleanup() { + walClose(pWal); + walCleanUp(); + taosMemoryFree(pSyncNode); +} void logStoreTest() { - logStorePrint2((char*)"logStoreTest", pSyncNode->pLogStore); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + assert(pLogStore->getLastIndex(pLogStore) == SYNC_INDEX_INVALID); - assert(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) == SYNC_INDEX_INVALID); + logStoreLog2((char*)"logStoreTest", pLogStore); for (int i = 0; i < 5; ++i) { int32_t dataLen = 10; @@ -93,58 +63,35 @@ void logStoreTest() { pEntry->originalRpcType = 2; pEntry->seqNum = 3; pEntry->isWeak = true; - pEntry->term = 100; - pEntry->index = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) + 1; + pEntry->term = 100 + i; + pEntry->index = pLogStore->getLastIndex(pLogStore) + 1; snprintf(pEntry->data, dataLen, "value%d", i); - // syncEntryPrint2((char*)"write entry:", pEntry); - pSyncNode->pLogStore->appendEntry(pSyncNode->pLogStore, pEntry); + syncEntryLog2((char*)"==write entry== :", pEntry); + pLogStore->appendEntry(pLogStore, pEntry); syncEntryDestory(pEntry); if (i == 0) { - assert(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) == SYNC_INDEX_BEGIN); + assert(pLogStore->getLastIndex(pLogStore) == SYNC_INDEX_BEGIN); } } - logStorePrint2((char*)"after appendEntry", pSyncNode->pLogStore); + logStoreLog2((char*)"after appendEntry", pLogStore); - pSyncNode->pLogStore->truncate(pSyncNode->pLogStore, 3); - logStorePrint2((char*)"after truncate 3", pSyncNode->pLogStore); -} + pLogStore->truncate(pLogStore, 3); + logStoreLog2((char*)"after truncate 3", pLogStore); -void initRaftId(SSyncNode* pSyncNode) { - for (int i = 0; i < replicaNum; ++i) { - ids[i] = pSyncNode->replicasId[i]; - char* s = syncUtilRaftId2Str(&ids[i]); - printf("raftId[%d] : %s\n", i, s); - taosMemoryFree(s); - } + logStoreDestory(pLogStore); } int main(int argc, char** argv) { - // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; - sDebugFlag = 143 + 64; - - myIndex = 0; - if (argc >= 2) { - myIndex = atoi(argv[1]); - } - - int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); - assert(ret == 0); - - ret = syncEnvStart(); - assert(ret == 0); - - taosRemoveDir("./wal_test"); - - pSyncNode = syncInitTest(); - assert(pSyncNode != NULL); - - // syncNodePrint((char*)"syncLogStoreTest", pSyncNode); - // initRaftId(pSyncNode); + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + init(); logStoreTest(); + taosMsleep(2000); + cleanup(); + return 0; } diff --git a/source/libs/sync/test/syncPingReplyTest.cpp b/source/libs/sync/test/syncPingReplyTest.cpp index 40592eab6f..2bd8671e6c 100644 --- a/source/libs/sync/test/syncPingReplyTest.cpp +++ b/source/libs/sync/test/syncPingReplyTest.cpp @@ -20,13 +20,13 @@ SyncPingReply *createMsg() { srcId.vgId = 100; destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); destId.vgId = 100; - SyncPingReply *pMsg = syncPingReplyBuild3(&srcId, &destId); + SyncPingReply *pMsg = syncPingReplyBuild3(&srcId, &destId, 1000); return pMsg; } void test1() { SyncPingReply *pMsg = createMsg(); - syncPingReplyPrint2((char *)"test1:", pMsg); + syncPingReplyLog2((char *)"test1:", pMsg); syncPingReplyDestroy(pMsg); } @@ -37,7 +37,7 @@ void test2() { syncPingReplySerialize(pMsg, serialized, len); SyncPingReply *pMsg2 = syncPingReplyBuild(pMsg->dataLen); syncPingReplyDeserialize(serialized, len, pMsg2); - syncPingReplyPrint2((char *)"test2: syncPingReplySerialize -> syncPingReplyDeserialize ", pMsg2); + syncPingReplyLog2((char *)"test2: syncPingReplySerialize -> syncPingReplyDeserialize ", pMsg2); taosMemoryFree(serialized); syncPingReplyDestroy(pMsg); @@ -49,7 +49,7 @@ void test3() { uint32_t len; char * serialized = syncPingReplySerialize2(pMsg, &len); SyncPingReply *pMsg2 = syncPingReplyDeserialize2(serialized, len); - syncPingReplyPrint2((char *)"test3: syncPingReplySerialize3 -> syncPingReplyDeserialize2 ", pMsg2); + syncPingReplyLog2((char *)"test3: syncPingReplySerialize2 -> syncPingReplyDeserialize2 ", pMsg2); taosMemoryFree(serialized); syncPingReplyDestroy(pMsg); @@ -62,8 +62,9 @@ void test4() { syncPingReply2RpcMsg(pMsg, &rpcMsg); SyncPingReply *pMsg2 = (SyncPingReply *)taosMemoryMalloc(rpcMsg.contLen); syncPingReplyFromRpcMsg(&rpcMsg, pMsg2); - syncPingReplyPrint2((char *)"test4: syncPingReply2RpcMsg -> syncPingReplyFromRpcMsg ", pMsg2); + syncPingReplyLog2((char *)"test4: syncPingReply2RpcMsg -> syncPingReplyFromRpcMsg ", pMsg2); + rpcFreeCont(rpcMsg.pCont); syncPingReplyDestroy(pMsg); syncPingReplyDestroy(pMsg2); } @@ -73,16 +74,30 @@ void test5() { SRpcMsg rpcMsg; syncPingReply2RpcMsg(pMsg, &rpcMsg); SyncPingReply *pMsg2 = syncPingReplyFromRpcMsg2(&rpcMsg); - syncPingReplyPrint2((char *)"test5: syncPingReply2RpcMsg -> syncPingReplyFromRpcMsg2 ", pMsg2); + syncPingReplyLog2((char *)"test5: syncPingReply2RpcMsg -> syncPingReplyFromRpcMsg2 ", pMsg2); + rpcFreeCont(rpcMsg.pCont); + syncPingReplyDestroy(pMsg); + syncPingReplyDestroy(pMsg2); +} + +void test6() { + SyncPingReply *pMsg = createMsg(); + int32_t bufLen = syncPingReplySerialize3(pMsg, NULL, 0); + char * serialized = (char *)taosMemoryMalloc(bufLen); + syncPingReplySerialize3(pMsg, serialized, bufLen); + SyncPingReply *pMsg2 = syncPingReplyDeserialize3(serialized, bufLen); + assert(pMsg2 != NULL); + syncPingReplyLog2((char *)"test6: syncPingReplySerialize3 -> syncPingReplyDeserialize3 ", pMsg2); + + taosMemoryFree(serialized); syncPingReplyDestroy(pMsg); syncPingReplyDestroy(pMsg2); } int main() { - // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; - sDebugFlag = 143 + 64; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; logTest(); test1(); @@ -90,6 +105,7 @@ int main() { test3(); test4(); test5(); + test6(); return 0; } diff --git a/source/libs/sync/test/syncPingSelfTest.cpp b/source/libs/sync/test/syncPingSelfTest.cpp index 677518c6ac..641ff059be 100644 --- a/source/libs/sync/test/syncPingSelfTest.cpp +++ b/source/libs/sync/test/syncPingSelfTest.cpp @@ -87,7 +87,7 @@ int main(int argc, char** argv) { SSyncNode* pSyncNode = syncInitTest(); assert(pSyncNode != NULL); - syncNodePrint2((char*)"", pSyncNode); + syncNodeLog2((char*)"", pSyncNode); initRaftId(pSyncNode); diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index eb774d77c3..a02f5e4b43 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -20,13 +20,13 @@ SyncPing *createMsg() { srcId.vgId = 100; destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); destId.vgId = 100; - SyncPing *pMsg = syncPingBuild3(&srcId, &destId); + SyncPing *pMsg = syncPingBuild3(&srcId, &destId, 1000); return pMsg; } void test1() { SyncPing *pMsg = createMsg(); - syncPingPrint2((char *)"test1:", pMsg); + syncPingLog2((char *)"test1:", pMsg); syncPingDestroy(pMsg); } @@ -37,7 +37,7 @@ void test2() { syncPingSerialize(pMsg, serialized, len); SyncPing *pMsg2 = syncPingBuild(pMsg->dataLen); syncPingDeserialize(serialized, len, pMsg2); - syncPingPrint2((char *)"test2: syncPingSerialize -> syncPingDeserialize ", pMsg2); + syncPingLog2((char *)"test2: syncPingSerialize -> syncPingDeserialize ", pMsg2); taosMemoryFree(serialized); syncPingDestroy(pMsg); @@ -49,7 +49,7 @@ void test3() { uint32_t len; char * serialized = syncPingSerialize2(pMsg, &len); SyncPing *pMsg2 = syncPingDeserialize2(serialized, len); - syncPingPrint2((char *)"test3: syncPingSerialize3 -> syncPingDeserialize2 ", pMsg2); + syncPingLog2((char *)"test3: syncPingSerialize2 -> syncPingDeserialize2 ", pMsg2); taosMemoryFree(serialized); syncPingDestroy(pMsg); @@ -62,10 +62,11 @@ void test4() { syncPing2RpcMsg(pMsg, &rpcMsg); SyncPing *pMsg2 = (SyncPing *)taosMemoryMalloc(rpcMsg.contLen); syncPingFromRpcMsg(&rpcMsg, pMsg2); - syncPingPrint2((char *)"test4: syncPing2RpcMsg -> syncPingFromRpcMsg ", pMsg2); + syncPingLog2((char *)"test4: syncPing2RpcMsg -> syncPingFromRpcMsg ", pMsg2); syncPingDestroy(pMsg); syncPingDestroy(pMsg2); + rpcFreeCont(rpcMsg.pCont); } void test5() { @@ -73,16 +74,30 @@ void test5() { SRpcMsg rpcMsg; syncPing2RpcMsg(pMsg, &rpcMsg); SyncPing *pMsg2 = syncPingFromRpcMsg2(&rpcMsg); - syncPingPrint2((char *)"test5: syncPing2RpcMsg -> syncPingFromRpcMsg2 ", pMsg2); + syncPingLog2((char *)"test5: syncPing2RpcMsg -> syncPingFromRpcMsg2 ", pMsg2); syncPingDestroy(pMsg); syncPingDestroy(pMsg2); + rpcFreeCont(rpcMsg.pCont); +} + +void test6() { + SyncPing *pMsg = createMsg(); + int32_t bufLen = syncPingSerialize3(pMsg, NULL, 0); + char * serialized = (char *)taosMemoryMalloc(bufLen); + syncPingSerialize3(pMsg, serialized, bufLen); + SyncPing *pMsg2 = syncPingDeserialize3(serialized, bufLen); + assert(pMsg2 != NULL); + syncPingLog2((char *)"test6: syncPingSerialize3 -> syncPingDeserialize3 ", pMsg2); + + taosMemoryFree(serialized); + syncPingDestroy(pMsg); + syncPingDestroy(pMsg2); } int main() { - // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; - sDebugFlag = 143 + 64; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; logTest(); test1(); @@ -90,6 +105,7 @@ int main() { test3(); test4(); test5(); + test6(); return 0; } diff --git a/source/libs/sync/test/syncPingTimerTest.cpp b/source/libs/sync/test/syncPingTimerTest.cpp index 3c5e76ca14..29e99435be 100644 --- a/source/libs/sync/test/syncPingTimerTest.cpp +++ b/source/libs/sync/test/syncPingTimerTest.cpp @@ -87,7 +87,7 @@ int main(int argc, char** argv) { SSyncNode* pSyncNode = syncInitTest(); assert(pSyncNode != NULL); - syncNodePrint2((char*)"----1", pSyncNode); + syncNodeLog2((char*)"----1", pSyncNode); initRaftId(pSyncNode); @@ -96,7 +96,7 @@ int main(int argc, char** argv) { sTrace("syncNodeStartPingTimer ..."); ret = syncNodeStartPingTimer(pSyncNode); assert(ret == 0); - syncNodePrint2((char*)"----2", pSyncNode); + syncNodeLog2((char*)"----2", pSyncNode); sTrace("sleep ..."); taosMsleep(10000); @@ -104,7 +104,7 @@ int main(int argc, char** argv) { sTrace("syncNodeStopPingTimer ..."); ret = syncNodeStopPingTimer(pSyncNode); assert(ret == 0); - syncNodePrint2((char*)"----3", pSyncNode); + syncNodeLog2((char*)"----3", pSyncNode); sTrace("sleep ..."); taosMsleep(5000); @@ -112,7 +112,7 @@ int main(int argc, char** argv) { sTrace("syncNodeStartPingTimer ..."); ret = syncNodeStartPingTimer(pSyncNode); assert(ret == 0); - syncNodePrint2((char*)"----4", pSyncNode); + syncNodeLog2((char*)"----4", pSyncNode); sTrace("sleep ..."); taosMsleep(10000); @@ -120,7 +120,7 @@ int main(int argc, char** argv) { sTrace("syncNodeStopPingTimer ..."); ret = syncNodeStopPingTimer(pSyncNode); assert(ret == 0); - syncNodePrint2((char*)"----5", pSyncNode); + syncNodeLog2((char*)"----5", pSyncNode); while (1) { sTrace("while 1 sleep ..."); diff --git a/source/libs/sync/test/syncPingTimerTest2.cpp b/source/libs/sync/test/syncPingTimerTest2.cpp index 554f67d365..285828125d 100644 --- a/source/libs/sync/test/syncPingTimerTest2.cpp +++ b/source/libs/sync/test/syncPingTimerTest2.cpp @@ -87,7 +87,7 @@ int main(int argc, char** argv) { SSyncNode* pSyncNode = syncInitTest(); assert(pSyncNode != NULL); - syncNodePrint2((char*)"", pSyncNode); + syncNodeLog2((char*)"", pSyncNode); initRaftId(pSyncNode); diff --git a/source/libs/sync/test/syncRaftCfgTest.cpp b/source/libs/sync/test/syncRaftCfgTest.cpp new file mode 100644 index 0000000000..d3c06fa83e --- /dev/null +++ b/source/libs/sync/test/syncRaftCfgTest.cpp @@ -0,0 +1,101 @@ +#include "syncRaftStore.h" +//#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftCfg.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SSyncCfg* createSyncCfg() { + SSyncCfg* pCfg = (SSyncCfg*)taosMemoryMalloc(sizeof(SSyncCfg)); + memset(pCfg, 0, sizeof(SSyncCfg)); + + pCfg->replicaNum = 3; + pCfg->myIndex = 1; + for (int i = 0; i < pCfg->replicaNum; ++i) { + ((pCfg->nodeInfo)[i]).nodePort = i * 100; + snprintf(((pCfg->nodeInfo)[i]).nodeFqdn, sizeof(((pCfg->nodeInfo)[i]).nodeFqdn), "100.200.300.%d", i); + } + + return pCfg; +} + +void test1() { + SSyncCfg* pCfg = createSyncCfg(); + syncCfgLog2((char*)__FUNCTION__, pCfg); + taosMemoryFree(pCfg); +} + +void test2() { + SSyncCfg* pCfg = createSyncCfg(); + char* s = syncCfg2Str(pCfg); + + SSyncCfg* pCfg2 = (SSyncCfg*)taosMemoryMalloc(sizeof(SSyncCfg)); + int32_t ret = syncCfgFromStr(s, pCfg2); + assert(ret == 0); + syncCfgLog2((char*)__FUNCTION__, pCfg2); + + taosMemoryFree(pCfg); + taosMemoryFree(s); + taosMemoryFree(pCfg2); +} + +void test3() { + SSyncCfg* pCfg = createSyncCfg(); + char* s = (char*)"./test3_raft_cfg.json"; + + if (taosCheckExistFile(s)) { + printf("%s file: %s already exist! \n", (char*)__FUNCTION__, s); + } else { + syncCfgCreateFile(pCfg, s); + printf("%s create json file: %s \n", (char*)__FUNCTION__, s); + } + + taosMemoryFree(pCfg); +} + +void test4() { + SRaftCfg* pCfg = raftCfgOpen("./test3_raft_cfg.json"); + assert(pCfg != NULL); + + raftCfgLog2((char*)__FUNCTION__, pCfg); + + int32_t ret = raftCfgClose(pCfg); + assert(ret == 0); +} + +void test5() { + SRaftCfg* pCfg = raftCfgOpen("./test3_raft_cfg.json"); + assert(pCfg != NULL); + + pCfg->cfg.myIndex = taosGetTimestampSec(); + raftCfgPersist(pCfg); + + printf("%s update json file: %s myIndex->%d \n", (char*)__FUNCTION__, "./test3_raft_cfg.json", pCfg->cfg.myIndex); + + int32_t ret = raftCfgClose(pCfg); + assert(ret == 0); +} + +int main() { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + + logTest(); + test1(); + test2(); + test3(); + test4(); + test5(); + + return 0; +} diff --git a/source/libs/sync/test/syncRaftStoreTest.cpp b/source/libs/sync/test/syncRaftStoreTest.cpp index 460ff90f4f..070f8c0f07 100644 --- a/source/libs/sync/test/syncRaftStoreTest.cpp +++ b/source/libs/sync/test/syncRaftStoreTest.cpp @@ -30,9 +30,8 @@ void initRaftId() { } int main() { - // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; - sDebugFlag = 143 + 64; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; logTest(); @@ -40,31 +39,33 @@ int main() { SRaftStore* pRaftStore = raftStoreOpen("./test_raft_store.json"); assert(pRaftStore != NULL); - raftStorePrint2((char*)"==raftStoreOpen==", pRaftStore); + raftStoreLog2((char*)"==raftStoreOpen==", pRaftStore); raftStoreSetTerm(pRaftStore, 100); - raftStorePrint2((char*)"==raftStoreSetTerm==", pRaftStore); + raftStoreLog2((char*)"==raftStoreSetTerm==", pRaftStore); raftStoreVote(pRaftStore, &ids[0]); - raftStorePrint2((char*)"==raftStoreVote==", pRaftStore); + raftStoreLog2((char*)"==raftStoreVote==", pRaftStore); raftStoreClearVote(pRaftStore); - raftStorePrint2((char*)"==raftStoreClearVote==", pRaftStore); + raftStoreLog2((char*)"==raftStoreClearVote==", pRaftStore); raftStoreVote(pRaftStore, &ids[1]); - raftStorePrint2((char*)"==raftStoreVote==", pRaftStore); + raftStoreLog2((char*)"==raftStoreVote==", pRaftStore); raftStoreNextTerm(pRaftStore); - raftStorePrint2((char*)"==raftStoreNextTerm==", pRaftStore); + raftStoreLog2((char*)"==raftStoreNextTerm==", pRaftStore); raftStoreNextTerm(pRaftStore); - raftStorePrint2((char*)"==raftStoreNextTerm==", pRaftStore); + raftStoreLog2((char*)"==raftStoreNextTerm==", pRaftStore); raftStoreNextTerm(pRaftStore); - raftStorePrint2((char*)"==raftStoreNextTerm==", pRaftStore); + raftStoreLog2((char*)"==raftStoreNextTerm==", pRaftStore); raftStoreNextTerm(pRaftStore); - raftStorePrint2((char*)"==raftStoreNextTerm==", pRaftStore); + raftStoreLog2((char*)"==raftStoreNextTerm==", pRaftStore); + + raftStoreClose(pRaftStore); return 0; } diff --git a/source/libs/sync/test/syncReplicateLoadTest.cpp b/source/libs/sync/test/syncReplicateLoadTest.cpp index 0841083b4a..f58c372445 100644 --- a/source/libs/sync/test/syncReplicateLoadTest.cpp +++ b/source/libs/sync/test/syncReplicateLoadTest.cpp @@ -27,29 +27,26 @@ SSyncInfo syncInfo; SSyncFSM *pFsm; SWal * pWal; -void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, - ESyncState state) { +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", - pFsm, index, isWeak, code, state, syncUtilState2String(state)); - syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } -void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, - ESyncState state) { +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), - "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, index, isWeak, - code, state, syncUtilState2String(state)); - syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, + cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } -void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, - ESyncState state) { +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", - pFsm, index, isWeak, code, state, syncUtilState2String(state)); - syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } void initFsm() { @@ -97,8 +94,9 @@ int64_t syncNodeInit() { // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); } - int64_t rid = syncStart(&syncInfo); + int64_t rid = syncOpen(&syncInfo); assert(rid > 0); + syncStart(rid); SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); assert(pSyncNode != NULL); @@ -141,7 +139,7 @@ SRpcMsg *step0(int i) { } SyncClientRequest *step1(const SRpcMsg *pMsg) { - SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true); + SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true, 1000); return pRetMsg; } @@ -170,7 +168,7 @@ int main(int argc, char **argv) { SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); assert(pSyncNode != NULL); - syncNodePrint2((char *)"", pSyncNode); + syncNodeLog2((char *)"", pSyncNode); initRaftId(pSyncNode); // only load ... diff --git a/source/libs/sync/test/syncReplicateTest.cpp b/source/libs/sync/test/syncReplicateTest.cpp index 9783bac7e5..9b5aaf5c18 100644 --- a/source/libs/sync/test/syncReplicateTest.cpp +++ b/source/libs/sync/test/syncReplicateTest.cpp @@ -28,29 +28,26 @@ SSyncFSM * pFsm; SWal * pWal; SSyncNode *gSyncNode; -void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, - ESyncState state) { +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", - pFsm, index, isWeak, code, state, syncUtilState2String(state)); - syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } -void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, - ESyncState state) { +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), - "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, index, isWeak, - code, state, syncUtilState2String(state)); - syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, + cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } -void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, - ESyncState state) { +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", - pFsm, index, isWeak, code, state, syncUtilState2String(state)); - syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } void initFsm() { @@ -136,7 +133,7 @@ SRpcMsg *step0(int i) { } SyncClientRequest *step1(const SRpcMsg *pMsg) { - SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true); + SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true, 1000); return pRetMsg; } @@ -163,18 +160,18 @@ int main(int argc, char **argv) { gSyncNode = syncInitTest(); assert(gSyncNode != NULL); - syncNodePrint2((char *)"", gSyncNode); + syncNodeLog2((char *)"", gSyncNode); initRaftId(gSyncNode); for (int i = 0; i < 30; ++i) { // step0 SRpcMsg *pMsg0 = step0(i); - syncRpcMsgPrint2((char *)"==step0==", pMsg0); + syncRpcMsgLog2((char *)"==step0==", pMsg0); // step1 SyncClientRequest *pMsg1 = step1(pMsg0); - syncClientRequestPrint2((char *)"==step1==", pMsg1); + syncClientRequestLog2((char *)"==step1==", pMsg1); SyncClientRequest *pSyncClientRequest = pMsg1; SRpcMsg rpcMsg; diff --git a/source/libs/sync/test/syncReplicateTest2.cpp b/source/libs/sync/test/syncReplicateTest2.cpp index 430df7eebd..1e2f823be5 100644 --- a/source/libs/sync/test/syncReplicateTest2.cpp +++ b/source/libs/sync/test/syncReplicateTest2.cpp @@ -27,29 +27,26 @@ SSyncInfo syncInfo; SSyncFSM *pFsm; SWal * pWal; -void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, - ESyncState state) { +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", - pFsm, index, isWeak, code, state, syncUtilState2String(state)); - syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } -void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, - ESyncState state) { +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), - "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, index, isWeak, - code, state, syncUtilState2String(state)); - syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, + cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } -void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, - ESyncState state) { +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", - pFsm, index, isWeak, code, state, syncUtilState2String(state)); - syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } void initFsm() { @@ -97,8 +94,9 @@ int64_t syncNodeInit() { // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); } - int64_t rid = syncStart(&syncInfo); + int64_t rid = syncOpen(&syncInfo); assert(rid > 0); + syncStart(rid); SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); assert(pSyncNode != NULL); @@ -141,7 +139,7 @@ SRpcMsg *step0(int i) { } SyncClientRequest *step1(const SRpcMsg *pMsg) { - SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true); + SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true, 1000); return pRetMsg; } @@ -170,13 +168,13 @@ int main(int argc, char **argv) { SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); assert(pSyncNode != NULL); - syncNodePrint2((char *)"", pSyncNode); + syncNodeLog2((char *)"", pSyncNode); initRaftId(pSyncNode); for (int i = 0; i < 30; ++i) { // step0 SRpcMsg *pMsg0 = step0(i); - syncRpcMsgPrint2((char *)"==step0==", pMsg0); + syncRpcMsgLog2((char *)"==step0==", pMsg0); syncPropose(rid, pMsg0, true); taosMsleep(1000); diff --git a/source/libs/sync/test/syncReplicateTest3.cpp b/source/libs/sync/test/syncReplicateTest3.cpp index 792b9c94cf..e8e24b507d 100644 --- a/source/libs/sync/test/syncReplicateTest3.cpp +++ b/source/libs/sync/test/syncReplicateTest3.cpp @@ -29,29 +29,26 @@ SSyncInfo syncInfo; SSyncFSM *pFsm; SWal * pWal; -void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, - ESyncState state) { +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", - pFsm, index, isWeak, code, state, syncUtilState2String(state)); - syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } -void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, - ESyncState state) { +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), - "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, index, isWeak, - code, state, syncUtilState2String(state)); - syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, + cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } -void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, - ESyncState state) { +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", - pFsm, index, isWeak, code, state, syncUtilState2String(state)); - syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } void initFsm() { @@ -99,8 +96,9 @@ int64_t syncNodeInit() { // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); } - int64_t rid = syncStart(&syncInfo); + int64_t rid = syncOpen(&syncInfo); assert(rid > 0); + syncStart(rid); SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); assert(pSyncNode != NULL); @@ -143,7 +141,7 @@ SRpcMsg *step0(int i) { } SyncClientRequest *step1(const SRpcMsg *pMsg) { - SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true); + SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true, 1000); return pRetMsg; } @@ -182,13 +180,13 @@ int main(int argc, char **argv) { SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); assert(pSyncNode != NULL); - syncNodePrint2((char *)"", pSyncNode); + syncNodeLog2((char *)"", pSyncNode); initRaftId(pSyncNode); for (int i = 0; i < recordCount; ++i) { // step0 SRpcMsg *pMsg0 = step0(i); - syncRpcMsgPrint2((char *)"==step0==", pMsg0); + syncRpcMsgLog2((char *)"==step0==", pMsg0); syncPropose(rid, pMsg0, true); taosMsleep(sleepMS); diff --git a/source/libs/sync/test/syncRequestVoteReplyTest.cpp b/source/libs/sync/test/syncRequestVoteReplyTest.cpp index 9e0ebee313..d9fe6fa011 100644 --- a/source/libs/sync/test/syncRequestVoteReplyTest.cpp +++ b/source/libs/sync/test/syncRequestVoteReplyTest.cpp @@ -15,7 +15,7 @@ void logTest() { } SyncRequestVoteReply *createMsg() { - SyncRequestVoteReply *pMsg = syncRequestVoteReplyBuild(); + SyncRequestVoteReply *pMsg = syncRequestVoteReplyBuild(1000); pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); pMsg->srcId.vgId = 100; pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); @@ -27,7 +27,7 @@ SyncRequestVoteReply *createMsg() { void test1() { SyncRequestVoteReply *pMsg = createMsg(); - syncRequestVoteReplyPrint2((char *)"test1:", pMsg); + syncRequestVoteReplyLog2((char *)"test1:", pMsg); syncRequestVoteReplyDestroy(pMsg); } @@ -36,9 +36,9 @@ void test2() { uint32_t len = pMsg->bytes; char * serialized = (char *)taosMemoryMalloc(len); syncRequestVoteReplySerialize(pMsg, serialized, len); - SyncRequestVoteReply *pMsg2 = syncRequestVoteReplyBuild(); + SyncRequestVoteReply *pMsg2 = syncRequestVoteReplyBuild(1000); syncRequestVoteReplyDeserialize(serialized, len, pMsg2); - syncRequestVoteReplyPrint2((char *)"test2: syncRequestVoteReplySerialize -> syncRequestVoteReplyDeserialize ", pMsg2); + syncRequestVoteReplyLog2((char *)"test2: syncRequestVoteReplySerialize -> syncRequestVoteReplyDeserialize ", pMsg2); taosMemoryFree(serialized); syncRequestVoteReplyDestroy(pMsg); @@ -50,8 +50,7 @@ void test3() { uint32_t len; char * serialized = syncRequestVoteReplySerialize2(pMsg, &len); SyncRequestVoteReply *pMsg2 = syncRequestVoteReplyDeserialize2(serialized, len); - syncRequestVoteReplyPrint2((char *)"test3: syncRequestVoteReplySerialize3 -> syncRequestVoteReplyDeserialize2 ", - pMsg2); + syncRequestVoteReplyLog2((char *)"test3: syncRequestVoteReplySerialize3 -> syncRequestVoteReplyDeserialize2 ", pMsg2); taosMemoryFree(serialized); syncRequestVoteReplyDestroy(pMsg); @@ -62,10 +61,11 @@ void test4() { SyncRequestVoteReply *pMsg = createMsg(); SRpcMsg rpcMsg; syncRequestVoteReply2RpcMsg(pMsg, &rpcMsg); - SyncRequestVoteReply *pMsg2 = syncRequestVoteReplyBuild(); + SyncRequestVoteReply *pMsg2 = syncRequestVoteReplyBuild(1000); syncRequestVoteReplyFromRpcMsg(&rpcMsg, pMsg2); - syncRequestVoteReplyPrint2((char *)"test4: syncRequestVoteReply2RpcMsg -> syncRequestVoteReplyFromRpcMsg ", pMsg2); + syncRequestVoteReplyLog2((char *)"test4: syncRequestVoteReply2RpcMsg -> syncRequestVoteReplyFromRpcMsg ", pMsg2); + rpcFreeCont(rpcMsg.pCont); syncRequestVoteReplyDestroy(pMsg); syncRequestVoteReplyDestroy(pMsg2); } @@ -75,16 +75,16 @@ void test5() { SRpcMsg rpcMsg; syncRequestVoteReply2RpcMsg(pMsg, &rpcMsg); SyncRequestVoteReply *pMsg2 = syncRequestVoteReplyFromRpcMsg2(&rpcMsg); - syncRequestVoteReplyPrint2((char *)"test5: syncRequestVoteReply2RpcMsg -> syncRequestVoteReplyFromRpcMsg2 ", pMsg2); + syncRequestVoteReplyLog2((char *)"test5: syncRequestVoteReply2RpcMsg -> syncRequestVoteReplyFromRpcMsg2 ", pMsg2); + rpcFreeCont(rpcMsg.pCont); syncRequestVoteReplyDestroy(pMsg); syncRequestVoteReplyDestroy(pMsg2); } int main() { - // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; - sDebugFlag = 143 + 64; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; logTest(); test1(); diff --git a/source/libs/sync/test/syncRequestVoteTest.cpp b/source/libs/sync/test/syncRequestVoteTest.cpp index 6b2120e124..4c6d7f94ad 100644 --- a/source/libs/sync/test/syncRequestVoteTest.cpp +++ b/source/libs/sync/test/syncRequestVoteTest.cpp @@ -15,7 +15,7 @@ void logTest() { } SyncRequestVote *createMsg() { - SyncRequestVote *pMsg = syncRequestVoteBuild(); + SyncRequestVote *pMsg = syncRequestVoteBuild(1000); pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); pMsg->srcId.vgId = 100; pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); @@ -28,7 +28,7 @@ SyncRequestVote *createMsg() { void test1() { SyncRequestVote *pMsg = createMsg(); - syncRequestVotePrint2((char *)"test1:", pMsg); + syncRequestVoteLog2((char *)"test1:", pMsg); syncRequestVoteDestroy(pMsg); } @@ -37,9 +37,9 @@ void test2() { uint32_t len = pMsg->bytes; char * serialized = (char *)taosMemoryMalloc(len); syncRequestVoteSerialize(pMsg, serialized, len); - SyncRequestVote *pMsg2 = syncRequestVoteBuild(); + SyncRequestVote *pMsg2 = syncRequestVoteBuild(1000); syncRequestVoteDeserialize(serialized, len, pMsg2); - syncRequestVotePrint2((char *)"test2: syncRequestVoteSerialize -> syncRequestVoteDeserialize ", pMsg2); + syncRequestVoteLog2((char *)"test2: syncRequestVoteSerialize -> syncRequestVoteDeserialize ", pMsg2); taosMemoryFree(serialized); syncRequestVoteDestroy(pMsg); @@ -51,7 +51,7 @@ void test3() { uint32_t len; char * serialized = syncRequestVoteSerialize2(pMsg, &len); SyncRequestVote *pMsg2 = syncRequestVoteDeserialize2(serialized, len); - syncRequestVotePrint2((char *)"test3: syncRequestVoteSerialize3 -> syncRequestVoteDeserialize2 ", pMsg2); + syncRequestVoteLog2((char *)"test3: syncRequestVoteSerialize3 -> syncRequestVoteDeserialize2 ", pMsg2); taosMemoryFree(serialized); syncRequestVoteDestroy(pMsg); @@ -62,10 +62,11 @@ void test4() { SyncRequestVote *pMsg = createMsg(); SRpcMsg rpcMsg; syncRequestVote2RpcMsg(pMsg, &rpcMsg); - SyncRequestVote *pMsg2 = syncRequestVoteBuild(); + SyncRequestVote *pMsg2 = syncRequestVoteBuild(1000); syncRequestVoteFromRpcMsg(&rpcMsg, pMsg2); - syncRequestVotePrint2((char *)"test4: syncRequestVote2RpcMsg -> syncRequestVoteFromRpcMsg ", pMsg2); + syncRequestVoteLog2((char *)"test4: syncRequestVote2RpcMsg -> syncRequestVoteFromRpcMsg ", pMsg2); + rpcFreeCont(rpcMsg.pCont); syncRequestVoteDestroy(pMsg); syncRequestVoteDestroy(pMsg2); } @@ -75,16 +76,16 @@ void test5() { SRpcMsg rpcMsg; syncRequestVote2RpcMsg(pMsg, &rpcMsg); SyncRequestVote *pMsg2 = syncRequestVoteFromRpcMsg2(&rpcMsg); - syncRequestVotePrint2((char *)"test5: syncRequestVote2RpcMsg -> syncRequestVoteFromRpcMsg2 ", pMsg2); + syncRequestVoteLog2((char *)"test5: syncRequestVote2RpcMsg -> syncRequestVoteFromRpcMsg2 ", pMsg2); + rpcFreeCont(rpcMsg.pCont); syncRequestVoteDestroy(pMsg); syncRequestVoteDestroy(pMsg2); } int main() { - // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; - sDebugFlag = 143 + 64; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; logTest(); test1(); diff --git a/source/libs/sync/test/syncRespMgrTest.cpp b/source/libs/sync/test/syncRespMgrTest.cpp new file mode 100644 index 0000000000..d22c37d75d --- /dev/null +++ b/source/libs/sync/test/syncRespMgrTest.cpp @@ -0,0 +1,144 @@ +#include "syncRespMgr.h" +//#include +#include +#include "syncIO.h" +#include "syncInt.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} +SSyncRespMgr *pMgr = NULL; + +void syncRespMgrInsert(uint64_t count) { + for (uint64_t i = 0; i < count; ++i) { + SRespStub stub; + memset(&stub, 0, sizeof(SRespStub)); + stub.createTime = taosGetTimestampMs(); + stub.rpcMsg.code = (pMgr->seqNum + 1); + stub.rpcMsg.ahandle = (void *)(200 + i); + stub.rpcMsg.handle = (void *)(300 + i); + uint64_t ret = syncRespMgrAdd(pMgr, &stub); + printf("insert %lu \n", ret); + } +} + +void syncRespMgrDelTest(uint64_t begin, uint64_t end) { + for (uint64_t i = begin; i <= end; ++i) { + int32_t ret = syncRespMgrDel(pMgr, i); + assert(ret == 0); + } +} + +void printStub(SRespStub *p) { + printf("createTime:%ld, rpcMsg.code:%d rpcMsg.ahandle:%ld rpcMsg.handle:%ld \n", p->createTime, p->rpcMsg.code, + (int64_t)(p->rpcMsg.ahandle), (int64_t)(p->rpcMsg.handle)); +} +void syncRespMgrPrint() { + printf("\n----------------syncRespMgrPrint--------------\n"); + taosThreadMutexLock(&(pMgr->mutex)); + + SRespStub *p = (SRespStub *)taosHashIterate(pMgr->pRespHash, NULL); + while (p) { + printStub(p); + p = (SRespStub *)taosHashIterate(pMgr->pRespHash, p); + } + + taosThreadMutexUnlock(&(pMgr->mutex)); +} + +void syncRespMgrGetTest(uint64_t i) { + printf("------syncRespMgrGetTest------- %lu -- \n", i); + SRespStub stub; + int32_t ret = syncRespMgrGet(pMgr, i, &stub); + if (ret == 1) { + printStub(&stub); + } else if (ret == 0) { + printf("%ld notFound \n", i); + } +} + +void syncRespMgrGetAndDelTest(uint64_t i) { + printf("------syncRespMgrGetAndDelTest-------%lu-- \n", i); + SRespStub stub; + int32_t ret = syncRespMgrGetAndDel(pMgr, i, &stub); + if (ret == 1) { + printStub(&stub); + } else if (ret == 0) { + printf("%ld notFound \n", i); + } +} + +void test1() { + printf("------- test1 ---------\n"); + pMgr = syncRespMgrCreate(NULL, 0); + assert(pMgr != NULL); + + syncRespMgrInsert(10); + syncRespMgrPrint(); + + printf("====== get print \n"); + for (uint64_t i = 1; i <= 10; ++i) { + syncRespMgrGetTest(i); + } + + printf("===== delete 5 - 7 \n"); + syncRespMgrDelTest(5, 7); + syncRespMgrPrint(); + + printf("====== get print \n"); + for (uint64_t i = 1; i <= 10; ++i) { + syncRespMgrGetTest(i); + } + + syncRespMgrDestroy(pMgr); +} + +void test2() { + printf("------- test2 ---------\n"); + pMgr = syncRespMgrCreate(NULL, 0); + assert(pMgr != NULL); + + syncRespMgrInsert(10); + syncRespMgrPrint(); + + printf("====== get and delete 3 - 7 \n"); + for (uint64_t i = 3; i <= 7; ++i) { + syncRespMgrGetAndDelTest(i); + } + + syncRespMgrPrint(); + syncRespMgrDestroy(pMgr); +} + +void test3() { + printf("------- test3 ---------\n"); + pMgr = syncRespMgrCreate(NULL, 0); + assert(pMgr != NULL); + + syncRespMgrInsert(10); + syncRespMgrPrint(); + + printf("====== get and delete 0 - 20 \n"); + for (uint64_t i = 0; i <= 20; ++i) { + syncRespMgrGetAndDelTest(i); + } + + syncRespMgrPrint(); + syncRespMgrDestroy(pMgr); +} + +int main() { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + logTest(); + test1(); + test2(); + test3(); + + return 0; +} diff --git a/source/libs/sync/test/syncRpcMsgTest.cpp b/source/libs/sync/test/syncRpcMsgTest.cpp index ee6f6c0800..0b21f41080 100644 --- a/source/libs/sync/test/syncRpcMsgTest.cpp +++ b/source/libs/sync/test/syncRpcMsgTest.cpp @@ -16,7 +16,7 @@ void logTest() { int gg = 0; SyncTimeout *createSyncTimeout() { - SyncTimeout *pMsg = syncTimeoutBuild2(SYNC_TIMEOUT_PING, 999, 333, &gg); + SyncTimeout *pMsg = syncTimeoutBuild2(SYNC_TIMEOUT_PING, 999, 333, 1000, &gg); return pMsg; } @@ -26,7 +26,7 @@ SyncPing *createSyncPing() { srcId.vgId = 100; destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); destId.vgId = 100; - SyncPing *pMsg = syncPingBuild3(&srcId, &destId); + SyncPing *pMsg = syncPingBuild3(&srcId, &destId, 1000); return pMsg; } @@ -36,7 +36,7 @@ SyncPingReply *createSyncPingReply() { srcId.vgId = 100; destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); destId.vgId = 100; - SyncPingReply *pMsg = syncPingReplyBuild3(&srcId, &destId); + SyncPingReply *pMsg = syncPingReplyBuild3(&srcId, &destId, 1000); return pMsg; } @@ -47,12 +47,12 @@ SyncClientRequest *createSyncClientRequest() { rpcMsg.contLen = 20; rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); strcpy((char *)rpcMsg.pCont, "hello rpc"); - SyncClientRequest *pMsg = syncClientRequestBuild2(&rpcMsg, 123, true); + SyncClientRequest *pMsg = syncClientRequestBuild2(&rpcMsg, 123, true, 1000); return pMsg; } SyncRequestVote *createSyncRequestVote() { - SyncRequestVote *pMsg = syncRequestVoteBuild(); + SyncRequestVote *pMsg = syncRequestVoteBuild(1000); pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); pMsg->srcId.vgId = 100; pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); @@ -64,7 +64,7 @@ SyncRequestVote *createSyncRequestVote() { } SyncRequestVoteReply *createSyncRequestVoteReply() { - SyncRequestVoteReply *pMsg = syncRequestVoteReplyBuild(); + SyncRequestVoteReply *pMsg = syncRequestVoteReplyBuild(1000); pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); pMsg->srcId.vgId = 100; pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); @@ -75,7 +75,7 @@ SyncRequestVoteReply *createSyncRequestVoteReply() { } SyncAppendEntries *createSyncAppendEntries() { - SyncAppendEntries *pMsg = syncAppendEntriesBuild(20); + SyncAppendEntries *pMsg = syncAppendEntriesBuild(20, 1000); pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); pMsg->srcId.vgId = 100; pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); @@ -88,7 +88,7 @@ SyncAppendEntries *createSyncAppendEntries() { } SyncAppendEntriesReply *createSyncAppendEntriesReply() { - SyncAppendEntriesReply *pMsg = syncAppendEntriesReplyBuild(); + SyncAppendEntriesReply *pMsg = syncAppendEntriesReplyBuild(1000); pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); pMsg->srcId.vgId = 100; pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); @@ -102,7 +102,7 @@ void test1() { SyncTimeout *pMsg = createSyncTimeout(); SRpcMsg rpcMsg; syncTimeout2RpcMsg(pMsg, &rpcMsg); - syncRpcMsgPrint2((char *)"test1", &rpcMsg); + syncRpcMsgLog2((char *)"test1", &rpcMsg); syncTimeoutDestroy(pMsg); } @@ -110,7 +110,7 @@ void test2() { SyncPing *pMsg = createSyncPing(); SRpcMsg rpcMsg; syncPing2RpcMsg(pMsg, &rpcMsg); - syncRpcMsgPrint2((char *)"test2", &rpcMsg); + syncRpcMsgLog2((char *)"test2", &rpcMsg); syncPingDestroy(pMsg); } @@ -118,7 +118,7 @@ void test3() { SyncPingReply *pMsg = createSyncPingReply(); SRpcMsg rpcMsg; syncPingReply2RpcMsg(pMsg, &rpcMsg); - syncRpcMsgPrint2((char *)"test3", &rpcMsg); + syncRpcMsgLog2((char *)"test3", &rpcMsg); syncPingReplyDestroy(pMsg); } @@ -126,7 +126,7 @@ void test4() { SyncRequestVote *pMsg = createSyncRequestVote(); SRpcMsg rpcMsg; syncRequestVote2RpcMsg(pMsg, &rpcMsg); - syncRpcMsgPrint2((char *)"test4", &rpcMsg); + syncRpcMsgLog2((char *)"test4", &rpcMsg); syncRequestVoteDestroy(pMsg); } @@ -134,7 +134,7 @@ void test5() { SyncRequestVoteReply *pMsg = createSyncRequestVoteReply(); SRpcMsg rpcMsg; syncRequestVoteReply2RpcMsg(pMsg, &rpcMsg); - syncRpcMsgPrint2((char *)"test5", &rpcMsg); + syncRpcMsgLog2((char *)"test5", &rpcMsg); syncRequestVoteReplyDestroy(pMsg); } @@ -142,7 +142,7 @@ void test6() { SyncAppendEntries *pMsg = createSyncAppendEntries(); SRpcMsg rpcMsg; syncAppendEntries2RpcMsg(pMsg, &rpcMsg); - syncRpcMsgPrint2((char *)"test6", &rpcMsg); + syncRpcMsgLog2((char *)"test6", &rpcMsg); syncAppendEntriesDestroy(pMsg); } @@ -150,7 +150,7 @@ void test7() { SyncAppendEntriesReply *pMsg = createSyncAppendEntriesReply(); SRpcMsg rpcMsg; syncAppendEntriesReply2RpcMsg(pMsg, &rpcMsg); - syncRpcMsgPrint2((char *)"test7", &rpcMsg); + syncRpcMsgLog2((char *)"test7", &rpcMsg); syncAppendEntriesReplyDestroy(pMsg); } @@ -158,14 +158,13 @@ void test8() { SyncClientRequest *pMsg = createSyncClientRequest(); SRpcMsg rpcMsg; syncClientRequest2RpcMsg(pMsg, &rpcMsg); - syncRpcMsgPrint2((char *)"test8", &rpcMsg); + syncRpcMsgLog2((char *)"test8", &rpcMsg); syncClientRequestDestroy(pMsg); } int main() { - // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; - sDebugFlag = 143 + 64; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; logTest(); test1(); diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index ec402ff57a..76024e061e 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -2,22 +2,62 @@ #include #include "syncIO.h" #include "syncInt.h" -#include "syncRaftStore.h" -void logTest() { - sTrace("--- sync log test: trace"); - sDebug("--- sync log test: debug"); - sInfo("--- sync log test: info"); - sWarn("--- sync log test: warn"); - sError("--- sync log test: error"); - sFatal("--- sync log test: fatal"); +/* +typedef enum { + DEBUG_FATAL = 1, + DEBUG_ERROR = 1, + DEBUG_WARN = 2, + DEBUG_INFO = 2, + DEBUG_DEBUG = 4, + DEBUG_TRACE = 8, + DEBUG_DUMP = 16, + DEBUG_SCREEN = 64, + DEBUG_FILE = 128 +} ELogLevel; +*/ + +void logTest(char* s) { + sFatal("==%s== sync log test: fatal", s); + sError("==%s== sync log test: error", s); + + sWarn("==%s== sync log test: warn", s); + sInfo("==%s== sync log test: info", s); + + sDebug("==%s== sync log test: debug", s); + + sTrace("==%s== sync log test: trace", s); +} + +void test1() { + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + logTest((char*)__FUNCTION__); +} + +void test2() { + sDebugFlag = DEBUG_DEBUG + DEBUG_SCREEN + DEBUG_FILE; + logTest((char*)__FUNCTION__); +} + +void test3() { + sDebugFlag = DEBUG_INFO + DEBUG_SCREEN + DEBUG_FILE; + logTest((char*)__FUNCTION__); +} + +void test4() { + sDebugFlag = DEBUG_ERROR + DEBUG_SCREEN + DEBUG_FILE; + logTest((char*)__FUNCTION__); } int main() { - // taosInitLog((char *)"syncTest.log", 100000, 10); + // taosInitLog("tmp/syncTest.log", 100); tsAsyncLog = 0; - sDebugFlag = 143 + 64; - logTest(); + test1(); + test2(); + test3(); + test4(); + + // taosCloseLog(); return 0; } diff --git a/source/libs/sync/test/syncTimeoutTest.cpp b/source/libs/sync/test/syncTimeoutTest.cpp index ab36f42d0d..30f25bd1d8 100644 --- a/source/libs/sync/test/syncTimeoutTest.cpp +++ b/source/libs/sync/test/syncTimeoutTest.cpp @@ -17,13 +17,13 @@ void logTest() { int gg = 0; SyncTimeout *createMsg() { - SyncTimeout *pMsg = syncTimeoutBuild2(SYNC_TIMEOUT_PING, 999, 333, &gg); + SyncTimeout *pMsg = syncTimeoutBuild2(SYNC_TIMEOUT_PING, 999, 333, 1000, &gg); return pMsg; } void test1() { SyncTimeout *pMsg = createMsg(); - syncTimeoutPrint2((char *)"test1:", pMsg); + syncTimeoutLog2((char *)"test1:", pMsg); syncTimeoutDestroy(pMsg); } @@ -34,7 +34,7 @@ void test2() { syncTimeoutSerialize(pMsg, serialized, len); SyncTimeout *pMsg2 = syncTimeoutBuild(); syncTimeoutDeserialize(serialized, len, pMsg2); - syncTimeoutPrint2((char *)"test2: syncTimeoutSerialize -> syncTimeoutDeserialize ", pMsg2); + syncTimeoutLog2((char *)"test2: syncTimeoutSerialize -> syncTimeoutDeserialize ", pMsg2); taosMemoryFree(serialized); syncTimeoutDestroy(pMsg); @@ -46,7 +46,7 @@ void test3() { uint32_t len; char * serialized = syncTimeoutSerialize2(pMsg, &len); SyncTimeout *pMsg2 = syncTimeoutDeserialize2(serialized, len); - syncTimeoutPrint2((char *)"test3: syncTimeoutSerialize3 -> syncTimeoutDeserialize2 ", pMsg2); + syncTimeoutLog2((char *)"test3: syncTimeoutSerialize3 -> syncTimeoutDeserialize2 ", pMsg2); taosMemoryFree(serialized); syncTimeoutDestroy(pMsg); @@ -59,8 +59,9 @@ void test4() { syncTimeout2RpcMsg(pMsg, &rpcMsg); SyncTimeout *pMsg2 = (SyncTimeout *)taosMemoryMalloc(rpcMsg.contLen); syncTimeoutFromRpcMsg(&rpcMsg, pMsg2); - syncTimeoutPrint2((char *)"test4: syncTimeout2RpcMsg -> syncTimeoutFromRpcMsg ", pMsg2); + syncTimeoutLog2((char *)"test4: syncTimeout2RpcMsg -> syncTimeoutFromRpcMsg ", pMsg2); + rpcFreeCont(rpcMsg.pCont); syncTimeoutDestroy(pMsg); syncTimeoutDestroy(pMsg2); } @@ -70,16 +71,16 @@ void test5() { SRpcMsg rpcMsg; syncTimeout2RpcMsg(pMsg, &rpcMsg); SyncTimeout *pMsg2 = syncTimeoutFromRpcMsg2(&rpcMsg); - syncTimeoutPrint2((char *)"test5: syncTimeout2RpcMsg -> syncTimeoutFromRpcMsg2 ", pMsg2); + syncTimeoutLog2((char *)"test5: syncTimeout2RpcMsg -> syncTimeoutFromRpcMsg2 ", pMsg2); + rpcFreeCont(rpcMsg.pCont); syncTimeoutDestroy(pMsg); syncTimeoutDestroy(pMsg2); } int main() { - // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; - sDebugFlag = 143 + 64; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; logTest(); test1(); diff --git a/source/libs/sync/test/syncUtilTest.cpp b/source/libs/sync/test/syncUtilTest.cpp index ae2afd2f53..411915fe22 100644 --- a/source/libs/sync/test/syncUtilTest.cpp +++ b/source/libs/sync/test/syncUtilTest.cpp @@ -22,9 +22,8 @@ void electRandomMSTest() { } int main() { - // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; - sDebugFlag = 143 + 64; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; logTest(); electRandomMSTest(); diff --git a/source/libs/sync/test/syncVotesGrantedTest.cpp b/source/libs/sync/test/syncVotesGrantedTest.cpp index 6164aed351..02a35b3d00 100644 --- a/source/libs/sync/test/syncVotesGrantedTest.cpp +++ b/source/libs/sync/test/syncVotesGrantedTest.cpp @@ -72,9 +72,8 @@ void initRaftId(SSyncNode* pSyncNode) { } int main(int argc, char** argv) { - // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; - sDebugFlag = 143 + 64; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; myIndex = 0; if (argc >= 2) { @@ -118,7 +117,7 @@ int main(int argc, char** argv) { } for (int i = 0; i < replicaNum; ++i) { - SyncRequestVoteReply* reply = syncRequestVoteReplyBuild(); + SyncRequestVoteReply* reply = syncRequestVoteReplyBuild(1000); reply->destId = pSyncNode->myRaftId; reply->srcId = ids[i]; reply->term = term; diff --git a/source/libs/sync/test/syncVotesRespondTest.cpp b/source/libs/sync/test/syncVotesRespondTest.cpp index 39ba6253a1..f276d34745 100644 --- a/source/libs/sync/test/syncVotesRespondTest.cpp +++ b/source/libs/sync/test/syncVotesRespondTest.cpp @@ -72,9 +72,8 @@ void initRaftId(SSyncNode* pSyncNode) { } int main(int argc, char** argv) { - // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; - sDebugFlag = 143 + 64; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; myIndex = 0; if (argc >= 2) { @@ -118,7 +117,7 @@ int main(int argc, char** argv) { } for (int i = 0; i < replicaNum; ++i) { - SyncRequestVoteReply* reply = syncRequestVoteReplyBuild(); + SyncRequestVoteReply* reply = syncRequestVoteReplyBuild(1000); reply->destId = pSyncNode->myRaftId; reply->srcId = ids[i]; reply->term = term; diff --git a/source/libs/sync/test/syncWriteTest.cpp b/source/libs/sync/test/syncWriteTest.cpp index a3c24f69ff..cff76a3ae7 100644 --- a/source/libs/sync/test/syncWriteTest.cpp +++ b/source/libs/sync/test/syncWriteTest.cpp @@ -28,29 +28,28 @@ SSyncFSM * pFsm; SWal * pWal; SSyncNode *gSyncNode; -void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, - ESyncState state) { +const char *pDir = "./syncWriteTest"; + +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", - pFsm, index, isWeak, code, state, syncUtilState2String(state)); - syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } -void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, - ESyncState state) { +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), - "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, index, isWeak, - code, state, syncUtilState2String(state)); - syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, + cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } -void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, - ESyncState state) { +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", - pFsm, index, isWeak, code, state, syncUtilState2String(state)); - syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } void initFsm() { @@ -67,7 +66,7 @@ SSyncNode *syncNodeInit() { syncInfo.queue = gSyncIO->pMsgQ; syncInfo.FpEqMsg = syncIOEqMsg; syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./write_test"); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", pDir); int code = walInit(); assert(code == 0); @@ -108,6 +107,8 @@ SSyncNode *syncNodeInit() { gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; gSyncIO->pSyncNode = pSyncNode; + syncNodeStart(pSyncNode); + return pSyncNode; } @@ -133,7 +134,7 @@ SRpcMsg *step0() { } SyncClientRequest *step1(const SRpcMsg *pMsg) { - SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true); + SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true, 1000); return pRetMsg; } @@ -160,17 +161,17 @@ int main(int argc, char **argv) { gSyncNode = syncInitTest(); assert(gSyncNode != NULL); - syncNodePrint2((char *)"", gSyncNode); + syncNodeLog2((char *)"", gSyncNode); initRaftId(gSyncNode); // step0 SRpcMsg *pMsg0 = step0(); - syncRpcMsgPrint2((char *)"==step0==", pMsg0); + syncRpcMsgLog2((char *)"==step0==", pMsg0); // step1 SyncClientRequest *pMsg1 = step1(pMsg0); - syncClientRequestPrint2((char *)"==step1==", pMsg1); + syncClientRequestLog2((char *)"==step1==", pMsg1); for (int i = 0; i < 10; ++i) { SyncClientRequest *pSyncClientRequest = pMsg1; From 01008939f2c441c4b28f01c6eb1963720b4000b5 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 19 Apr 2022 11:51:51 +0800 Subject: [PATCH 02/83] sync snapshot --- include/libs/sync/sync.h | 10 +- include/libs/sync/syncTools.h | 6 + source/libs/sync/test/CMakeLists.txt | 14 ++ source/libs/sync/test/syncSnapshotTest.cpp | 213 +++++++++++++++++++++ 4 files changed, 236 insertions(+), 7 deletions(-) create mode 100644 source/libs/sync/test/syncSnapshotTest.cpp diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 86666f4c68..6bafa29a50 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -50,14 +50,10 @@ typedef struct SSyncCfg { SNodeInfo nodeInfo[TSDB_MAX_REPLICA]; } SSyncCfg; -typedef struct SRaftId { - SyncNodeId addr; - SyncGroupId vgId; -} SRaftId; - typedef struct SSnapshot { void* data; SyncIndex lastApplyIndex; + SyncTerm lastApplyTerm; } SSnapshot; typedef enum { @@ -80,8 +76,8 @@ typedef struct SSyncFSM { void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta); void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta); - int32_t (*FpTakeSnapshot)(SSnapshot* snapshot); - int32_t (*FpRestoreSnapshot)(const SSnapshot* snapshot); + int32_t (*FpGetSnapshot)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot); + int32_t (*FpRestoreSnapshot)(struct SSyncFSM* pFsm, const SSnapshot* snapshot); } SSyncFSM; diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index 6d28f54e29..567889191a 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -27,6 +27,12 @@ extern "C" { #include "trpc.h" #include "wal.h" +// ------------------ ds ------------------- +typedef struct SRaftId { + SyncNodeId addr; + SyncGroupId vgId; +} SRaftId; + // ------------------ control ------------------- struct SSyncNode; typedef struct SSyncNode SSyncNode; diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 85fe285661..e2ac38150f 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -40,6 +40,7 @@ add_executable(syncRefTest "") add_executable(syncLogStoreCheck "") add_executable(syncRaftCfgTest "") add_executable(syncRespMgrTest "") +add_executable(syncSnapshotTest "") target_sources(syncTest @@ -210,6 +211,10 @@ target_sources(syncRespMgrTest PRIVATE "syncRespMgrTest.cpp" ) +target_sources(syncSnapshotTest + PRIVATE + "syncSnapshotTest.cpp" +) target_include_directories(syncTest @@ -422,6 +427,11 @@ target_include_directories(syncRespMgrTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncSnapshotTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -592,6 +602,10 @@ target_link_libraries(syncRespMgrTest sync gtest_main ) +target_link_libraries(syncSnapshotTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncSnapshotTest.cpp b/source/libs/sync/test/syncSnapshotTest.cpp new file mode 100644 index 0000000000..e55f37d2d6 --- /dev/null +++ b/source/libs/sync/test/syncSnapshotTest.cpp @@ -0,0 +1,213 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftEntry.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 1; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM * pFsm; +SWal * pWal; +SSyncNode *gSyncNode; +SyncIndex snapshotLastApplyIndex = SYNC_INDEX_INVALID; + +const char *pDir = "./syncSnapshotTest"; +const char *pWalDir = "./syncSnapshotTest_wal"; + +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { + SyncIndex beginIndex = SYNC_INDEX_INVALID; + if (pFsm->FpGetSnapshot != NULL) { + SSnapshot snapshot; + pFsm->FpGetSnapshot(pFsm, &snapshot); + beginIndex = snapshot.lastApplyIndex; + } + + if (cbMeta.index > beginIndex) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); + } else { + sTrace("==callback== ==CommitCb== do not apply again %ld", cbMeta.index); + } +} + +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, + cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); +} + +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); +} + +int32_t GetSnapshotCb(struct SSyncFSM *pFsm, SSnapshot *pSnapshot) { + pSnapshot->data = NULL; + pSnapshot->lastApplyIndex = snapshotLastApplyIndex; + pSnapshot->lastApplyTerm = 100; + return 0; +} + +void initFsm() { + pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); + pFsm->FpCommitCb = CommitCb; + pFsm->FpPreCommitCb = PreCommitCb; + pFsm->FpRollBackCb = RollBackCb; + pFsm->FpGetSnapshot = GetSnapshotCb; +} + +SSyncNode *syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", pDir); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + pWal = walOpen(pWalDir, &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + SSyncNode *pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->pSyncNode = pSyncNode; + + syncNodeStart(pSyncNode); + + return pSyncNode; +} + +SSyncNode *syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode *pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char *s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + taosMemoryFree(s); + } +} + +SRpcMsg *step0() { + SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); + memset(pMsg, 0, sizeof(SRpcMsg)); + pMsg->msgType = 9999; + pMsg->contLen = 32; + pMsg->pCont = taosMemoryMalloc(pMsg->contLen); + snprintf((char *)(pMsg->pCont), pMsg->contLen, "hello, world"); + return pMsg; +} + +SyncClientRequest *step1(const SRpcMsg *pMsg) { + SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true, 1000); + return pRetMsg; +} + +int main(int argc, char **argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + void logTest(); + + myIndex = 0; + if (argc >= 2) { + snapshotLastApplyIndex = atoi(argv[1]); + } + sTrace("--snapshotLastApplyIndex : %ld \n", snapshotLastApplyIndex); + + int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + // taosRemoveDir(pWalDir); + + initFsm(); + + gSyncNode = syncInitTest(); + assert(gSyncNode != NULL); + syncNodeLog2((char *)"", gSyncNode); + + initRaftId(gSyncNode); + + // step0 + SRpcMsg *pMsg0 = step0(); + syncRpcMsgLog2((char *)"==step0==", pMsg0); + + // step1 + SyncClientRequest *pMsg1 = step1(pMsg0); + syncClientRequestLog2((char *)"==step1==", pMsg1); + + for (int i = 0; i < 10; ++i) { + SyncClientRequest *pSyncClientRequest = pMsg1; + SRpcMsg rpcMsg; + syncClientRequest2RpcMsg(pSyncClientRequest, &rpcMsg); + gSyncNode->FpEqMsg(gSyncNode->queue, &rpcMsg); + + taosMsleep(1000); + } + + while (1) { + sTrace("while 1 sleep"); + taosMsleep(1000); + } + + return 0; +} From 9a7924049f566f9c008f1c38cc7af8b75f7787e5 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 19 Apr 2022 15:43:24 +0800 Subject: [PATCH 03/83] sync integration --- include/libs/sync/sync.h | 3 + source/dnode/mgmt/mgmt_vnode/inc/vmInt.h | 1 + source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 22 +++ source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 1 + source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 9 ++ source/dnode/vnode/CMakeLists.txt | 1 + source/dnode/vnode/inc/vnode.h | 2 + source/dnode/vnode/src/inc/vnodeInt.h | 31 ++-- source/dnode/vnode/src/inc/vnodeSync.h | 41 +++++ source/dnode/vnode/src/vnd/vnodeOpen.c | 6 + source/dnode/vnode/src/vnd/vnodeSvr.c | 82 +++++++++- source/dnode/vnode/src/vnd/vnodeSync.c | 162 ++++++++++++++++++++ source/libs/sync/inc/syncRaftLog.h | 3 - source/libs/sync/test/syncSnapshotTest.cpp | 2 +- 14 files changed, 347 insertions(+), 19 deletions(-) create mode 100644 source/dnode/vnode/src/inc/vnodeSync.h create mode 100644 source/dnode/vnode/src/vnd/vnodeSync.c diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 6bafa29a50..6705846d05 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -84,6 +84,9 @@ typedef struct SSyncFSM { struct SSyncRaftEntry; typedef struct SSyncRaftEntry SSyncRaftEntry; +#define SYNC_INDEX_BEGIN 0 +#define SYNC_INDEX_INVALID -1 + // abstract definition of log store in raft // SWal implements it typedef struct SSyncLogStore { diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index ab8328d038..73cf07690d 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -109,6 +109,7 @@ void vmStopWorker(SVnodesMgmt *pMgmt); int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); +int32_t vmPutMsgToSyncQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc); // sync integration int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); int32_t vmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 7eb08a8f4f..95ef865312 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -106,6 +106,16 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->hashBegin = pCreate->hashBegin; pCfg->hashEnd = pCreate->hashEnd; pCfg->hashMethod = pCreate->hashMethod; + + // sync integration + pCfg->syncCfg.myIndex = pCreate->selfIndex; + pCfg->syncCfg.replicaNum = pCreate->replica; + memset(&(pCfg->syncCfg.nodeInfo), 0, sizeof(pCfg->syncCfg.nodeInfo)); + for (int i = 0; i < pCreate->replica; ++i) { + (pCfg->syncCfg.nodeInfo)[i].nodePort = (pCreate->replicas)[i].port; + snprintf((pCfg->syncCfg.nodeInfo)[i].nodeFqdn, sizeof((pCfg->syncCfg.nodeInfo)[i].nodeFqdn), "%s", + (pCreate->replicas)[i].fqdn); + } } static void vmGenerateWrapperCfg(SVnodesMgmt *pMgmt, SCreateVnodeReq *pCreate, SWrapperCfg *pCfg) { @@ -157,6 +167,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue; msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue; msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue; + msgCb.queueFps[SYNC_QUEUE] = vmPutMsgToSyncQueue; // sync integration msgCb.qsizeFp = vmGetQueueSize; SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, msgCb); @@ -357,4 +368,15 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); + + // sync integration + dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_TIMEOUT, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_PING, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_PING_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_CLIENT_REQUEST, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_CLIENT_REQUEST_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_REQUEST_VOTE, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_REQUEST_VOTE_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_APPEND_ENTRIES, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE); } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index a212d7edf4..cde73e6fa0 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -137,6 +137,7 @@ static void *vmOpenVnodeFunc(void *param) { msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue; msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue; msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue; + msgCb.queueFps[SYNC_QUEUE] = vmPutMsgToSyncQueue; // sync integration msgCb.qsizeFp = vmGetQueueSize; snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pCfg->vgId); SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, msgCb); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 2fb29ce944..f660990184 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -306,6 +306,10 @@ static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueT dTrace("msg:%p, will be put into vnode-merge queue", pMsg); taosWriteQitem(pVnode->pMergeQ, pMsg); break; + case SYNC_QUEUE: // sync integration + dTrace("msg:%p, will be put into vnode-sync queue", pMsg); + taosWriteQitem(pVnode->pSyncQ, pMsg); + break; default: code = -1; terrno = TSDB_CODE_INVALID_PARA; @@ -332,6 +336,11 @@ int32_t vmPutMsgToMergeQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { return vmPutRpcMsgToQueue(pWrapper, pRpc, MERGE_QUEUE); } +// sync integration +int32_t vmPutMsgToSyncQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + return vmPutRpcMsgToQueue(pWrapper, pRpc, SYNC_QUEUE); +} + int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) { int32_t size = -1; SVnodeObj *pVnode = vmAcquireVnode(pWrapper->pMgmt, vgId); diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 5a6fdd1b40..727d6d72ae 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -16,6 +16,7 @@ target_sources( "src/vnd/vnodeWrite.c" "src/vnd/vnodeModule.c" "src/vnd/vnodeSvr.c" + "src/vnd/vnodeSync.c" # meta "src/meta/metaOpen.c" diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 86e876ecba..a20b30a47b 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -24,6 +24,7 @@ #include "tarray.h" #include "tfs.h" #include "wal.h" +#include "sync.h" #include "tcommon.h" #include "tfs.h" @@ -144,6 +145,7 @@ struct SVnodeCfg { bool isWeak; STsdbCfg tsdbCfg; SWalCfg walCfg; + SSyncCfg syncCfg; // sync integration uint32_t hashBegin; uint32_t hashEnd; int8_t hashMethod; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 0ff1408c91..55b1473e4e 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -54,8 +54,8 @@ typedef struct SQWorkerMgmt SQHandle; #define VNODE_META_DIR "meta" #define VNODE_TSDB_DIR "tsdb" -#define VNODE_TQ_DIR "tq" -#define VNODE_WAL_DIR "wal" +#define VNODE_TQ_DIR "tq" +#define VNODE_WAL_DIR "wal" typedef struct { int8_t streamType; // sma or other @@ -68,8 +68,8 @@ typedef struct { } SStreamSinkInfo; typedef struct { - SVnode* pVnode; - SHashObj* pHash; // streamId -> SStreamSinkInfo + SVnode *pVnode; + SHashObj *pHash; // streamId -> SStreamSinkInfo } SSink; // SVState @@ -85,25 +85,26 @@ struct SVnodeInfo { }; struct SVnode { - char* path; + char *path; SVnodeCfg config; SVState state; - STfs* pTfs; + STfs *pTfs; SMsgCb msgCb; - SVBufPool* pBufPool; - SMeta* pMeta; - STsdb* pTsdb; - SWal* pWal; - STQ* pTq; - SSink* pSink; + SVBufPool *pBufPool; + SMeta *pMeta; + STsdb *pTsdb; + SWal *pWal; + STQ *pTq; + SSink *pSink; + int64_t sync; // sync integration tsem_t canCommit; - SQHandle* pQuery; + SQHandle *pQuery; }; #define TD_VID(PVNODE) (PVNODE)->config.vgId // sma -void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data); +void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data); #include "vnd.h" @@ -113,6 +114,8 @@ void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data); #include "tq.h" +#include "vnodeSync.h" + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/vnodeSync.h b/source/dnode/vnode/src/inc/vnodeSync.h new file mode 100644 index 0000000000..a3612bec92 --- /dev/null +++ b/source/dnode/vnode/src/inc/vnodeSync.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_VNODE_SYNC_H_ +#define _TD_VNODE_SYNC_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t vnodeSyncOpen(SVnode *pVnode); +int32_t vnodeSyncStart(SVnode *pVnode); +void vnodeSyncClose(SVnode *pVnode); + +int32_t vnodeSyncEqMsg(void *qHandle, SRpcMsg *pMsg); +int32_t vnodeSendMsg(void *rpcHandle, const SEpSet *pEpSet, SRpcMsg *pMsg); + +void vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta); +void vnodeSyncPreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta); +void vnodeSyncRollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta); +int32_t vnodeSyncGetSnapshotCb(struct SSyncFSM *pFsm, SSnapshot *pSnapshot); + +SSyncFSM *syncVnodeMakeFsm(); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_VNODE_SYNC_H_*/ diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 9e4aa714e2..b61bb48db3 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -124,6 +124,12 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { goto _err; } + // sync integration + // open sync + if (vnodeSyncOpen(pVnode)) { + goto _err; + } + #if 0 if (vnodeBegin() < 0) { goto _err; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 288241eb66..274ce5ec3e 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +#include "sync.h" +#include "syncTools.h" #include "vnodeInt.h" static int vnodeProcessCreateStbReq(SVnode *pVnode, void *pReq); @@ -198,8 +200,86 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { tsdbInsertTSmaData(((SVnode *)pVnode)->pTsdb, smaId, (const char *)data); } +// sync integration int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - /*vInfo("sync message is processed");*/ + SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync); + assert(pSyncNode != NULL); + + ESyncState state = syncGetMyRole(pVnode->sync); + SyncTerm currentTerm = syncGetMyTerm(pVnode->sync); + + SMsgHead *pHead = pMsg->pCont; + + char logBuf[512]; + char *syncNodeStr = sync2SimpleStr(pVnode->sync); + snprintf(logBuf, sizeof(logBuf), "==vnodeProcessSyncReq== msgType:%d, syncNode: %s", pMsg->msgType, syncNodeStr); + syncRpcMsgLog2(logBuf, pMsg); + taosMemoryFree(syncNodeStr); + + SRpcMsg *pRpcMsg = pMsg; + + if (pRpcMsg->msgType == TDMT_VND_SYNC_TIMEOUT) { + SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + + syncNodeOnTimeoutCb(pSyncNode, pSyncMsg); + syncTimeoutDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING) { + SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + + syncNodeOnPingCb(pSyncNode, pSyncMsg); + syncPingDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING_REPLY) { + SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + + syncNodeOnPingReplyCb(pSyncNode, pSyncMsg); + syncPingReplyDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) { + SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + + syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); + syncClientRequestDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) { + SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + + syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg); + syncRequestVoteDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) { + SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + + syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg); + syncRequestVoteReplyDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) { + SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + + syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg); + syncAppendEntriesDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) { + SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + + syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg); + syncAppendEntriesReplyDestroy(pSyncMsg); + + } else { + vError("==vnodeProcessSyncReq== error msg type:%d", pRpcMsg->msgType); + } + + syncNodeRelease(pSyncNode); + return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c new file mode 100644 index 0000000000..bf1ff736a3 --- /dev/null +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sync.h" +#include "syncTools.h" +#include "tmsgcb.h" +#include "vnodeInt.h" + +// sync integration + +int32_t vnodeSyncOpen(SVnode *pVnode) { + SSyncInfo syncInfo; + syncInfo.vgId = pVnode->config.vgId; + SSyncCfg *pCfg = &(syncInfo.syncCfg); + pCfg->replicaNum = pVnode->config.syncCfg.replicaNum; + pCfg->myIndex = pVnode->config.syncCfg.myIndex; + memcpy(pCfg->nodeInfo, pVnode->config.syncCfg.nodeInfo, sizeof(pCfg->nodeInfo)); + + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s/sync", pVnode->path); + syncInfo.pWal = pVnode->pWal; + + syncInfo.pFsm = syncVnodeMakeFsm(pVnode); + syncInfo.rpcClient = NULL; + syncInfo.FpSendMsg = vnodeSendMsg; + syncInfo.queue = NULL; + syncInfo.FpEqMsg = vnodeSyncEqMsg; + + pVnode->sync = syncOpen(&syncInfo); + assert(pVnode->sync > 0); + + // for test + setPingTimerMS(pVnode->sync, 3000); + setElectTimerMS(pVnode->sync, 500); + setHeartbeatTimerMS(pVnode->sync, 100); + + return 0; +} + +int32_t vnodeSyncStart(SVnode *pVnode) { + syncStart(pVnode->sync); + return 0; +} + +void vnodeSyncClose(SVnode *pVnode) { + // stop by ref id + syncStop(pVnode->sync); +} + +void vnodeSyncSetQ(SVnode *pVnode, void *qHandle) { syncSetQ(pVnode->sync, (void *)(&(pVnode->msgCb))); } + +void vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle) { syncSetRpc(pVnode->sync, (void *)(&(pVnode->msgCb))); } + +int32_t vnodeSyncEqMsg(void *qHandle, SRpcMsg *pMsg) { + int32_t ret = 0; + SMsgCb *pMsgCb = qHandle; + if (pMsgCb->queueFps[SYNC_QUEUE] != NULL) { + tmsgPutToQueue(qHandle, SYNC_QUEUE, pMsg); + } else { + vError("vnodeSyncEqMsg queue is NULL, SYNC_QUEUE:%d", SYNC_QUEUE); + } + return ret; +} + +int32_t vnodeSendMsg(void *rpcHandle, const SEpSet *pEpSet, SRpcMsg *pMsg) { + int32_t ret = 0; + SMsgCb *pMsgCb = rpcHandle; + if (pMsgCb->queueFps[SYNC_QUEUE] != NULL) { + tmsgSendReq(rpcHandle, pEpSet, pMsg); + } else { + vError("vnodeSendMsg queue is NULL, SYNC_QUEUE:%d", SYNC_QUEUE); + } + return ret; +} + +int32_t vnodeSyncGetSnapshotCb(struct SSyncFSM *pFsm, SSnapshot *pSnapshot) { + pSnapshot->data = NULL; + pSnapshot->lastApplyIndex = 0; + pSnapshot->lastApplyTerm = 0; + return 0; +} + +void vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { + SyncIndex beginIndex = SYNC_INDEX_INVALID; + if (pFsm->FpGetSnapshot != NULL) { + SSnapshot snapshot; + pFsm->FpGetSnapshot(pFsm, &snapshot); + beginIndex = snapshot.lastApplyIndex; + } + + if (cbMeta.index > beginIndex) { + char logBuf[256]; + snprintf( + logBuf, sizeof(logBuf), + "==callback== ==CommitCb== execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, beginIndex :%ld\n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); + + SRpcMsg applyMsg; + applyMsg = *pMsg; + applyMsg.pCont = rpcMallocCont(applyMsg.contLen); + assert(applyMsg.contLen == pMsg->contLen); + memcpy(applyMsg.pCont, pMsg->pCont, applyMsg.contLen); + + // recover handle for response + SVnode *pVnode = (SVnode *)(pFsm->data); + SRpcMsg saveRpcMsg; + int32_t ret = syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &saveRpcMsg); + if (ret == 1) { + applyMsg.handle = saveRpcMsg.handle; + applyMsg.ahandle = saveRpcMsg.ahandle; + } + + // put to applyQ + tmsgPutToQueue(&(pVnode->msgCb), APPLY_QUEUE, &applyMsg); + + } else { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==CommitCb== do not execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, " + "beginIndex :%ld\n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), + beginIndex); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); + } +} + +void vnodeSyncPreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, + cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); +} + +void vnodeSyncRollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); +} + +SSyncFSM *syncVnodeMakeFsm(SVnode *pVnode) { + SSyncFSM *pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); + pFsm->data = pVnode; + pFsm->FpCommitCb = vnodeSyncCommitCb; + pFsm->FpPreCommitCb = vnodeSyncPreCommitCb; + pFsm->FpRollBackCb = vnodeSyncRollBackCb; + pFsm->FpGetSnapshot = vnodeSyncGetSnapshotCb; + return pFsm; +} diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index 2196d6c207..7db62e14d5 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -27,9 +27,6 @@ extern "C" { #include "syncRaftEntry.h" #include "taosdef.h" -#define SYNC_INDEX_BEGIN 0 -#define SYNC_INDEX_INVALID -1 - typedef struct SSyncLogStoreData { SSyncNode* pSyncNode; SWal* pWal; diff --git a/source/libs/sync/test/syncSnapshotTest.cpp b/source/libs/sync/test/syncSnapshotTest.cpp index e55f37d2d6..7ad14e0f5e 100644 --- a/source/libs/sync/test/syncSnapshotTest.cpp +++ b/source/libs/sync/test/syncSnapshotTest.cpp @@ -112,7 +112,7 @@ SSyncNode *syncNodeInit() { for (int i = 0; i < replicaNum; ++i) { pCfg->nodeInfo[i].nodePort = ports[i]; snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); } SSyncNode *pSyncNode = syncNodeOpen(&syncInfo); From 785f25c49d96ad44611398de5d2a81b07f08bfd5 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 19 Apr 2022 16:16:02 +0800 Subject: [PATCH 04/83] sync integration --- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 33 +++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index f660990184..8bd334b018 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -117,6 +117,36 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO vnodePreprocessWriteReqs(pVnode->pImpl, pArray); + // sync integration response + /* + for (int i = 0; i < taosArrayGetSize(pArray); i++) { + SNodeMsg *pMsg; + SRpcMsg *pRpc; + + pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); + pRpc = &pMsg->rpcMsg; + + // set request version + void *pBuf = POINTER_SHIFT(pRpc->pCont, sizeof(SMsgHead)); + // int64_t ver = pVnode->pImpl->state.processed++; // ??????? + int64_t ver; + taosEncodeFixedI64(&pBuf, ver); + + int32_t ret = syncPropose(pVnode->pImpl->sync, pRpc, false); + if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { + // not leader + // send response + } else if (ret == TAOS_SYNC_PROPOSE_OTHER_ERROR) { + // send response + } else if (ret == TAOS_SYNC_PROPOSE_SUCCESS) { + // ok + // send response in applyQ + } else { + assert(0); + } + } + */ + numOfMsgs = taosArrayGetSize(pArray); for (int32_t i = 0; i < numOfMsgs; i++) { SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); @@ -154,6 +184,9 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO // todo SRpcMsg *pRsp = NULL; (void)vnodeProcessWriteReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp); + + // sync integration response + // send response } } From 12294b0100d6c2afdbd094fcee021e840ab66b32 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 19 Apr 2022 16:16:59 +0800 Subject: [PATCH 05/83] sync integration --- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 8bd334b018..40a29b5f59 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -186,7 +186,7 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO (void)vnodeProcessWriteReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp); // sync integration response - // send response + // if leader send response } } From 4fbcb4929aae3203f36bf9369286007b076b3023 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 19 Apr 2022 16:24:05 +0800 Subject: [PATCH 06/83] sync integration --- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 5 ++++- source/dnode/vnode/src/vnd/vnodeSync.c | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 40a29b5f59..ef05278129 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -186,7 +186,10 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO (void)vnodeProcessWriteReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp); // sync integration response - // if leader send response + // leader + // if (pMsg->rpcMsg.handle != NULL && pMsg->rpcMsg.ahandle !-NULL) { + // send response; + //} } } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index bf1ff736a3..e0e421b95c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -117,9 +117,12 @@ void vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cb SVnode *pVnode = (SVnode *)(pFsm->data); SRpcMsg saveRpcMsg; int32_t ret = syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &saveRpcMsg); - if (ret == 1) { + if (ret == 1 && cbMeta.state == TAOS_SYNC_STATE_LEADER) { applyMsg.handle = saveRpcMsg.handle; applyMsg.ahandle = saveRpcMsg.ahandle; + } else { + applyMsg.handle = NULL; + applyMsg.ahandle = NULL; } // put to applyQ From 6f491b70b1e772c11af855c4077596cdf435a7ac Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 19 Apr 2022 16:31:18 +0800 Subject: [PATCH 07/83] sync integration --- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 34 ++++++++++----------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index ef05278129..afc06bb1e7 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -117,6 +117,23 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO vnodePreprocessWriteReqs(pVnode->pImpl, pArray); + numOfMsgs = taosArrayGetSize(pArray); + for (int32_t i = 0; i < numOfMsgs; i++) { + SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); + SRpcMsg *pRpc = &pMsg->rpcMsg; + SRpcMsg *pRsp = NULL; + + int32_t code = vnodeProcessWriteReq(pVnode->pImpl, pRpc, &pRsp); + if (pRsp != NULL) { + pRsp->ahandle = pRpc->ahandle; + tmsgSendRsp(pRsp); + taosMemoryFree(pRsp); + } else { + if (code != 0 && terrno != 0) code = terrno; + vmSendRsp(pVnode->pWrapper, pMsg, code); + } + } + // sync integration response /* for (int i = 0; i < taosArrayGetSize(pArray); i++) { @@ -147,23 +164,6 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO } */ - numOfMsgs = taosArrayGetSize(pArray); - for (int32_t i = 0; i < numOfMsgs; i++) { - SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); - SRpcMsg *pRpc = &pMsg->rpcMsg; - SRpcMsg *pRsp = NULL; - - int32_t code = vnodeProcessWriteReq(pVnode->pImpl, pRpc, &pRsp); - if (pRsp != NULL) { - pRsp->ahandle = pRpc->ahandle; - tmsgSendRsp(pRsp); - taosMemoryFree(pRsp); - } else { - if (code != 0 && terrno != 0) code = terrno; - vmSendRsp(pVnode->pWrapper, pMsg, code); - } - } - for (int32_t i = 0; i < numOfMsgs; i++) { SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); dTrace("msg:%p, is freed", pMsg); From 167657c1926a1959327887657c3d6d48be30029a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 19 Apr 2022 09:07:42 +0000 Subject: [PATCH 08/83] refactor: sync --- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 57 +++++++++++---------- source/dnode/vnode/inc/vnode.h | 6 ++- source/dnode/vnode/src/inc/vnodeInt.h | 4 +- source/dnode/vnode/src/vnd/vnodeOpen.c | 2 + 4 files changed, 38 insertions(+), 31 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 754a174b6b..17347eb59a 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -97,7 +97,7 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - int64_t version; + SRpcMsg rsp; SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SNodeMsg *)); if (pArray == NULL) { @@ -116,13 +116,15 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO } } +#if 1 + int64_t version; + vnodePreprocessWriteReqs(pVnode->pImpl, pArray, &version); numOfMsgs = taosArrayGetSize(pArray); for (int32_t i = 0; i < numOfMsgs; i++) { SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); SRpcMsg *pRpc = &pMsg->rpcMsg; - SRpcMsg rsp; rsp.pCont = NULL; rsp.contLen = 0; @@ -132,20 +134,9 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO int32_t code = vnodeProcessWriteReq(pVnode->pImpl, pRpc, version++, &rsp); tmsgSendRsp(&rsp); - -#if 0 - if (pRsp != NULL) { - pRsp->ahandle = pRpc->ahandle; - taosMemoryFree(pRsp); - } else { - if (code != 0 && terrno != 0) code = terrno; - vmSendRsp(pVnode->pWrapper, pMsg, code); - } -#endif } - +#else // sync integration response - /* for (int i = 0; i < taosArrayGetSize(pArray); i++) { SNodeMsg *pMsg; SRpcMsg *pRpc; @@ -153,18 +144,18 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); pRpc = &pMsg->rpcMsg; - // set request version - void *pBuf = POINTER_SHIFT(pRpc->pCont, sizeof(SMsgHead)); - // int64_t ver = pVnode->pImpl->state.processed++; // ??????? - int64_t ver; - taosEncodeFixedI64(&pBuf, ver); + rsp.ahandle = pRpc->ahandle; + rsp.handle = pRpc->handle; + rsp.pCont = NULL; + rsp.contLen = 0; - int32_t ret = syncPropose(pVnode->pImpl->sync, pRpc, false); + int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pRpc, false); if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { - // not leader - // send response + rsp.code = -1; + tmsgSendRsp(&rsp); } else if (ret == TAOS_SYNC_PROPOSE_OTHER_ERROR) { - // send response + rsp.code = -2; + tmsgSendRsp(&rsp); } else if (ret == TAOS_SYNC_PROPOSE_SUCCESS) { // ok // send response in applyQ @@ -172,7 +163,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO assert(0); } } - */ +#endif for (int32_t i = 0; i < numOfMsgs; i++) { SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); @@ -187,13 +178,25 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; SNodeMsg *pMsg = NULL; + SRpcMsg rsp; for (int32_t i = 0; i < numOfMsgs; ++i) { +#if 0 taosGetQitem(qall, (void **)&pMsg); - // todo - SRpcMsg *pRsp = NULL; - // (void)vnodeProcessWriteReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp); + if (pMsg->rpcMsg.handle != NULL && pMsg->rpcMsg.ahandle != NULL) { + rsp.ahandle = pMsg->rpcMsg.ahandle; + rsp.handle = pMsg->rpcMsg.handle; + rsp.code = 0; + rsp.pCont = NULL; + rsp.contLen = 0; + + if (vnodeProcessWriteReq(pVnode->pImpl, &pMsg->rpcMsg, &rsp) < 0) { + rsp.code = terrno; + tmsgSendRsp(&rsp); + } + } +#endif } } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 4e42927be0..ff9f87d65e 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -21,10 +21,10 @@ #include "tqueue.h" #include "trpc.h" +#include "sync.h" #include "tarray.h" #include "tfs.h" #include "wal.h" -#include "sync.h" #include "tcommon.h" #include "tfs.h" @@ -61,6 +61,8 @@ int32_t vnodeSync(SVnode *pVnode); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); int vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName); +int64_t vnodeGetSyncHandle(SVnode *pVnode); + // meta typedef struct SMeta SMeta; // todo: remove typedef struct SMTbCursor SMTbCursor; @@ -148,7 +150,7 @@ struct SVnodeCfg { bool isWeak; STsdbCfg tsdbCfg; SWalCfg walCfg; - SSyncCfg syncCfg; // sync integration + SSyncCfg syncCfg; // sync integration uint32_t hashBegin; uint32_t hashEnd; int8_t hashMethod; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 55b1473e4e..06ec03741c 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -54,8 +54,8 @@ typedef struct SQWorkerMgmt SQHandle; #define VNODE_META_DIR "meta" #define VNODE_TSDB_DIR "tsdb" -#define VNODE_TQ_DIR "tq" -#define VNODE_WAL_DIR "wal" +#define VNODE_TQ_DIR "tq" +#define VNODE_WAL_DIR "wal" typedef struct { int8_t streamType; // sma or other diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index b61bb48db3..ae5c99ff03 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -165,3 +165,5 @@ void vnodeClose(SVnode *pVnode) { taosMemoryFree(pVnode); } } + +int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; } \ No newline at end of file From a63559cfef3d196fcd70885efe9517c12cb2a14b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 19 Apr 2022 09:11:40 +0000 Subject: [PATCH 09/83] refactor: sync --- include/libs/transport/trpc.h | 29 +++++++++++---------- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 8 +++--- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 3e2f596784..614f358ac2 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -38,11 +38,11 @@ typedef struct SRpcConnInfo { typedef struct SRpcMsg { tmsg_t msgType; - void * pCont; + void *pCont; int contLen; int32_t code; - void * handle; // rpc handle returned to app - void * ahandle; // app handle set by client + void *handle; // rpc handle returned to app + void *ahandle; // app handle set by client int noResp; // has response or not(default 0, 0: resp, 1: no resp); int persistHandle; // persist handle or not @@ -63,7 +63,7 @@ typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, cha typedef struct SRpcInit { uint16_t localPort; // local port - char * label; // for debug purpose + char *label; // for debug purpose int numOfThreads; // number of threads to handle connections int sessions; // number of sessions allowed int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS @@ -80,36 +80,37 @@ typedef struct SRpcInit { RpcCfp cfp; // call back to retrieve the client auth info, for server app only - RpcAfp afp;; + RpcAfp afp; + ; void *parent; } SRpcInit; typedef struct { - void *val; + void *val; int32_t (*clone)(void *src, void **dst); - void (*freeFunc)(const void *arg); + void (*freeFunc)(const void *arg); } SRpcCtxVal; typedef struct { - int32_t msgType; - void *val; + int32_t msgType; + void *val; int32_t (*clone)(void *src, void **dst); - void (*freeFunc)(const void *arg); + void (*freeFunc)(const void *arg); } SRpcBrokenlinkVal; typedef struct { - SHashObj * args; + SHashObj *args; SRpcBrokenlinkVal brokenVal; } SRpcCtx; int32_t rpcInit(); void rpcCleanup(); -void * rpcOpen(const SRpcInit *pRpc); +void *rpcOpen(const SRpcInit *pRpc); void rpcClose(void *); -void * rpcMallocCont(int contLen); +void *rpcMallocCont(int contLen); void rpcFreeCont(void *pCont); -void * rpcReallocCont(void *ptr, int contLen); +void *rpcReallocCont(void *ptr, int contLen); // Because taosd supports multi-process mode // These functions should not be used on the server side diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 17347eb59a..d15f97b042 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -116,7 +116,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO } } -#if 1 +#if 0 int64_t version; vnodePreprocessWriteReqs(pVnode->pImpl, pArray, &version); @@ -180,8 +180,10 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO SNodeMsg *pMsg = NULL; SRpcMsg rsp; + static int64_t version = 0; + for (int32_t i = 0; i < numOfMsgs; ++i) { -#if 0 +#if 1 taosGetQitem(qall, (void **)&pMsg); if (pMsg->rpcMsg.handle != NULL && pMsg->rpcMsg.ahandle != NULL) { @@ -191,7 +193,7 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO rsp.pCont = NULL; rsp.contLen = 0; - if (vnodeProcessWriteReq(pVnode->pImpl, &pMsg->rpcMsg, &rsp) < 0) { + if (vnodeProcessWriteReq(pVnode->pImpl, &pMsg->rpcMsg, version++, &rsp) < 0) { rsp.code = terrno; tmsgSendRsp(&rsp); } From f7fb9b023f46a319323cb003d95006458994e989 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 19 Apr 2022 20:14:55 +0800 Subject: [PATCH 10/83] sync integration --- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 12 ++++++++++++ source/dnode/vnode/inc/vnode.h | 5 +++++ source/dnode/vnode/src/inc/vnodeSync.h | 3 +++ 3 files changed, 20 insertions(+) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index cde73e6fa0..48bae163d9 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -73,6 +73,12 @@ int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { return -1; } + // sync integration + vnodeSyncSetQ(pImpl, NULL); + vnodeSyncSetRpc(pImpl, NULL); + int32_t ret = vnodeSyncStart(pImpl); + assert(ret == 0); + taosWLockLatch(&pMgmt->latch); int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); taosWUnLockLatch(&pMgmt->latch); @@ -305,6 +311,12 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) { goto _OVER; } + // sync integration + if (syncInit() != 0) { + dError("failed to open sync since %s", terrstr()); + return -1; + } + if (vnodeInit(tsNumOfCommitThreads) != 0) { dError("failed to init vnode since %s", terrstr()); goto _OVER; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index ff9f87d65e..00bcec7be9 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -170,6 +170,11 @@ typedef struct { uint64_t uid; } STableKeyInfo; +// sync integration +void vnodeSyncSetQ(SVnode *pVnode, void *qHandle); +void vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle); +int32_t vnodeSyncStart(SVnode *pVnode); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/vnodeSync.h b/source/dnode/vnode/src/inc/vnodeSync.h index a3612bec92..6bbdc4ae72 100644 --- a/source/dnode/vnode/src/inc/vnodeSync.h +++ b/source/dnode/vnode/src/inc/vnodeSync.h @@ -24,6 +24,9 @@ int32_t vnodeSyncOpen(SVnode *pVnode); int32_t vnodeSyncStart(SVnode *pVnode); void vnodeSyncClose(SVnode *pVnode); +void vnodeSyncSetQ(SVnode *pVnode, void *qHandle); +void vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle); + int32_t vnodeSyncEqMsg(void *qHandle, SRpcMsg *pMsg); int32_t vnodeSendMsg(void *rpcHandle, const SEpSet *pEpSet, SRpcMsg *pMsg); From e0332e9c124282dd5f4f1836fe7af5c2a3523701 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 19 Apr 2022 22:04:02 +0800 Subject: [PATCH 11/83] sync integration --- source/dnode/vnode/src/inc/vnodeSync.h | 2 +- source/dnode/vnode/src/vnd/vnodeCfg.c | 27 ++++++++++++++++++++++++++ source/dnode/vnode/src/vnd/vnodeOpen.c | 2 +- source/dnode/vnode/src/vnd/vnodeSync.c | 4 ++-- 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeSync.h b/source/dnode/vnode/src/inc/vnodeSync.h index 6bbdc4ae72..fea94c4607 100644 --- a/source/dnode/vnode/src/inc/vnodeSync.h +++ b/source/dnode/vnode/src/inc/vnodeSync.h @@ -20,7 +20,7 @@ extern "C" { #endif -int32_t vnodeSyncOpen(SVnode *pVnode); +int32_t vnodeSyncOpen(SVnode *pVnode, char *path); int32_t vnodeSyncStart(SVnode *pVnode); void vnodeSyncClose(SVnode *pVnode); diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 625a2b3aed..0f8bab7856 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -82,6 +82,18 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1; + // sync integration + if (tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1; + SJson *pNodeInfoArr = tjsonCreateArray(); + tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", pNodeInfoArr); + for (int i = 0; i < pCfg->syncCfg.replicaNum; ++i) { + SJson *pNodeInfo = tjsonCreateObject(); + tjsonAddIntegerToObject(pNodeInfo, "nodePort", (pCfg->syncCfg.nodeInfo)[i].nodePort); + tjsonAddStringToObject(pNodeInfo, "nodeFqdn", (pCfg->syncCfg.nodeInfo)[i].nodeFqdn); + tjsonAddItemToArray(pNodeInfoArr, pNodeInfo); + } + return 0; } @@ -120,6 +132,21 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (tjsonGetNumberValue(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1; if (tjsonGetNumberValue(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1; + // sync integration + if (tjsonGetNumberValue(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1; + if (tjsonGetNumberValue(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1; + + SJson *pNodeInfoArr = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo"); + int arraySize = tjsonGetArraySize(pNodeInfoArr); + assert(arraySize == pCfg->syncCfg.replicaNum); + + for (int i = 0; i < arraySize; ++i) { + cJSON *pNodeInfo = tjsonGetArrayItem(pNodeInfoArr, i); + assert(pNodeInfo != NULL); + tjsonGetNumberValue(pNodeInfo, "nodePort", (pCfg->syncCfg.nodeInfo)[i].nodePort); + tjsonGetStringValue(pNodeInfo, "nodeFqdn", (pCfg->syncCfg.nodeInfo)[i].nodeFqdn); + } + return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index ae5c99ff03..30eb9e2a42 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -126,7 +126,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { // sync integration // open sync - if (vnodeSyncOpen(pVnode)) { + if (vnodeSyncOpen(pVnode, dir)) { goto _err; } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index e0e421b95c..4db483851e 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -20,7 +20,7 @@ // sync integration -int32_t vnodeSyncOpen(SVnode *pVnode) { +int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { SSyncInfo syncInfo; syncInfo.vgId = pVnode->config.vgId; SSyncCfg *pCfg = &(syncInfo.syncCfg); @@ -28,7 +28,7 @@ int32_t vnodeSyncOpen(SVnode *pVnode) { pCfg->myIndex = pVnode->config.syncCfg.myIndex; memcpy(pCfg->nodeInfo, pVnode->config.syncCfg.nodeInfo, sizeof(pCfg->nodeInfo)); - snprintf(syncInfo.path, sizeof(syncInfo.path), "%s/sync", pVnode->path); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s/sync", path); syncInfo.pWal = pVnode->pWal; syncInfo.pFsm = syncVnodeMakeFsm(pVnode); From 75684fed32cadfa51e980542fe8a12a0dd5f3cc4 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 20 Apr 2022 10:24:37 +0800 Subject: [PATCH 12/83] report status by dnodeId equal --- source/dnode/mnode/impl/src/mndDnode.c | 33 ++++++++++++++------------ 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 5ce87a413c..f225f2fbe7 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -52,9 +52,9 @@ static int32_t mndProcessConfigDnodeReq(SNodeMsg *pReq); static int32_t mndProcessConfigDnodeRsp(SNodeMsg *pRsp); static int32_t mndProcessStatusReq(SNodeMsg *pReq); -static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); +static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter); -static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); +static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter); int32_t mndInitDnode(SMnode *pMnode) { @@ -335,10 +335,13 @@ static int32_t mndProcessStatusReq(SNodeMsg *pReq) { } bool roleChanged = false; for (int32_t vg = 0; vg < pVgroup->replica; ++vg) { - if (pVgroup->vnodeGid[vg].role != pVload->role) { - roleChanged = true; + // sync integration + if (pVgroup->vnodeGid[vg].dnodeId == statusReq.dnodeId) { + if (pVgroup->vnodeGid[vg].role != pVload->role) { + roleChanged = true; + } + pVgroup->vnodeGid[vg].role = pVload->role; } - pVgroup->vnodeGid[vg].role = pVload->role; } if (roleChanged) { // notify scheduler role has changed @@ -634,7 +637,7 @@ static int32_t mndProcessConfigDnodeRsp(SNodeMsg *pRsp) { return TSDB_CODE_SUCCESS; } -static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { +static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->pNode; int32_t totalRows = 0; int32_t numOfRows = 0; @@ -667,12 +670,12 @@ static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* STR_WITH_MAXSIZE_TO_VARSTR(buf, cfgOpts[i], TSDB_CONFIG_OPTION_LEN); - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*) buf, false); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)buf, false); STR_WITH_MAXSIZE_TO_VARSTR(bufVal, cfgVals[i], TSDB_CONIIG_VALUE_LEN); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*) bufVal, false); + colDataAppend(pColInfo, numOfRows, (const char *)bufVal, false); numOfRows++; } @@ -683,7 +686,7 @@ static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter) {} -static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { +static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; @@ -698,8 +701,8 @@ static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p cols = 0; - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*) &pDnode->id, false); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->id, false); char buf[tListLen(pDnode->ep) + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(buf, pDnode->ep, pShow->bytes[cols]); @@ -709,19 +712,19 @@ static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); int16_t id = mndGetVnodesNum(pMnode, pDnode->id); - colDataAppend(pColInfo, numOfRows, (const char*) &id, false); + colDataAppend(pColInfo, numOfRows, (const char *)&id, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->numOfSupportVnodes, false); char b1[9] = {0}; - STR_TO_VARSTR(b1, online? "ready":"offline"); + STR_TO_VARSTR(b1, online ? "ready" : "offline"); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, b1, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*) &pDnode->createdTime, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->createdTime, false); char b[tListLen(offlineReason) + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(b, online ? "" : offlineReason[pDnode->offlineReason]); From 58792537a4a0aceb24ffcef32db7d00010a24ec4 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 20 Apr 2022 10:40:51 +0800 Subject: [PATCH 13/83] =?UTF-8?q?sync=20integration=20send=20response?= =?UTF-8?q?=E2=80=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index d15f97b042..a53127afce 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -184,19 +184,22 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO for (int32_t i = 0; i < numOfMsgs; ++i) { #if 1 + // sync integration + taosGetQitem(qall, (void **)&pMsg); + rsp.code = 0; + rsp.pCont = NULL; + rsp.contLen = 0; + if (vnodeProcessWriteReq(pVnode->pImpl, &pMsg->rpcMsg, version++, &rsp) < 0) { + rsp.code = terrno; + dTrace("vnodeProcessWriteReq error, code:%d", terrno); + } + if (pMsg->rpcMsg.handle != NULL && pMsg->rpcMsg.ahandle != NULL) { rsp.ahandle = pMsg->rpcMsg.ahandle; rsp.handle = pMsg->rpcMsg.handle; - rsp.code = 0; - rsp.pCont = NULL; - rsp.contLen = 0; - - if (vnodeProcessWriteReq(pVnode->pImpl, &pMsg->rpcMsg, version++, &rsp) < 0) { - rsp.code = terrno; - tmsgSendRsp(&rsp); - } + tmsgSendRsp(&rsp); } #endif } From 704671d2fe035e37a7f9135ca8a418d2c176ac16 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 20 Apr 2022 10:54:27 +0800 Subject: [PATCH 14/83] sync integration add vnodeSyncClose --- source/dnode/vnode/src/vnd/vnodeOpen.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 30eb9e2a42..dee54be5ac 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -155,6 +155,10 @@ void vnodeClose(SVnode *pVnode) { vnodeSyncCommit(pVnode); // close vnode vnodeQueryClose(pVnode); + + // sync integration + vnodeSyncClose(pVnode); + walClose(pVnode->pWal); tqClose(pVnode->pTq); tsdbClose(pVnode->pTsdb); From 6035f0305d39911dbff10c35fd32a0aa57068d14 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 20 Apr 2022 11:51:00 +0800 Subject: [PATCH 15/83] sync integration add SyncApplyMsg --- include/libs/sync/syncTools.h | 33 +++++++++++++++++++++++++++++- source/libs/sync/src/syncMessage.c | 28 +++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index 567889191a..327450c10a 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -363,6 +363,37 @@ void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg); void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg); void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg); +// --------------------------------------------- +typedef struct SyncApplyMsg { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; // user SyncApplyMsg msgType + uint32_t originalRpcType; // user RpcMsg msgType + SFsmCbMeta fsmMeta; + uint32_t dataLen; // user RpcMsg.contLen + char data[]; // user RpcMsg.pCont +} SyncApplyMsg; + +SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen); +SyncApplyMsg* syncApplyMsgBuild2(const SRpcMsg* pOriginalRpcMsg, int32_t vgId, SFsmCbMeta* pMeta); +void syncApplyMsgDestroy(SyncApplyMsg* pMsg); +void syncApplyMsgSerialize(const SyncApplyMsg* pMsg, char* buf, uint32_t bufLen); +void syncApplyMsgDeserialize(const char* buf, uint32_t len, SyncApplyMsg* pMsg); +char* syncApplyMsgSerialize2(const SyncApplyMsg* pMsg, uint32_t* len); +SyncApplyMsg* syncApplyMsgDeserialize2(const char* buf, uint32_t len); +void syncApplyMsg2RpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pRpcMsg); // SyncApplyMsg to SRpcMsg, put it into ApplyQ +void syncApplyMsgFromRpcMsg(const SRpcMsg* pRpcMsg, SyncApplyMsg* pMsg); // get SRpcMsg from ApplyQ, to SyncApplyMsg +void syncApplyMsg2OriginalRpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pOriginalRpcMsg); // SyncApplyMsg to OriginalRpcMsg +SyncApplyMsg* syncApplyMsgFromRpcMsg2(const SRpcMsg* pRpcMsg); +cJSON* syncApplyMsg2Json(const SyncApplyMsg* pMsg); +char* syncApplyMsg2Str(const SyncApplyMsg* pMsg); + +// for debug ---------------------- +void syncApplyMsgPrint(const SyncApplyMsg* pMsg); +void syncApplyMsgPrint2(char* s, const SyncApplyMsg* pMsg); +void ssyncApplyMsgLog(const SyncApplyMsg* pMsg); +void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg); + // on message ---------------------- int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); @@ -373,7 +404,7 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg); int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg); -//--------------------- +// --------------------------------------------- #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 76c68569c3..1299b37769 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -1531,3 +1531,31 @@ void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg) { sTrace("syncAppendEntriesReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } + +// ---- message process SyncApplyMsg---- +SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen) { return NULL; } +SyncApplyMsg* syncApplyMsgBuild2(const SRpcMsg* pOriginalRpcMsg, int32_t vgId, SFsmCbMeta* pMeta) { return NULL; } +void syncApplyMsgDestroy(SyncApplyMsg* pMsg) {} +void syncApplyMsgSerialize(const SyncApplyMsg* pMsg, char* buf, uint32_t bufLen) {} +void syncApplyMsgDeserialize(const char* buf, uint32_t len, SyncApplyMsg* pMsg) {} +char* syncApplyMsgSerialize2(const SyncApplyMsg* pMsg, uint32_t* len) { return NULL; } +SyncApplyMsg* syncApplyMsgDeserialize2(const char* buf, uint32_t len) { return NULL; } + +// SyncApplyMsg to SRpcMsg, put it into ApplyQ +void syncApplyMsg2RpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pRpcMsg) {} + +// get SRpcMsg from ApplyQ, to SyncApplyMsg +void syncApplyMsgFromRpcMsg(const SRpcMsg* pRpcMsg, SyncApplyMsg* pMsg) {} + +// SyncApplyMsg to OriginalRpcMsg +void syncApplyMsg2OriginalRpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pOriginalRpcMsg) {} + +SyncApplyMsg* syncApplyMsgFromRpcMsg2(const SRpcMsg* pRpcMsg) { return NULL; } +cJSON* syncApplyMsg2Json(const SyncApplyMsg* pMsg) { return NULL; } +char* syncApplyMsg2Str(const SyncApplyMsg* pMsg) { return NULL; } + +// for debug ---------------------- +void syncApplyMsgPrint(const SyncApplyMsg* pMsg) {} +void syncApplyMsgPrint2(char* s, const SyncApplyMsg* pMsg) {} +void ssyncApplyMsgLog(const SyncApplyMsg* pMsg) {} +void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg) {} From 9cb2192a879ee85b30df4378f3d8d56e05a0050f Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 20 Apr 2022 14:30:34 +0800 Subject: [PATCH 16/83] add SyncApplyMsg and test --- include/common/tmsgdef.h | 2 + include/libs/sync/syncTools.h | 8 +- source/libs/sync/src/syncMessage.c | 157 ++++++++++++++++++--- source/libs/sync/test/CMakeLists.txt | 14 ++ source/libs/sync/test/syncApplyMsgTest.cpp | 123 ++++++++++++++++ 5 files changed, 283 insertions(+), 21 deletions(-) create mode 100644 source/libs/sync/test/syncApplyMsgTest.cpp diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 24d259b0f2..b3c47353b8 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -206,6 +206,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL) + // sync integration TD_DEF_MSG_TYPE(TDMT_VND_SYNC_TIMEOUT, "vnode-sync-timeout", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING, "vnode-sync-ping", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING_REPLY, "vnode-sync-ping-reply", NULL, NULL) @@ -218,6 +219,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_SYNC_NOOP, "vnode-sync-noop", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SYNC_UNKNOWN, "vnode-sync-unknown", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SYNC_COMMON_RESPONSE, "vnode-sync-common-response", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPLY_MSG, "vnode-sync-apply-msg", NULL, NULL) // Requests handled by QNODE TD_NEW_MSG_SEG(TDMT_QND_MSG) diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index 327450c10a..0438d6d436 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -383,15 +383,15 @@ char* syncApplyMsgSerialize2(const SyncApplyMsg* pMsg, uint32_t* len); SyncApplyMsg* syncApplyMsgDeserialize2(const char* buf, uint32_t len); void syncApplyMsg2RpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pRpcMsg); // SyncApplyMsg to SRpcMsg, put it into ApplyQ void syncApplyMsgFromRpcMsg(const SRpcMsg* pRpcMsg, SyncApplyMsg* pMsg); // get SRpcMsg from ApplyQ, to SyncApplyMsg -void syncApplyMsg2OriginalRpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pOriginalRpcMsg); // SyncApplyMsg to OriginalRpcMsg SyncApplyMsg* syncApplyMsgFromRpcMsg2(const SRpcMsg* pRpcMsg); -cJSON* syncApplyMsg2Json(const SyncApplyMsg* pMsg); -char* syncApplyMsg2Str(const SyncApplyMsg* pMsg); +void syncApplyMsg2OriginalRpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pOriginalRpcMsg); // SyncApplyMsg to OriginalRpcMsg +cJSON* syncApplyMsg2Json(const SyncApplyMsg* pMsg); +char* syncApplyMsg2Str(const SyncApplyMsg* pMsg); // for debug ---------------------- void syncApplyMsgPrint(const SyncApplyMsg* pMsg); void syncApplyMsgPrint2(char* s, const SyncApplyMsg* pMsg); -void ssyncApplyMsgLog(const SyncApplyMsg* pMsg); +void syncApplyMsgLog(const SyncApplyMsg* pMsg); void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg); // on message ---------------------- diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 1299b37769..2b1b40997e 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -1533,29 +1533,152 @@ void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg) { } // ---- message process SyncApplyMsg---- -SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen) { return NULL; } -SyncApplyMsg* syncApplyMsgBuild2(const SRpcMsg* pOriginalRpcMsg, int32_t vgId, SFsmCbMeta* pMeta) { return NULL; } -void syncApplyMsgDestroy(SyncApplyMsg* pMsg) {} -void syncApplyMsgSerialize(const SyncApplyMsg* pMsg, char* buf, uint32_t bufLen) {} -void syncApplyMsgDeserialize(const char* buf, uint32_t len, SyncApplyMsg* pMsg) {} -char* syncApplyMsgSerialize2(const SyncApplyMsg* pMsg, uint32_t* len) { return NULL; } -SyncApplyMsg* syncApplyMsgDeserialize2(const char* buf, uint32_t len) { return NULL; } +SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen) { + uint32_t bytes = sizeof(SyncApplyMsg) + dataLen; + SyncApplyMsg* pMsg = taosMemoryMalloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->msgType = TDMT_VND_SYNC_APPLY_MSG; + pMsg->dataLen = dataLen; + return pMsg; +} + +SyncApplyMsg* syncApplyMsgBuild2(const SRpcMsg* pOriginalRpcMsg, int32_t vgId, SFsmCbMeta* pMeta) { + SyncApplyMsg* pMsg = syncApplyMsgBuild(pOriginalRpcMsg->contLen); + pMsg->vgId = vgId; + pMsg->originalRpcType = pOriginalRpcMsg->msgType; + pMsg->fsmMeta = *pMeta; + memcpy(pMsg->data, pOriginalRpcMsg->pCont, pOriginalRpcMsg->contLen); + return pMsg; +} + +void syncApplyMsgDestroy(SyncApplyMsg* pMsg) { + if (pMsg != NULL) { + taosMemoryFree(pMsg); + } +} + +void syncApplyMsgSerialize(const SyncApplyMsg* pMsg, char* buf, uint32_t bufLen) { + assert(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncApplyMsgDeserialize(const char* buf, uint32_t len, SyncApplyMsg* pMsg) { + memcpy(pMsg, buf, len); + assert(len == pMsg->bytes); +} + +char* syncApplyMsgSerialize2(const SyncApplyMsg* pMsg, uint32_t* len) { + char* buf = taosMemoryMalloc(pMsg->bytes); + assert(buf != NULL); + syncApplyMsgSerialize(pMsg, buf, pMsg->bytes); + if (len != NULL) { + *len = pMsg->bytes; + } + return buf; +} + +SyncApplyMsg* syncApplyMsgDeserialize2(const char* buf, uint32_t len) { + uint32_t bytes = *((uint32_t*)buf); + SyncApplyMsg* pMsg = taosMemoryMalloc(bytes); + assert(pMsg != NULL); + syncApplyMsgDeserialize(buf, len, pMsg); + assert(len == pMsg->bytes); + return pMsg; +} // SyncApplyMsg to SRpcMsg, put it into ApplyQ -void syncApplyMsg2RpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pRpcMsg) {} +void syncApplyMsg2RpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncApplyMsgSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} // get SRpcMsg from ApplyQ, to SyncApplyMsg -void syncApplyMsgFromRpcMsg(const SRpcMsg* pRpcMsg, SyncApplyMsg* pMsg) {} +void syncApplyMsgFromRpcMsg(const SRpcMsg* pRpcMsg, SyncApplyMsg* pMsg) { + syncApplyMsgDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +SyncApplyMsg* syncApplyMsgFromRpcMsg2(const SRpcMsg* pRpcMsg) { + SyncApplyMsg* pMsg = syncApplyMsgDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + return pMsg; +} // SyncApplyMsg to OriginalRpcMsg -void syncApplyMsg2OriginalRpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pOriginalRpcMsg) {} +void syncApplyMsg2OriginalRpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pOriginalRpcMsg) { + memset(pOriginalRpcMsg, 0, sizeof(*pOriginalRpcMsg)); + pOriginalRpcMsg->msgType = pMsg->originalRpcType; + pOriginalRpcMsg->contLen = pMsg->dataLen; + pOriginalRpcMsg->pCont = rpcMallocCont(pOriginalRpcMsg->contLen); + memcpy(pOriginalRpcMsg->pCont, pMsg->data, pOriginalRpcMsg->contLen); +} -SyncApplyMsg* syncApplyMsgFromRpcMsg2(const SRpcMsg* pRpcMsg) { return NULL; } -cJSON* syncApplyMsg2Json(const SyncApplyMsg* pMsg) { return NULL; } -char* syncApplyMsg2Str(const SyncApplyMsg* pMsg) { return NULL; } +cJSON* syncApplyMsg2Json(const SyncApplyMsg* pMsg) { + char u64buf[128]; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + cJSON_AddNumberToObject(pRoot, "originalRpcType", pMsg->originalRpcType); + + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->fsmMeta.index); + cJSON_AddStringToObject(pRoot, "fsmMeta.index", u64buf); + cJSON_AddNumberToObject(pRoot, "fsmMeta.isWeak", pMsg->fsmMeta.isWeak); + cJSON_AddNumberToObject(pRoot, "fsmMeta.code", pMsg->fsmMeta.code); + cJSON_AddNumberToObject(pRoot, "fsmMeta.state", pMsg->fsmMeta.state); + cJSON_AddStringToObject(pRoot, "fsmMeta.state.str", syncUtilState2String(pMsg->fsmMeta.state)); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->fsmMeta.seqNum); + cJSON_AddStringToObject(pRoot, "fsmMeta.seqNum", u64buf); + + cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); + char* s; + s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data", s); + taosMemoryFree(s); + s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data2", s); + taosMemoryFree(s); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncApplyMsg", pRoot); + return pJson; +} + +char* syncApplyMsg2Str(const SyncApplyMsg* pMsg) { + cJSON* pJson = syncApplyMsg2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} // for debug ---------------------- -void syncApplyMsgPrint(const SyncApplyMsg* pMsg) {} -void syncApplyMsgPrint2(char* s, const SyncApplyMsg* pMsg) {} -void ssyncApplyMsgLog(const SyncApplyMsg* pMsg) {} -void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg) {} +void syncApplyMsgPrint(const SyncApplyMsg* pMsg) { + char* serialized = syncApplyMsg2Str(pMsg); + printf("syncApplyMsgPrint | len:%lu | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncApplyMsgPrint2(char* s, const SyncApplyMsg* pMsg) { + char* serialized = syncApplyMsg2Str(pMsg); + printf("syncApplyMsgPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncApplyMsgLog(const SyncApplyMsg* pMsg) { + char* serialized = syncApplyMsg2Str(pMsg); + sTrace("ssyncApplyMsgLog | len:%lu | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg) { + char* serialized = syncApplyMsg2Str(pMsg); + sTrace("syncApplyMsgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); +} diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index e2ac38150f..5d43d47358 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -41,6 +41,7 @@ add_executable(syncLogStoreCheck "") add_executable(syncRaftCfgTest "") add_executable(syncRespMgrTest "") add_executable(syncSnapshotTest "") +add_executable(syncApplyMsgTest "") target_sources(syncTest @@ -215,6 +216,10 @@ target_sources(syncSnapshotTest PRIVATE "syncSnapshotTest.cpp" ) +target_sources(syncApplyMsgTest + PRIVATE + "syncApplyMsgTest.cpp" +) target_include_directories(syncTest @@ -432,6 +437,11 @@ target_include_directories(syncSnapshotTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncApplyMsgTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -606,6 +616,10 @@ target_link_libraries(syncSnapshotTest sync gtest_main ) +target_link_libraries(syncApplyMsgTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncApplyMsgTest.cpp b/source/libs/sync/test/syncApplyMsgTest.cpp new file mode 100644 index 0000000000..7544613c5d --- /dev/null +++ b/source/libs/sync/test/syncApplyMsgTest.cpp @@ -0,0 +1,123 @@ +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SyncApplyMsg *createMsg() { + SRpcMsg rpcMsg; + memset(&rpcMsg, 0, sizeof(rpcMsg)); + rpcMsg.msgType = 12345; + rpcMsg.contLen = 20; + rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); + strcpy((char *)rpcMsg.pCont, "hello rpc"); + + SFsmCbMeta meta; + meta.code = 11; + meta.index = 22; + meta.isWeak = 1; + meta.seqNum = 33; + meta.state = TAOS_SYNC_STATE_LEADER; + + SyncApplyMsg *pMsg = syncApplyMsgBuild2(&rpcMsg, 123, &meta); + rpcFreeCont(rpcMsg.pCont); + return pMsg; +} + +void test1() { + SyncApplyMsg *pMsg = createMsg(); + syncApplyMsgLog2((char *)"test1:", pMsg); + syncApplyMsgDestroy(pMsg); +} + +void test2() { + SyncApplyMsg *pMsg = createMsg(); + uint32_t len = pMsg->bytes; + char * serialized = (char *)taosMemoryMalloc(len); + syncApplyMsgSerialize(pMsg, serialized, len); + SyncApplyMsg *pMsg2 = syncApplyMsgBuild(pMsg->dataLen); + syncApplyMsgDeserialize(serialized, len, pMsg2); + syncApplyMsgLog2((char *)"test2: syncApplyMsgSerialize -> syncApplyMsgDeserialize ", pMsg2); + + taosMemoryFree(serialized); + syncApplyMsgDestroy(pMsg); + syncApplyMsgDestroy(pMsg2); +} + +void test3() { + SyncApplyMsg *pMsg = createMsg(); + uint32_t len; + char * serialized = syncApplyMsgSerialize2(pMsg, &len); + SyncApplyMsg *pMsg2 = syncApplyMsgDeserialize2(serialized, len); + syncApplyMsgLog2((char *)"test3: syncApplyMsgSerialize2 -> syncApplyMsgDeserialize2 ", pMsg2); + + taosMemoryFree(serialized); + syncApplyMsgDestroy(pMsg); + syncApplyMsgDestroy(pMsg2); +} + +void test4() { + SyncApplyMsg *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncApplyMsg2RpcMsg(pMsg, &rpcMsg); + SyncApplyMsg *pMsg2 = (SyncApplyMsg *)taosMemoryMalloc(rpcMsg.contLen); + syncApplyMsgFromRpcMsg(&rpcMsg, pMsg2); + syncApplyMsgLog2((char *)"test4: syncApplyMsg2RpcMsg -> syncApplyMsgFromRpcMsg ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncApplyMsgDestroy(pMsg); + syncApplyMsgDestroy(pMsg2); +} + +void test5() { + SyncApplyMsg *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncApplyMsg2RpcMsg(pMsg, &rpcMsg); + SyncApplyMsg *pMsg2 = syncApplyMsgFromRpcMsg2(&rpcMsg); + syncApplyMsgLog2((char *)"test5: syncClientRequest2RpcMsg -> syncApplyMsgFromRpcMsg2 ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncApplyMsgDestroy(pMsg); + syncApplyMsgDestroy(pMsg2); +} + +void test6() { + SyncApplyMsg *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncApplyMsg2RpcMsg(pMsg, &rpcMsg); + SyncApplyMsg *pMsg2 = syncApplyMsgFromRpcMsg2(&rpcMsg); + + SRpcMsg originalRpcMsg; + syncApplyMsg2OriginalRpcMsg(pMsg2, &originalRpcMsg); + syncRpcMsgLog2((char*)"test6", &originalRpcMsg); + + rpcFreeCont(originalRpcMsg.pCont); + rpcFreeCont(rpcMsg.pCont); + syncApplyMsgDestroy(pMsg); + syncApplyMsgDestroy(pMsg2); +} + +int main() { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + logTest(); + + test1(); + test2(); + test3(); + test4(); + test5(); + test6(); + + return 0; +} From ec73953a0f518e9259bcd55e539bbb03f96efdb9 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 20 Apr 2022 15:02:00 +0800 Subject: [PATCH 17/83] add SyncApplyMsg into vmProcessApplyQueue --- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 17 +++++++++++++++-- source/dnode/vnode/src/vnd/vnodeSync.c | 19 +++++++++++++------ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index a53127afce..9e42f8dfb6 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -14,6 +14,8 @@ */ #define _DEFAULT_SOURCE +#include "sync.h" +#include "syncTools.h" #include "vmInt.h" static inline void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { @@ -180,7 +182,7 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO SNodeMsg *pMsg = NULL; SRpcMsg rsp; - static int64_t version = 0; + // static int64_t version = 0; for (int32_t i = 0; i < numOfMsgs; ++i) { #if 1 @@ -191,11 +193,22 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO rsp.code = 0; rsp.pCont = NULL; rsp.contLen = 0; - if (vnodeProcessWriteReq(pVnode->pImpl, &pMsg->rpcMsg, version++, &rsp) < 0) { + + // get original rpc msg + assert(pMsg->rpcMsg.msgType == TDMT_VND_SYNC_APPLY_MSG); + SyncApplyMsg *pSyncApplyMsg = syncApplyMsgFromRpcMsg2(&pMsg->rpcMsg); + syncApplyMsgLog2("==vmProcessApplyQueue==", pSyncApplyMsg); + SRpcMsg originalRpcMsg; + syncApplyMsg2OriginalRpcMsg(pSyncApplyMsg, &originalRpcMsg); + + if (vnodeProcessWriteReq(pVnode->pImpl, &originalRpcMsg, pSyncApplyMsg->fsmMeta.index, &rsp) < 0) { rsp.code = terrno; dTrace("vnodeProcessWriteReq error, code:%d", terrno); } + syncApplyMsgDestroy(pSyncApplyMsg); + rpcFreeCont(originalRpcMsg.pCont); + if (pMsg->rpcMsg.handle != NULL && pMsg->rpcMsg.ahandle != NULL) { rsp.ahandle = pMsg->rpcMsg.ahandle; rsp.handle = pMsg->rpcMsg.handle; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 4db483851e..93c40aa019 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -107,14 +107,21 @@ void vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cb pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); - SRpcMsg applyMsg; - applyMsg = *pMsg; - applyMsg.pCont = rpcMallocCont(applyMsg.contLen); - assert(applyMsg.contLen == pMsg->contLen); - memcpy(applyMsg.pCont, pMsg->pCont, applyMsg.contLen); + SVnode *pVnode = (SVnode *)(pFsm->data); + SyncApplyMsg *pSyncApplyMsg = syncApplyMsgBuild2(pMsg, pVnode->config.vgId, &cbMeta); + SRpcMsg applyMsg; + syncApplyMsg2RpcMsg(pSyncApplyMsg, &applyMsg); + syncApplyMsgDestroy(pSyncApplyMsg); + + /* + SRpcMsg applyMsg; + applyMsg = *pMsg; + applyMsg.pCont = rpcMallocCont(applyMsg.contLen); + assert(applyMsg.contLen == pMsg->contLen); + memcpy(applyMsg.pCont, pMsg->pCont, applyMsg.contLen); + */ // recover handle for response - SVnode *pVnode = (SVnode *)(pFsm->data); SRpcMsg saveRpcMsg; int32_t ret = syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &saveRpcMsg); if (ret == 1 && cbMeta.state == TAOS_SYNC_STATE_LEADER) { From d7edb105f263538b0dee13abbff3838845bccd75 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 20 Apr 2022 15:05:14 +0800 Subject: [PATCH 18/83] add SyncApplyMsg into vmProcessApplyQueue --- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 9e42f8dfb6..b3afba51c0 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -190,6 +190,7 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO taosGetQitem(qall, (void **)&pMsg); + // init response rpc msg rsp.code = 0; rsp.pCont = NULL; rsp.contLen = 0; @@ -201,6 +202,7 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO SRpcMsg originalRpcMsg; syncApplyMsg2OriginalRpcMsg(pSyncApplyMsg, &originalRpcMsg); + // apply data into tsdb if (vnodeProcessWriteReq(pVnode->pImpl, &originalRpcMsg, pSyncApplyMsg->fsmMeta.index, &rsp) < 0) { rsp.code = terrno; dTrace("vnodeProcessWriteReq error, code:%d", terrno); @@ -209,6 +211,7 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO syncApplyMsgDestroy(pSyncApplyMsg); rpcFreeCont(originalRpcMsg.pCont); + // if leader, send response if (pMsg->rpcMsg.handle != NULL && pMsg->rpcMsg.ahandle != NULL) { rsp.ahandle = pMsg->rpcMsg.ahandle; rsp.handle = pMsg->rpcMsg.handle; From eef910f7e2c820c81b68ed5d1308973e47d51ce0 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 20 Apr 2022 16:33:20 +0800 Subject: [PATCH 19/83] sync integration vnodeCfg cJson->SJson --- source/dnode/vnode/src/vnd/vnodeCfg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 0f8bab7856..714497e786 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -141,7 +141,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { assert(arraySize == pCfg->syncCfg.replicaNum); for (int i = 0; i < arraySize; ++i) { - cJSON *pNodeInfo = tjsonGetArrayItem(pNodeInfoArr, i); + SJson *pNodeInfo = tjsonGetArrayItem(pNodeInfoArr, i); assert(pNodeInfo != NULL); tjsonGetNumberValue(pNodeInfo, "nodePort", (pCfg->syncCfg.nodeInfo)[i].nodePort); tjsonGetStringValue(pNodeInfo, "nodeFqdn", (pCfg->syncCfg.nodeInfo)[i].nodeFqdn); From bf3fd9b9c3c0eca158ad4a030dffd4a1a90c24b1 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 20 Apr 2022 16:35:51 +0800 Subject: [PATCH 20/83] [test: modify cases] --- tests/script/tsim/sync/oneReplica1VgElect.sim | 1 - tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim | 1 - tests/script/tsim/sync/threeReplica1VgElect.sim | 1 - tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim | 1 - 4 files changed, 4 deletions(-) diff --git a/tests/script/tsim/sync/oneReplica1VgElect.sim b/tests/script/tsim/sync/oneReplica1VgElect.sim index 9726c0fdf5..8e99bc4b2e 100644 --- a/tests/script/tsim/sync/oneReplica1VgElect.sim +++ b/tests/script/tsim/sync/oneReplica1VgElect.sim @@ -203,7 +203,6 @@ if $data[0][4] == LEADER then $dnodeId = $data[0][3] elif $data[0][6] == LEADER then $dnodeId = $data[0][5] -endi elif $data[0][8] == LEADER then $dnodeId = $data[0][7] else diff --git a/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim b/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim index 972447ba9b..e85d5ea437 100644 --- a/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim +++ b/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim @@ -212,7 +212,6 @@ if $data[0][4] == LEADER then $dnodeId = $data[0][3] elif $data[0][6] == LEADER then $dnodeId = $data[0][5] -endi elif $data[0][8] == LEADER then $dnodeId = $data[0][7] else diff --git a/tests/script/tsim/sync/threeReplica1VgElect.sim b/tests/script/tsim/sync/threeReplica1VgElect.sim index f709c72dd7..e06bd86daa 100644 --- a/tests/script/tsim/sync/threeReplica1VgElect.sim +++ b/tests/script/tsim/sync/threeReplica1VgElect.sim @@ -203,7 +203,6 @@ if $data[0][4] == LEADER then $dnodeId = $data[0][3] elif $data[0][6] == LEADER then $dnodeId = $data[0][5] -endi elif $data[0][8] == LEADER then $dnodeId = $data[0][7] else diff --git a/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim b/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim index d934e82b98..797baea811 100644 --- a/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim +++ b/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim @@ -212,7 +212,6 @@ if $data[0][4] == LEADER then $dnodeId = $data[0][3] elif $data[0][6] == LEADER then $dnodeId = $data[0][5] -endi elif $data[0][8] == LEADER then $dnodeId = $data[0][7] else From a1a498a19adb8d8a1ff19224892f5900cec26cba Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 20 Apr 2022 16:56:00 +0800 Subject: [PATCH 21/83] sync integration add error code --- include/util/taoserror.h | 4 ++++ source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 4 ++-- source/libs/sync/test/syncApplyMsgTest.cpp | 16 ++++++++-------- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index c5b477343d..93ab133dc1 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -407,6 +407,10 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909) #define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) +// sync integration +#define TSDB_CODE_SYN_NOT_LEADER TAOS_DEF_ERROR_CODE(0, 0x0910) +#define TSDB_CODE_SYN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x09FF) + // tq #define TSDB_CODE_TQ_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0A00) #define TSDB_CODE_TQ_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0A01) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index b3afba51c0..206f53d36b 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -153,10 +153,10 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pRpc, false); if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { - rsp.code = -1; + rsp.code = TSDB_CODE_SYN_NOT_LEADER; tmsgSendRsp(&rsp); } else if (ret == TAOS_SYNC_PROPOSE_OTHER_ERROR) { - rsp.code = -2; + rsp.code = TSDB_CODE_SYN_INTERNAL_ERROR; tmsgSendRsp(&rsp); } else if (ret == TAOS_SYNC_PROPOSE_SUCCESS) { // ok diff --git a/source/libs/sync/test/syncApplyMsgTest.cpp b/source/libs/sync/test/syncApplyMsgTest.cpp index 7544613c5d..30019f71e5 100644 --- a/source/libs/sync/test/syncApplyMsgTest.cpp +++ b/source/libs/sync/test/syncApplyMsgTest.cpp @@ -42,8 +42,8 @@ void test1() { void test2() { SyncApplyMsg *pMsg = createMsg(); - uint32_t len = pMsg->bytes; - char * serialized = (char *)taosMemoryMalloc(len); + uint32_t len = pMsg->bytes; + char * serialized = (char *)taosMemoryMalloc(len); syncApplyMsgSerialize(pMsg, serialized, len); SyncApplyMsg *pMsg2 = syncApplyMsgBuild(pMsg->dataLen); syncApplyMsgDeserialize(serialized, len, pMsg2); @@ -56,8 +56,8 @@ void test2() { void test3() { SyncApplyMsg *pMsg = createMsg(); - uint32_t len; - char * serialized = syncApplyMsgSerialize2(pMsg, &len); + uint32_t len; + char * serialized = syncApplyMsgSerialize2(pMsg, &len); SyncApplyMsg *pMsg2 = syncApplyMsgDeserialize2(serialized, len); syncApplyMsgLog2((char *)"test3: syncApplyMsgSerialize2 -> syncApplyMsgDeserialize2 ", pMsg2); @@ -68,7 +68,7 @@ void test3() { void test4() { SyncApplyMsg *pMsg = createMsg(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncApplyMsg2RpcMsg(pMsg, &rpcMsg); SyncApplyMsg *pMsg2 = (SyncApplyMsg *)taosMemoryMalloc(rpcMsg.contLen); syncApplyMsgFromRpcMsg(&rpcMsg, pMsg2); @@ -81,7 +81,7 @@ void test4() { void test5() { SyncApplyMsg *pMsg = createMsg(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncApplyMsg2RpcMsg(pMsg, &rpcMsg); SyncApplyMsg *pMsg2 = syncApplyMsgFromRpcMsg2(&rpcMsg); syncApplyMsgLog2((char *)"test5: syncClientRequest2RpcMsg -> syncApplyMsgFromRpcMsg2 ", pMsg2); @@ -93,13 +93,13 @@ void test5() { void test6() { SyncApplyMsg *pMsg = createMsg(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncApplyMsg2RpcMsg(pMsg, &rpcMsg); SyncApplyMsg *pMsg2 = syncApplyMsgFromRpcMsg2(&rpcMsg); SRpcMsg originalRpcMsg; syncApplyMsg2OriginalRpcMsg(pMsg2, &originalRpcMsg); - syncRpcMsgLog2((char*)"test6", &originalRpcMsg); + syncRpcMsgLog2((char *)"test6", &originalRpcMsg); rpcFreeCont(originalRpcMsg.pCont); rpcFreeCont(rpcMsg.pCont); From 6953688761f0e4674cf63f0471482aaf2530b17c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 20 Apr 2022 11:30:18 +0000 Subject: [PATCH 22/83] add commit snapshot --- source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/vnd/vnodeCommit.c | 15 ++++++++++++++- source/dnode/vnode/src/vnd/vnodeOpen.c | 7 ++++--- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 ++ 5 files changed, 22 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 00bcec7be9..08b97ed6c4 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -62,6 +62,7 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); int vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName); int64_t vnodeGetSyncHandle(SVnode *pVnode); +void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot); // meta typedef struct SMeta SMeta; // todo: remove diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 06ec03741c..0027424829 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -74,7 +74,7 @@ typedef struct { // SVState struct SVState { - int64_t processed; + // int64_t processed; int64_t committed; int64_t applied; }; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 55e4c5110a..a034b13fbe 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -158,12 +158,23 @@ int vnodeSyncCommit(SVnode *pVnode) { } static int vnodeCommit(void *arg) { - SVnode *pVnode = (SVnode *)arg; + SVnode *pVnode = (SVnode *)arg; + char dir[TSDB_FILENAME_LEN]; + SVnodeInfo info = {0}; + + snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); + info.config = pVnode->config; + info.state.committed = pVnode->state.applied; + info.state.applied = pVnode->state.applied; + + vnodeSaveInfo(dir, &info); // metaCommit(pVnode->pMeta); tqCommit(pVnode->pTq); tsdbCommit(pVnode->pTsdb); + vnodeCommitInfo(dir, &info); + vnodeBufPoolRecycle(pVnode); tsem_post(&(pVnode->canCommit)); return 0; @@ -185,6 +196,7 @@ static int vnodeEncodeState(const void *pObj, SJson *pJson) { const SVState *pState = (SVState *)pObj; if (tjsonAddIntegerToObject(pJson, "commit version", pState->committed) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "applied version", pState->applied) < 0) return -1; return 0; } @@ -193,6 +205,7 @@ static int vnodeDecodeState(const SJson *pJson, void *pObj) { SVState *pState = (SVState *)pObj; if (tjsonGetNumberValue(pJson, "commit version", pState->committed) < 0) return -1; + if (tjsonGetNumberValue(pJson, "applied version", pState->applied) < 0) return -1; return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index dee54be5ac..91a754815e 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -75,8 +75,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { pVnode->path = (char *)&pVnode[1]; strcpy(pVnode->path, path); pVnode->config = info.config; - pVnode->state.committed = info.state.committed; - pVnode->state.processed = pVnode->state.applied = pVnode->state.committed; + pVnode->state = info.state; pVnode->pTfs = pTfs; pVnode->msgCb = msgCb; @@ -170,4 +169,6 @@ void vnodeClose(SVnode *pVnode) { } } -int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; } \ No newline at end of file +int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; } + +void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) { pSnapshot->lastApplyIndex = pVnode->state.committed; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index fbeb59fe23..56d4ba392c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -23,6 +23,7 @@ static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq); static int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pSubmitReq, SRpcMsg *pRsp); int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) { +#if 0 SNodeMsg *pMsg; SRpcMsg *pRpc; @@ -40,6 +41,7 @@ int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) { walFsync(pVnode->pWal, false); +#endif return 0; } From 250622d1de6d675d0c3813019d595a224c0011e8 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 20 Apr 2022 19:58:36 +0800 Subject: [PATCH 23/83] snapshot from -1 --- source/dnode/vnode/src/vnd/vnodeOpen.c | 2 ++ source/dnode/vnode/src/vnd/vnodeSync.c | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 91a754815e..ecae050427 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -35,6 +35,8 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path); info.config = *pCfg; + info.state.committed = -1; + info.state.applied = -1; if (vnodeSaveInfo(dir, &info) < 0 || vnodeCommitInfo(dir, &info) < 0) { vError("vgId: %d failed to save vnode config since %s", pCfg->vgId, tstrerror(terrno)); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 93c40aa019..546e925106 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -85,9 +85,15 @@ int32_t vnodeSendMsg(void *rpcHandle, const SEpSet *pEpSet, SRpcMsg *pMsg) { } int32_t vnodeSyncGetSnapshotCb(struct SSyncFSM *pFsm, SSnapshot *pSnapshot) { + SVnode *pVnode = (SVnode *)(pFsm->data); + vnodeGetSnapshot(pVnode, pSnapshot); + + /* pSnapshot->data = NULL; pSnapshot->lastApplyIndex = 0; pSnapshot->lastApplyTerm = 0; + */ + return 0; } From a6ee17fd43365cb5db1318c5397059653d071fa5 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 20 Apr 2022 20:26:10 +0800 Subject: [PATCH 24/83] add syncElectTest1.cpp --- source/libs/sync/test/CMakeLists.txt | 14 +++ source/libs/sync/test/syncElectTest1.cpp | 129 +++++++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 source/libs/sync/test/syncElectTest1.cpp diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 5d43d47358..2500cd3c7b 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -28,6 +28,7 @@ add_executable(syncRpcMsgTest "") add_executable(syncPingTimerTest2 "") add_executable(syncPingSelfTest "") add_executable(syncElectTest "") +add_executable(syncElectTest1 "") add_executable(syncElectTest2 "") add_executable(syncElectTest3 "") add_executable(syncEncodeTest "") @@ -164,6 +165,10 @@ target_sources(syncElectTest PRIVATE "syncElectTest.cpp" ) +target_sources(syncElectTest1 + PRIVATE + "syncElectTest1.cpp" +) target_sources(syncElectTest2 PRIVATE "syncElectTest2.cpp" @@ -372,6 +377,11 @@ target_include_directories(syncElectTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncElectTest1 + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_include_directories(syncElectTest2 PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" @@ -564,6 +574,10 @@ target_link_libraries(syncElectTest sync gtest_main ) +target_link_libraries(syncElectTest1 + sync + gtest_main +) target_link_libraries(syncElectTest2 sync gtest_main diff --git a/source/libs/sync/test/syncElectTest1.cpp b/source/libs/sync/test/syncElectTest1.cpp new file mode 100644 index 0000000000..feea95af82 --- /dev/null +++ b/source/libs/sync/test/syncElectTest1.cpp @@ -0,0 +1,129 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 1; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SWal* pWal; +SSyncNode* gSyncNode; +const char* pDir = "./syncElectTest1"; + +SSyncNode* syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s_sync_%d", pDir, myIndex); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + + char tmpdir[128]; + snprintf(tmpdir, sizeof(tmpdir), "%s_wal_%d", pDir, myIndex); + pWal = walOpen(tmpdir, &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->pSyncNode = pSyncNode; + + syncNodeStart(pSyncNode); + + return pSyncNode; +} + +SSyncNode* syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + taosMemoryFree(s); + } +} + +int main(int argc, char** argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + gSyncNode = syncInitTest(); + assert(gSyncNode != NULL); + syncNodeLog2((char*)"", gSyncNode); + + initRaftId(gSyncNode); + + //--------------------------- + while (1) { + char* s = syncNode2SimpleStr(gSyncNode); + sTrace("%s", s); + taosMemoryFree(s); + taosMsleep(1000); + } + + return 0; +} From bbe85695cfd02d380761b4d2d1363e1e7b5b168c Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 20 Apr 2022 20:29:20 +0800 Subject: [PATCH 25/83] add syncMaybeAdvanceCommitIndex, maybe only one replica --- source/libs/sync/src/syncMain.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 6d51cee14e..e260ef4654 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -890,6 +890,7 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { // use this now syncNodeAppendNoop(pSyncNode); + syncMaybeAdvanceCommitIndex(pSyncNode); // maybe only one replica // do not use this // syncNodeEqNoop(pSyncNode); From d2a1203eeb59f142b0de02625c2c73e3fc0062b9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 20 Apr 2022 20:39:19 +0800 Subject: [PATCH 26/83] refactor: fix bugs in to_json --- source/libs/function/src/builtins.c | 2 +- source/libs/scalar/src/sclfunc.c | 8 +++----- source/util/src/tjson.c | 4 ++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 7e9fc7d80e..c04f8888c8 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -409,7 +409,7 @@ static int32_t translateToJson(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_JSON].bytes, .type = TSDB_DATA_TYPE_JSON}; + pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_BINARY].bytes, .type = TSDB_DATA_TYPE_BINARY}; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 8681c1dd47..16a4d768ba 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -887,13 +887,13 @@ int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu return TSDB_CODE_FAILED; } - char *input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[0]; - char *tmp = taosMemoryCalloc(pInput[0].columnData->info.bytes + 1, 1); + char tmp[TSDB_MAX_JSON_TAG_LEN] = {0}; for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { colDataAppendNULL(pOutput->columnData, i); continue; } + char *input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[i]; if(type == TSDB_DATA_TYPE_NCHAR){ if (varDataTLen(input) > TSDB_MAX_JSON_TAG_LEN){ @@ -912,7 +912,7 @@ int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu continue; } memcpy(tmp, varDataVal(input), varDataLen(input)); - tmp[varDataTLen(input)] = 0; + tmp[varDataLen(input)] = 0; } if(!tjsonValidateJson(tmp)){ @@ -921,9 +921,7 @@ int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu } colDataAppend(pOutput->columnData, i, input, false); - input += varDataTLen(input); } - taosMemoryFree(tmp); pOutput->numOfRows = pInput->numOfRows; diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 59dab76989..0b9727e0bf 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -300,8 +300,8 @@ bool tjsonValidateJson(const char *jIn) { char* jsonKey = item->string; if (!jsonKey) return false; - for (size_t j = 0; j < strlen(jsonKey); ++i) { - if (isprint(jsonKey[i]) == 0) return false; + for (size_t j = 0; j < strlen(jsonKey); ++j) { + if (isprint(jsonKey[j]) == 0) return false; } if (item->type == cJSON_Object || item->type == cJSON_Array) { From adf993cdc3bf3badce406f6be83a960f0ce01b95 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 21 Apr 2022 09:25:45 +0800 Subject: [PATCH 27/83] start/stop/restart from dnode --- source/dnode/mgmt/implement/src/dmHandle.c | 94 ++++++++++++++++++++++ source/dnode/mgmt/interface/inc/dmDef.h | 8 ++ 2 files changed, 102 insertions(+) diff --git a/source/dnode/mgmt/implement/src/dmHandle.c b/source/dnode/mgmt/implement/src/dmHandle.c index dd221f8404..2acaaccb43 100644 --- a/source/dnode/mgmt/implement/src/dmHandle.c +++ b/source/dnode/mgmt/implement/src/dmHandle.c @@ -216,6 +216,95 @@ static void dmStopMgmt(SMgmtWrapper *pWrapper) { dmStopStatusThread(pWrapper->pDnode); } +static int32_t dmSpawnUdfd(SDnodeData *pData); + +void dmUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) { + dInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal); + uv_close((uv_handle_t*)process, NULL); + SDnodeData *pData = process->data; + if (atomic_load_8(&pData->udfdStoping) != 0) { + dDebug("udfd process exit due to stopping"); + } else { + dmSpawnUdfd(pData); + } +} + +static int32_t dmSpawnUdfd(SDnodeData *pData) { + dInfo("dnode start spawning udfd"); + uv_process_options_t options = {0}; + + char path[PATH_MAX] = {0}; + size_t cwdSize; + uv_cwd(path, &cwdSize); + strcat(path, "/udfd"); + char* argsUdfd[] = {path, "-c", configDir, NULL}; + options.args = argsUdfd; + options.file = path; + + options.exit_cb = dmUdfdExit; + + options.stdio_count = 3; + uv_stdio_container_t child_stdio[3]; + child_stdio[0].flags = UV_IGNORE; + child_stdio[1].flags = UV_INHERIT_FD; + child_stdio[1].data.fd = 1; + child_stdio[2].flags = UV_INHERIT_FD; + child_stdio[2].data.fd = 2; + options.stdio = child_stdio; + + char dnodeIdEnvItem[32] = {0}; + char thrdPoolSizeEnvItem[32] = {0}; + snprintf(dnodeIdEnvItem, 32, "%s=%d", "DNODE_ID", pData->dnodeId); + float numCpuCores = 4; + taosGetCpuCores(&numCpuCores); + snprintf(thrdPoolSizeEnvItem,32, "%s=%d", "UV_THREADPOOL_SIZE", (int)numCpuCores*2); + char* envUdfd[] = {dnodeIdEnvItem, thrdPoolSizeEnvItem, NULL}; + options.env = envUdfd; + + int err = uv_spawn(&pData->udfdLoop, &pData->udfdProcess, &options); + + pData->udfdProcess.data = (void*)pData; + + if (err != 0) { + dError("can not spawn udfd. path: %s, error: %s", path, uv_strerror(err)); + } + return err; +} + +void dmWatchUdfd(void *args) { + SDnodeData *pData = args; + uv_loop_init(&pData->udfdLoop); + int err = dmSpawnUdfd(pData); + pData->udfdErrCode = err; + uv_barrier_wait(&pData->udfdBarrier); + if (pData->udfdErrCode == 0) { + uv_run(&pData->udfdLoop, UV_RUN_DEFAULT); + } + uv_loop_close(&pData->udfdLoop); + return; +} + +int32_t dmStartUdfd(SDnode *pDnode) { + SDnodeData *pData = &pDnode->data; + uv_barrier_init(&pData->udfdBarrier, 2); + pData->udfdStoping = 0; + uv_thread_create(&pData->udfdThread, dmWatchUdfd, pData); + uv_barrier_wait(&pData->udfdBarrier); + return pData->udfdErrCode; +} + +int32_t dmStopUdfd(SDnode *pDnode) { + SDnodeData *pData = &pDnode->data; + atomic_store_8(&pData->udfdStoping, 1); + + uv_barrier_destroy(&pData->udfdBarrier); + uv_process_kill(&pData->udfdProcess, SIGINT); + uv_thread_join(&pData->udfdThread); + + atomic_store_8(&pData->udfdStoping, 0); + return 0; +} + static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) { dInfo("dnode-mgmt start to init"); SDnode *pDnode = pWrapper->pDnode; @@ -247,6 +336,10 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) { } dmReportStartup(pDnode, "dnode-transport", "initialized"); + if (dmStartUdfd(pDnode) != 0) { + dError("failed to start udfd"); + } + dInfo("dnode-mgmt is initialized"); return 0; } @@ -254,6 +347,7 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) { static void dmCleanupMgmt(SMgmtWrapper *pWrapper) { dInfo("dnode-mgmt start to clean up"); SDnode *pDnode = pWrapper->pDnode; + dmStopUdfd(pDnode); dmStopWorker(pDnode); taosWLockLatch(&pDnode->data.latch); diff --git a/source/dnode/mgmt/interface/inc/dmDef.h b/source/dnode/mgmt/interface/inc/dmDef.h index e6537dcf73..fdce59b4df 100644 --- a/source/dnode/mgmt/interface/inc/dmDef.h +++ b/source/dnode/mgmt/interface/inc/dmDef.h @@ -16,6 +16,7 @@ #ifndef _TD_DM_DEF_H_ #define _TD_DM_DEF_H_ +#include "uv.h" #include "dmLog.h" #include "cJSON.h" @@ -135,6 +136,13 @@ typedef struct { int32_t numOfDisks; int32_t supportVnodes; uint16_t serverPort; + + uv_loop_t udfdLoop; + uv_thread_t udfdThread; + uv_barrier_t udfdBarrier; + uv_process_t udfdProcess; + int udfdErrCode; + int8_t udfdStoping; } SDnodeData; typedef struct { From b4d3008da3f7887c792917764ef351889462b194 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 21 Apr 2022 09:45:37 +0800 Subject: [PATCH 28/83] remove udfd start/stop from tudf.c --- source/libs/function/inc/tudf.h | 13 +------ source/libs/function/src/tudf.c | 68 ++------------------------------- 2 files changed, 5 insertions(+), 76 deletions(-) diff --git a/source/libs/function/inc/tudf.h b/source/libs/function/inc/tudf.h index c51b6e1264..4d3319f223 100644 --- a/source/libs/function/inc/tudf.h +++ b/source/libs/function/inc/tudf.h @@ -27,7 +27,7 @@ extern "C" { #endif #define UDF_LISTEN_PIPE_NAME_LEN 32 -#define UDF_LISTEN_PIPE_NAME_PREFIX "udf.sock." +#define UDF_LISTEN_PIPE_NAME_PREFIX "udfd.sock." //====================================================================================== //begin API to taosd and qworker @@ -38,17 +38,6 @@ enum { UDFC_CODE_PIPE_READ_ERR = -3, }; -/*TODO: no api for dnode startudfd/stopudfd*/ -/** - * start udfd dameon service - */ -int32_t startUdfd(int32_t dnodeId); - -/** - * stop udfd dameon service - */ -int32_t stopUdfd(int32_t dnodeId); - /** * create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf * @return error code diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index ad74daddc6..7c6f38933c 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -20,13 +20,10 @@ #include "tarray.h" #include "tdatablock.h" -//TODO: when startup, set thread poll size. add it to cfg -//TODO: test for udfd restart -//TODO: udfd restart when exist or aborts -//TODO: deal with uv task that has been started and then udfd core dumped //TODO: network error processing. //TODO: add unit test //TODO: include all global variable under context struct + /* Copyright (c) 2013, Ben Noordhuis * The QUEUE is copied from queue.h under libuv * */ @@ -185,8 +182,6 @@ typedef struct SClientUvConn { SClientConnBuf readBuf; } SClientUvConn; -uv_process_t gUdfdProcess; - uv_barrier_t gUdfInitBarrier; uv_loop_t gUdfdLoop; @@ -202,7 +197,6 @@ enum { UDFC_STATE_INITAL = 0, // initial state UDFC_STATE_STARTNG, // starting after createUdfdProxy UDFC_STATE_READY, // started and begin to receive quests - UDFC_STATE_RESTARTING, // udfd abnormal exit. cleaning up and restart. UDFC_STATE_STOPPING, // stopping after destroyUdfdProxy UDFC_STATUS_FINAL, // stopped }; @@ -902,8 +896,6 @@ void cleanUpUvTasks() { SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, recvTaskQueue); if (gUdfcState == UDFC_STATE_STOPPING) { task->errCode = UDFC_CODE_STOPPING; - } else if (gUdfcState == UDFC_STATE_RESTARTING) { - task->errCode = UDFC_CODE_RESTARTING; } uv_sem_post(&task->taskSem); } @@ -915,8 +907,6 @@ void cleanUpUvTasks() { SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, procTaskQueue); if (gUdfcState == UDFC_STATE_STOPPING) { task->errCode = UDFC_CODE_STOPPING; - } else if (gUdfcState == UDFC_STATE_RESTARTING) { - task->errCode = UDFC_CODE_RESTARTING; } uv_sem_post(&task->taskSem); } @@ -929,53 +919,6 @@ void udfStopAsyncCb(uv_async_t *async) { } } -int32_t udfcSpawnUdfd(); - -void onUdfdExit(uv_process_t *req, int64_t exit_status, int term_signal) { - //TODO: pipe close will be first received - debugPrint("Process exited with status %" PRId64 ", signal %d", exit_status, term_signal); - uv_close((uv_handle_t *) req, NULL); - //TODO: restart the udfd process - if (gUdfcState == UDFC_STATE_STOPPING) { - if (term_signal != SIGINT) { - //TODO: log error - } - } - if (gUdfcState == UDFC_STATE_READY) { - gUdfcState = UDFC_STATE_RESTARTING; - //TODO: asynchronous without blocking. how to do it - //cleanUpUvTasks(); - udfcSpawnUdfd(); - } -} - -int32_t udfcSpawnUdfd() { - //TODO: path - uv_process_options_t options = {0}; - static char path[256] = {0}; - size_t cwdSize; - uv_cwd(path, &cwdSize); - strcat(path, "/udfd"); - char* args[2] = {path, NULL}; - options.args = args; - options.file = path; - options.exit_cb = onUdfdExit; - options.stdio_count = 3; - uv_stdio_container_t child_stdio[3]; - child_stdio[0].flags = UV_IGNORE; - child_stdio[1].flags = UV_INHERIT_FD; - child_stdio[1].data.fd = 1; - child_stdio[2].flags = UV_INHERIT_FD; - child_stdio[2].data.fd = 2; - options.stdio = child_stdio; - //TODO spawn error - int err = uv_spawn(&gUdfdLoop, &gUdfdProcess, &options); - if (err != 0) { - debugPrint("can not spawn udfd. path: %s, error: %s", path, uv_strerror(err)); - } - return err; -} - void constructUdfService(void *argsThread) { uv_loop_init(&gUdfdLoop); @@ -990,24 +933,21 @@ void constructUdfService(void *argsThread) { uv_loop_close(&gUdfdLoop); } - int32_t createUdfdProxy(int32_t dnodeId) { gUdfcState = UDFC_STATE_STARTNG; uv_barrier_init(&gUdfInitBarrier, 2); uv_thread_create(&gUdfLoopThread, constructUdfService, 0); - uv_barrier_wait(&gUdfInitBarrier); gUdfcState = UDFC_STATE_READY; + uv_barrier_wait(&gUdfInitBarrier); + gUdfcState = UDFC_STATE_READY; return 0; } int32_t destroyUdfdProxy(int32_t dnodeId) { gUdfcState = UDFC_STATE_STOPPING; - uv_barrier_destroy(&gUdfInitBarrier); -// if (gUdfcState == UDFC_STATE_STOPPING) { -// uv_process_kill(&gUdfdProcess, SIGINT); -// } uv_async_send(&gUdfLoopStopAsync); uv_thread_join(&gUdfLoopThread); uv_mutex_destroy(&gUdfTaskQueueMutex); + uv_barrier_destroy(&gUdfInitBarrier); gUdfcState = UDFC_STATUS_FINAL; return 0; } From d044a4ee8195fa6e838ae53cad5c23a533d8da7c Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 21 Apr 2022 11:21:27 +0800 Subject: [PATCH 29/83] fix bug that udfd can not be spawned --- source/dnode/mgmt/implement/src/dmHandle.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mgmt/implement/src/dmHandle.c b/source/dnode/mgmt/implement/src/dmHandle.c index 2acaaccb43..df1ab79b99 100644 --- a/source/dnode/mgmt/implement/src/dmHandle.c +++ b/source/dnode/mgmt/implement/src/dmHandle.c @@ -233,10 +233,7 @@ static int32_t dmSpawnUdfd(SDnodeData *pData) { dInfo("dnode start spawning udfd"); uv_process_options_t options = {0}; - char path[PATH_MAX] = {0}; - size_t cwdSize; - uv_cwd(path, &cwdSize); - strcat(path, "/udfd"); + char path[] = "udfd"; char* argsUdfd[] = {path, "-c", configDir, NULL}; options.args = argsUdfd; options.file = path; @@ -295,6 +292,9 @@ int32_t dmStartUdfd(SDnode *pDnode) { int32_t dmStopUdfd(SDnode *pDnode) { SDnodeData *pData = &pDnode->data; + if (pData->udfdErrCode != 0) { + return 0; + } atomic_store_8(&pData->udfdStoping, 1); uv_barrier_destroy(&pData->udfdBarrier); From 0df5100836c0246cdd06defd99db581c1fbf00ac Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 21 Apr 2022 11:25:17 +0800 Subject: [PATCH 30/83] add syncLogStoreCheck.cpp --- source/libs/sync/test/syncLogStoreCheck.cpp | 96 +++++++-------------- 1 file changed, 33 insertions(+), 63 deletions(-) diff --git a/source/libs/sync/test/syncLogStoreCheck.cpp b/source/libs/sync/test/syncLogStoreCheck.cpp index b77a744c30..ae725ac5a2 100644 --- a/source/libs/sync/test/syncLogStoreCheck.cpp +++ b/source/libs/sync/test/syncLogStoreCheck.cpp @@ -16,90 +16,60 @@ void logTest() { sFatal("--- sync log test: fatal"); } -uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; -int32_t replicaNum = 1; -int32_t myIndex = 0; - -SRaftId ids[TSDB_MAX_REPLICA]; -SSyncInfo syncInfo; -SSyncFSM* pFsm; -SWal* pWal; -SSyncNode* pSyncNode; - -SSyncNode* syncNodeInit(const char* path) { - syncInfo.vgId = 1234; - syncInfo.rpcClient = gSyncIO->clientRpc; - syncInfo.FpSendMsg = syncIOSendMsg; - syncInfo.queue = gSyncIO->pMsgQ; - syncInfo.FpEqMsg = syncIOEqMsg; - syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./log_check"); - +void init() { int code = walInit(); assert(code == 0); +} + +void cleanup() { walCleanUp(); } + +SWal* createWal(char* path, int32_t vgId) { SWalCfg walCfg; memset(&walCfg, 0, sizeof(SWalCfg)); - walCfg.vgId = syncInfo.vgId; + walCfg.vgId = vgId; walCfg.fsyncPeriod = 1000; walCfg.retentionPeriod = 1000; walCfg.rollPeriod = 1000; walCfg.retentionSize = 1000; walCfg.segSize = 1000; walCfg.level = TAOS_WAL_FSYNC; - pWal = walOpen(path, &walCfg); + SWal* pWal = walOpen(path, &walCfg); assert(pWal != NULL); + return pWal; +} - syncInfo.pWal = pWal; - - SSyncCfg* pCfg = &syncInfo.syncCfg; - pCfg->myIndex = myIndex; - pCfg->replicaNum = replicaNum; - - for (int i = 0; i < replicaNum; ++i) { - pCfg->nodeInfo[i].nodePort = ports[i]; - snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); - } - - pSyncNode = syncNodeOpen(&syncInfo); - assert(pSyncNode != NULL); - - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; - gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; - gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; - gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; - gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; - gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; - gSyncIO->pSyncNode = pSyncNode; - +SSyncNode* createSyncNode(SWal* pWal) { + SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); + memset(pSyncNode, 0, sizeof(SSyncNode)); + pSyncNode->pWal = pWal; return pSyncNode; } -SSyncNode* logStoreCheck(const char* path) { return syncNodeInit(path); } +void usage(char* exe) { printf("usage: %s path vgId \n", exe); } int main(int argc, char** argv) { - // taosInitLog((char *)"syncTest.log", 100000, 10); - tsAsyncLog = 0; - sDebugFlag = 143 + 64; - - myIndex = 0; - if (argc >= 2) { - myIndex = atoi(argv[1]); + if (argc != 3) { + usage(argv[0]); + exit(-1); } + char* path = argv[1]; + int32_t vgId = atoi(argv[2]); - int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); - assert(ret == 0); - - ret = syncEnvStart(); - assert(ret == 0); - - pSyncNode = logStoreCheck(argv[1]); + init(); + SWal* pWal = createWal(path, vgId); + assert(pWal != NULL); + SSyncNode* pSyncNode = createSyncNode(pWal); assert(pSyncNode != NULL); - logStoreLog2((char*)"logStoreCheck", pSyncNode->pLogStore); + SSyncLogStore* pLog = logStoreCreate(pSyncNode); + assert(pLog != NULL); + logStorePrint2((char*)"==syncLogStoreCheck==", pLog); + + walClose(pWal); + logStoreDestory(pLog); + taosMemoryFree(pSyncNode); + + cleanup(); return 0; } From 672be6f4ff9125e32b0f051bef07699cbd59af9b Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 21 Apr 2022 11:35:22 +0800 Subject: [PATCH 31/83] del trace log flag --- source/libs/sync/src/syncEnv.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index 4213d7b45d..941478c9d6 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -27,8 +27,6 @@ static void syncEnvTick(void *param, void *tmrId); // -------------------------------- int32_t syncEnvStart() { - sDebugFlag = 207; - int32_t ret = 0; taosSeedRand(taosGetTimestampSec()); gSyncEnv = doSyncEnvStart(gSyncEnv); From a018c70117faefc34cd7805a67e214ec4a3301fd Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 21 Apr 2022 13:10:46 +0800 Subject: [PATCH 32/83] udfd proxy init and close for udf calling --- source/libs/function/inc/tudf.h | 23 ++-- source/libs/function/inc/udfc.h | 10 +- source/libs/function/src/tudf.c | 165 ++++++++++++++++------------- source/libs/function/src/udfd.c | 10 +- source/libs/function/test/runUdf.c | 9 +- 5 files changed, 117 insertions(+), 100 deletions(-) diff --git a/source/libs/function/inc/tudf.h b/source/libs/function/inc/tudf.h index 4d3319f223..b5c839c811 100644 --- a/source/libs/function/inc/tudf.h +++ b/source/libs/function/inc/tudf.h @@ -34,23 +34,24 @@ extern "C" { enum { UDFC_CODE_STOPPING = -1, - UDFC_CODE_RESTARTING = -2, UDFC_CODE_PIPE_READ_ERR = -3, }; +typedef void *UdfcHandle; +typedef void *UdfcFuncHandle; + /** * create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf * @return error code */ -int32_t createUdfdProxy(int32_t dnodeId); +int32_t udfcOpen(int32_t dnodeId, UdfcHandle* proxyHandle); /** * destroy udfd proxy * @return error code */ -int32_t destroyUdfdProxy(int32_t dnodeId); +int32_t udfcClose(UdfcHandle proxyhandle); -typedef void *UdfHandle; /** * setup udf @@ -58,7 +59,7 @@ typedef void *UdfHandle; * @param handle, out * @return error code */ -int32_t setupUdf(char udfName[], SEpSet *epSet, UdfHandle *handle); +int32_t setupUdf(UdfcHandle proxyHandle, char udfName[], SEpSet *epSet, UdfcFuncHandle *handle); typedef struct SUdfColumnMeta { int16_t type; @@ -105,26 +106,26 @@ typedef struct SUdfInterBuf { } SUdfInterBuf; // output: interBuf -int32_t callUdfAggInit(UdfHandle handle, SUdfInterBuf *interBuf); +int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf); // input: block, state // output: newState -int32_t callUdfAggProcess(UdfHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState); +int32_t callUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState); // input: interBuf // output: resultData -int32_t callUdfAggFinalize(UdfHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData); +int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData); // input: interbuf1, interbuf2 // output: resultBuf -int32_t callUdfAggMerge(UdfHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf); +int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf); // input: block // output: resultData -int32_t callUdfScalaProcess(UdfHandle handle, SSDataBlock *block, SSDataBlock *resultData); +int32_t callUdfScalaProcess(UdfcFuncHandle handle, SSDataBlock *block, SSDataBlock *resultData); /** * tearn down udf * @param handle * @return */ -int32_t teardownUdf(UdfHandle handle); +int32_t teardownUdf(UdfcFuncHandle handle); // end API to taosd and qworker //============================================================================================================================= diff --git a/source/libs/function/inc/udfc.h b/source/libs/function/inc/udfc.h index fed2818ced..1cf8c67686 100644 --- a/source/libs/function/inc/udfc.h +++ b/source/libs/function/inc/udfc.h @@ -30,20 +30,20 @@ typedef struct SUdfInfo { char *path; } SUdfInfo; -typedef void *UdfHandle; +typedef void *UdfcFuncHandle; int32_t createUdfdProxy(); int32_t destroyUdfdProxy(); -//int32_t setupUdf(SUdfInfo *udf, int32_t numOfUdfs, UdfHandle *handles); +//int32_t setupUdf(SUdfInfo *udf, int32_t numOfUdfs, UdfcFuncHandle *handles); -int32_t setupUdf(SUdfInfo* udf, UdfHandle* handle); +int32_t setupUdf(SUdfInfo* udf, UdfcFuncHandle* handle); -int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SSDataBlock input, char **newstate, +int32_t callUdf(UdfcFuncHandle handle, int8_t step, char *state, int32_t stateSize, SSDataBlock input, char **newstate, int32_t *newStateSize, SSDataBlock *output); -int32_t teardownUdf(UdfHandle handle); +int32_t teardownUdf(UdfcFuncHandle handle); typedef struct SUdfSetupRequest { char udfName[16]; // diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 7c6f38933c..e31a860e85 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -14,7 +14,6 @@ */ #include "uv.h" #include "os.h" -#include "tlog.h" #include "tudf.h" #include "tudfInt.h" #include "tarray.h" @@ -122,12 +121,35 @@ enum { UV_TASK_DISCONNECT = 2 }; +int64_t gUdfTaskSeqNum = 0; +typedef struct SUdfdProxy { + int32_t dnodeId; + uv_barrier_t gUdfInitBarrier; + + uv_loop_t gUdfdLoop; + uv_thread_t gUdfLoopThread; + uv_async_t gUdfLoopTaskAync; + + uv_async_t gUdfLoopStopAsync; + + uv_mutex_t gUdfTaskQueueMutex; + int8_t gUdfcState; + QUEUE gUdfTaskQueue; + QUEUE gUvProcTaskQueue; + // int8_t gUdfcState = UDFC_STATE_INITAL; + // QUEUE gUdfTaskQueue = {0}; + // QUEUE gUvProcTaskQueue = {0}; +} SUdfdProxy; + + typedef struct SUdfUvSession { + SUdfdProxy *udfc; int64_t severHandle; uv_pipe_t *udfSvcPipe; } SUdfUvSession; typedef struct SClientUvTaskNode { + SUdfdProxy *udfc; int8_t type; int errCode; @@ -166,7 +188,6 @@ typedef struct SClientUdfTask { } _teardown; }; - } SClientUdfTask; typedef struct SClientConnBuf { @@ -182,31 +203,13 @@ typedef struct SClientUvConn { SClientConnBuf readBuf; } SClientUvConn; -uv_barrier_t gUdfInitBarrier; - -uv_loop_t gUdfdLoop; -uv_thread_t gUdfLoopThread; -uv_async_t gUdfLoopTaskAync; - -uv_async_t gUdfLoopStopAsync; - -uv_mutex_t gUdfTaskQueueMutex; -int64_t gUdfTaskSeqNum = 0; - enum { UDFC_STATE_INITAL = 0, // initial state - UDFC_STATE_STARTNG, // starting after createUdfdProxy + UDFC_STATE_STARTNG, // starting after udfcOpen UDFC_STATE_READY, // started and begin to receive quests - UDFC_STATE_STOPPING, // stopping after destroyUdfdProxy + UDFC_STATE_STOPPING, // stopping after udfcClose UDFC_STATUS_FINAL, // stopped }; -int8_t gUdfcState = UDFC_STATE_INITAL; - -//double circular linked list - -QUEUE gUdfTaskQueue = {0}; - -QUEUE gUvProcTaskQueue = {0}; int32_t encodeUdfSetupRequest(void **buf, const SUdfSetupRequest *setup) { int32_t len = 0; @@ -771,13 +774,14 @@ void onUdfClientConnect(uv_connect_t *connect, int status) { int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskNode **pUvTask) { SClientUvTaskNode *uvTask = taosMemoryCalloc(1, sizeof(SClientUvTaskNode)); uvTask->type = uvTaskType; + uvTask->udfc = task->session->udfc; if (uvTaskType == UV_TASK_CONNECT) { } else if (uvTaskType == UV_TASK_REQ_RSP) { uvTask->pipe = task->session->udfSvcPipe; SUdfRequest request; request.type = task->type; - request.seqNum = gUdfTaskSeqNum++; + request.seqNum = atomic_fetch_add_64(&gUdfTaskSeqNum, 1); if (task->type == UDF_TASK_SETUP) { request.setup = task->_setup.req; @@ -809,11 +813,11 @@ int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskN int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) { debugPrint("%s, %d", "queue uv task", uvTask->type); - - uv_mutex_lock(&gUdfTaskQueueMutex); - QUEUE_INSERT_TAIL(&gUdfTaskQueue, &uvTask->recvTaskQueue); - uv_mutex_unlock(&gUdfTaskQueueMutex); - uv_async_send(&gUdfLoopTaskAync); + SUdfdProxy *udfc = uvTask->udfc; + uv_mutex_lock(&udfc->gUdfTaskQueueMutex); + QUEUE_INSERT_TAIL(&udfc->gUdfTaskQueue, &uvTask->recvTaskQueue); + uv_mutex_unlock(&udfc->gUdfTaskQueueMutex); + uv_async_send(&udfc->gUdfLoopTaskAync); uv_sem_wait(&uvTask->taskSem); uv_sem_destroy(&uvTask->taskSem); @@ -826,7 +830,7 @@ int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { switch (uvTask->type) { case UV_TASK_CONNECT: { uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t)); - uv_pipe_init(&gUdfdLoop, pipe, 0); + uv_pipe_init(&uvTask->udfc->gUdfdLoop, pipe, 0); uvTask->pipe = pipe; SClientUvConn *conn = taosMemoryMalloc(sizeof(SClientUvConn)); @@ -867,45 +871,46 @@ int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { } void udfClientAsyncCb(uv_async_t *async) { + SUdfdProxy *udfc = async->data; QUEUE wq; - uv_mutex_lock(&gUdfTaskQueueMutex); - QUEUE_MOVE(&gUdfTaskQueue, &wq); - uv_mutex_unlock(&gUdfTaskQueueMutex); + uv_mutex_lock(&udfc->gUdfTaskQueueMutex); + QUEUE_MOVE(&udfc->gUdfTaskQueue, &wq); + uv_mutex_unlock(&udfc->gUdfTaskQueueMutex); while (!QUEUE_EMPTY(&wq)) { QUEUE* h = QUEUE_HEAD(&wq); QUEUE_REMOVE(h); SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, recvTaskQueue); startUvUdfTask(task); - QUEUE_INSERT_TAIL(&gUvProcTaskQueue, &task->procTaskQueue); + QUEUE_INSERT_TAIL(&udfc->gUvProcTaskQueue, &task->procTaskQueue); } } -void cleanUpUvTasks() { +void cleanUpUvTasks(SUdfdProxy *udfc) { QUEUE wq; - uv_mutex_lock(&gUdfTaskQueueMutex); - QUEUE_MOVE(&gUdfTaskQueue, &wq); - uv_mutex_unlock(&gUdfTaskQueueMutex); + uv_mutex_lock(&udfc->gUdfTaskQueueMutex); + QUEUE_MOVE(&udfc->gUdfTaskQueue, &wq); + uv_mutex_unlock(&udfc->gUdfTaskQueueMutex); while (!QUEUE_EMPTY(&wq)) { QUEUE* h = QUEUE_HEAD(&wq); QUEUE_REMOVE(h); SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, recvTaskQueue); - if (gUdfcState == UDFC_STATE_STOPPING) { + if (udfc->gUdfcState == UDFC_STATE_STOPPING) { task->errCode = UDFC_CODE_STOPPING; } uv_sem_post(&task->taskSem); } // TODO: deal with tasks that are waiting result. - while (!QUEUE_EMPTY(&gUvProcTaskQueue)) { - QUEUE* h = QUEUE_HEAD(&gUvProcTaskQueue); + while (!QUEUE_EMPTY(&udfc->gUvProcTaskQueue)) { + QUEUE* h = QUEUE_HEAD(&udfc->gUvProcTaskQueue); QUEUE_REMOVE(h); SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, procTaskQueue); - if (gUdfcState == UDFC_STATE_STOPPING) { + if (udfc->gUdfcState == UDFC_STATE_STOPPING) { task->errCode = UDFC_CODE_STOPPING; } uv_sem_post(&task->taskSem); @@ -913,42 +918,51 @@ void cleanUpUvTasks() { } void udfStopAsyncCb(uv_async_t *async) { - cleanUpUvTasks(); - if (gUdfcState == UDFC_STATE_STOPPING) { - uv_stop(&gUdfdLoop); + SUdfdProxy *udfc = async->data; + cleanUpUvTasks(udfc); + if (udfc->gUdfcState == UDFC_STATE_STOPPING) { + uv_stop(&udfc->gUdfdLoop); } } void constructUdfService(void *argsThread) { - uv_loop_init(&gUdfdLoop); + SUdfdProxy *udfc = (SUdfdProxy*)argsThread; + uv_loop_init(&udfc->gUdfdLoop); - uv_async_init(&gUdfdLoop, &gUdfLoopTaskAync, udfClientAsyncCb); - uv_async_init(&gUdfdLoop, &gUdfLoopStopAsync, udfStopAsyncCb); - uv_mutex_init(&gUdfTaskQueueMutex); - QUEUE_INIT(&gUdfTaskQueue); - QUEUE_INIT(&gUvProcTaskQueue); - uv_barrier_wait(&gUdfInitBarrier); + uv_async_init(&udfc->gUdfdLoop, &udfc->gUdfLoopTaskAync, udfClientAsyncCb); + udfc->gUdfLoopTaskAync.data = udfc; + uv_async_init(&udfc->gUdfdLoop, &udfc->gUdfLoopStopAsync, udfStopAsyncCb); + udfc->gUdfLoopStopAsync.data = udfc; + uv_mutex_init(&udfc->gUdfTaskQueueMutex); + QUEUE_INIT(&udfc->gUdfTaskQueue); + QUEUE_INIT(&udfc->gUvProcTaskQueue); + uv_barrier_wait(&udfc->gUdfInitBarrier); //TODO return value of uv_run - uv_run(&gUdfdLoop, UV_RUN_DEFAULT); - uv_loop_close(&gUdfdLoop); + uv_run(&udfc->gUdfdLoop, UV_RUN_DEFAULT); + uv_loop_close(&udfc->gUdfdLoop); } -int32_t createUdfdProxy(int32_t dnodeId) { - gUdfcState = UDFC_STATE_STARTNG; - uv_barrier_init(&gUdfInitBarrier, 2); - uv_thread_create(&gUdfLoopThread, constructUdfService, 0); - uv_barrier_wait(&gUdfInitBarrier); - gUdfcState = UDFC_STATE_READY; +int32_t udfcOpen(int32_t dnodeId, UdfcHandle *udfc) { + SUdfdProxy *proxy = taosMemoryCalloc(1, sizeof(SUdfdProxy)); + proxy->dnodeId = dnodeId; + proxy->gUdfcState = UDFC_STATE_STARTNG; + uv_barrier_init(&proxy->gUdfInitBarrier, 2); + uv_thread_create(&proxy->gUdfLoopThread, constructUdfService, proxy); + uv_barrier_wait(&proxy->gUdfInitBarrier); + proxy->gUdfcState = UDFC_STATE_READY; + *udfc = proxy; return 0; } -int32_t destroyUdfdProxy(int32_t dnodeId) { - gUdfcState = UDFC_STATE_STOPPING; - uv_async_send(&gUdfLoopStopAsync); - uv_thread_join(&gUdfLoopThread); - uv_mutex_destroy(&gUdfTaskQueueMutex); - uv_barrier_destroy(&gUdfInitBarrier); - gUdfcState = UDFC_STATUS_FINAL; +int32_t udfcClose(UdfcHandle udfcHandle) { + SUdfdProxy *udfc = udfcHandle; + udfc->gUdfcState = UDFC_STATE_STOPPING; + uv_async_send(&udfc->gUdfLoopStopAsync); + uv_thread_join(&udfc->gUdfLoopThread); + uv_mutex_destroy(&udfc->gUdfTaskQueueMutex); + uv_barrier_destroy(&udfc->gUdfInitBarrier); + udfc->gUdfcState = UDFC_STATUS_FINAL; + taosMemoryFree(udfc); return 0; } @@ -966,11 +980,12 @@ int32_t udfcRunUvTask(SClientUdfTask *task, int8_t uvTaskType) { return task->errCode; } -int32_t setupUdf(char udfName[], SEpSet *epSet, UdfHandle *handle) { +int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle *funcHandle) { debugPrint("%s", "client setup udf"); SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); task->errCode = 0; task->session = taosMemoryMalloc(sizeof(SUdfUvSession)); + task->session->udfc = udfc; task->type = UDF_TASK_SETUP; SUdfSetupRequest *req = &task->_setup.req; @@ -986,13 +1001,13 @@ int32_t setupUdf(char udfName[], SEpSet *epSet, UdfHandle *handle) { SUdfSetupResponse *rsp = &task->_setup.rsp; task->session->severHandle = rsp->udfHandle; - *handle = task->session; + *funcHandle = task->session; int32_t err = task->errCode; taosMemoryFree(task); return err; } -int32_t callUdf(UdfHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2, +int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2, SSDataBlock* output, SUdfInterBuf *newState) { debugPrint("%s", "client call udf"); @@ -1061,7 +1076,7 @@ int32_t callUdf(UdfHandle handle, int8_t callType, SSDataBlock *input, SUdfInter } //TODO: translate these calls to callUdf -int32_t callUdfAggInit(UdfHandle handle, SUdfInterBuf *interBuf) { +int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf) { int8_t callType = TSDB_UDF_CALL_AGG_INIT; int32_t err = callUdf(handle, callType, NULL, NULL, NULL, NULL, interBuf); @@ -1071,7 +1086,7 @@ int32_t callUdfAggInit(UdfHandle handle, SUdfInterBuf *interBuf) { // input: block, state // output: interbuf, -int32_t callUdfAggProcess(UdfHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState) { +int32_t callUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState) { int8_t callType = TSDB_UDF_CALL_AGG_PROC; int32_t err = callUdf(handle, callType, block, state, NULL, NULL, newState); return err; @@ -1079,7 +1094,7 @@ int32_t callUdfAggProcess(UdfHandle handle, SSDataBlock *block, SUdfInterBuf *st // input: interbuf1, interbuf2 // output: resultBuf -int32_t callUdfAggMerge(UdfHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf) { +int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf) { int8_t callType = TSDB_UDF_CALL_AGG_MERGE; int32_t err = callUdf(handle, callType, NULL, interBuf1, interBuf2, NULL, resultBuf); return err; @@ -1087,7 +1102,7 @@ int32_t callUdfAggMerge(UdfHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf // input: interBuf // output: resultData -int32_t callUdfAggFinalize(UdfHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData) { +int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData) { int8_t callType = TSDB_UDF_CALL_AGG_PROC; int32_t err = callUdf(handle, callType, NULL, interBuf, NULL, NULL, resultData); return err; @@ -1095,13 +1110,13 @@ int32_t callUdfAggFinalize(UdfHandle handle, SUdfInterBuf *interBuf, SUdfInterBu // input: block // output: resultData -int32_t callUdfScalaProcess(UdfHandle handle, SSDataBlock *block, SSDataBlock *resultData) { +int32_t callUdfScalaProcess(UdfcFuncHandle handle, SSDataBlock *block, SSDataBlock *resultData) { int8_t callType = TSDB_UDF_CALL_SCALA_PROC; int32_t err = callUdf(handle, callType, block, NULL, NULL, resultData, NULL); return err; } -int32_t teardownUdf(UdfHandle handle) { +int32_t teardownUdf(UdfcFuncHandle handle) { debugPrint("%s", "client teardown udf"); SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 71434c695f..1dd0871ae9 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -76,9 +76,9 @@ typedef struct SUdf { // TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix // TODO: add private udf structure. -typedef struct SUdfHandle { +typedef struct SUdfcFuncHandle { SUdf *udf; -} SUdfHandle; +} SUdfcFuncHandle; int32_t udfdLoadUdf(char* udfName, SUdf* udf) { strcpy(udf->name, udfName); @@ -143,7 +143,7 @@ void udfdProcessRequest(uv_work_t *req) { } uv_mutex_unlock(&udf->lock); } - SUdfHandle *handle = taosMemoryMalloc(sizeof(SUdfHandle)); + SUdfcFuncHandle *handle = taosMemoryMalloc(sizeof(SUdfcFuncHandle)); handle->udf = udf; // TODO: allocate private structure and call init function and set it to handle SUdfResponse rsp; @@ -166,7 +166,7 @@ void udfdProcessRequest(uv_work_t *req) { case UDF_TASK_CALL: { SUdfCallRequest *call = &request.call; fnDebug("%"PRId64 "call request. call type %d, handle: %"PRIx64, request.seqNum, call->callType, call->udfHandle); - SUdfHandle *handle = (SUdfHandle *)(call->udfHandle); + SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(call->udfHandle); SUdf *udf = handle->udf; SUdfDataBlock input = {0}; @@ -204,7 +204,7 @@ void udfdProcessRequest(uv_work_t *req) { case UDF_TASK_TEARDOWN: { SUdfTeardownRequest *teardown = &request.teardown; fnInfo("teardown. %"PRId64"handle:%"PRIx64, request.seqNum, teardown->udfHandle) - SUdfHandle *handle = (SUdfHandle *)(teardown->udfHandle); + SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(teardown->udfHandle); SUdf *udf = handle->udf; bool unloadUdf = false; uv_mutex_lock(&global.udfsMutex); diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c index 41c7f65e0b..fb9c3c678a 100644 --- a/source/libs/function/test/runUdf.c +++ b/source/libs/function/test/runUdf.c @@ -8,7 +8,8 @@ #include "tdatablock.h" int main(int argc, char *argv[]) { - createUdfdProxy(1); + UdfcHandle udfc; + udfcOpen(1, &udfc); uv_sleep(1000); char path[256] = {0}; size_t cwdSize = 256; @@ -20,9 +21,9 @@ int main(int argc, char *argv[]) { fprintf(stdout, "current working directory:%s\n", path); strcat(path, "/libudf1.so"); - UdfHandle handle; + UdfcFuncHandle handle; SEpSet epSet; - setupUdf("udf1", &epSet, &handle); + setupUdf(udfc, "udf1", &epSet, &handle); SSDataBlock block = {0}; SSDataBlock* pBlock = █ @@ -53,5 +54,5 @@ int main(int argc, char *argv[]) { } teardownUdf(handle); - destroyUdfdProxy(1); + udfcClose(udfc); } From c11edd523dc6ee8f56387fbc0ac6a9a0b37a759f Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 21 Apr 2022 13:38:05 +0800 Subject: [PATCH 33/83] find the reason of uv_barrier_destory core dump --- source/dnode/mgmt/implement/src/dmHandle.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/mgmt/implement/src/dmHandle.c b/source/dnode/mgmt/implement/src/dmHandle.c index df1ab79b99..0ebdeccc06 100644 --- a/source/dnode/mgmt/implement/src/dmHandle.c +++ b/source/dnode/mgmt/implement/src/dmHandle.c @@ -291,6 +291,7 @@ int32_t dmStartUdfd(SDnode *pDnode) { } int32_t dmStopUdfd(SDnode *pDnode) { + dInfo("dnode-mgmt to stop udfd. spawn err: %d", pDnode->data.udfdErrCode); SDnodeData *pData = &pDnode->data; if (pData->udfdErrCode != 0) { return 0; From 52b6a9c0588f78f42ea2fac481c300856727da33 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 21 Apr 2022 13:59:42 +0800 Subject: [PATCH 34/83] add syncElectTest, config replicaNum myIndex --- source/libs/sync/test/syncElectTest.cpp | 101 ++++++++++++------------ 1 file changed, 49 insertions(+), 52 deletions(-) diff --git a/source/libs/sync/test/syncElectTest.cpp b/source/libs/sync/test/syncElectTest.cpp index 39b5eba1aa..f58b6b670b 100644 --- a/source/libs/sync/test/syncElectTest.cpp +++ b/source/libs/sync/test/syncElectTest.cpp @@ -3,9 +3,8 @@ #include "syncEnv.h" #include "syncIO.h" #include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" #include "syncUtil.h" +#include "wal.h" void logTest() { sTrace("--- sync log test: trace"); @@ -16,43 +15,41 @@ void logTest() { sFatal("--- sync log test: fatal"); } -uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; -int32_t replicaNum = 3; -int32_t myIndex = 0; - -SRaftId ids[TSDB_MAX_REPLICA]; -SSyncInfo syncInfo; -SSyncFSM* pFsm; -SWal* pWal; -SSyncNode* gSyncNode; -const char* pDir = "./syncElectTest"; - -SSyncNode* syncNodeInit() { - syncInfo.vgId = 1234; - syncInfo.rpcClient = gSyncIO->clientRpc; - syncInfo.FpSendMsg = syncIOSendMsg; - syncInfo.queue = gSyncIO->pMsgQ; - syncInfo.FpEqMsg = syncIOEqMsg; - syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "%s_sync_%d", pDir, myIndex); +uint16_t gPorts[] = {7010, 7110, 7210, 7310, 7410}; +const char* gDir = "./syncElectTest"; +int32_t gVgId = 1234; +void init() { int code = walInit(); assert(code == 0); +} + +void cleanup() { walCleanUp(); } + +SWal* createWal(char* path, int32_t vgId) { SWalCfg walCfg; memset(&walCfg, 0, sizeof(SWalCfg)); - walCfg.vgId = syncInfo.vgId; + walCfg.vgId = vgId; walCfg.fsyncPeriod = 1000; walCfg.retentionPeriod = 1000; walCfg.rollPeriod = 1000; walCfg.retentionSize = 1000; walCfg.segSize = 1000; walCfg.level = TAOS_WAL_FSYNC; - - char tmpdir[128]; - snprintf(tmpdir, sizeof(tmpdir), "%s_wal_%d", pDir, myIndex); - pWal = walOpen(tmpdir, &walCfg); + SWal* pWal = walOpen(path, &walCfg); assert(pWal != NULL); + return pWal; +} +SSyncNode* createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* pWal, char* path) { + SSyncInfo syncInfo; + syncInfo.vgId = vgId; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = NULL; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s_sync_replica%d_index%d", path, replicaNum, myIndex); syncInfo.pWal = pWal; SSyncCfg* pCfg = &syncInfo.syncCfg; @@ -60,9 +57,9 @@ SSyncNode* syncNodeInit() { pCfg->replicaNum = replicaNum; for (int i = 0; i < replicaNum; ++i) { - pCfg->nodeInfo[i].nodePort = ports[i]; - snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + pCfg->nodeInfo[i].nodePort = gPorts[i]; + taosGetFqdn(pCfg->nodeInfo[i].nodeFqdn); + // snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); } SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); @@ -84,46 +81,46 @@ SSyncNode* syncNodeInit() { return pSyncNode; } -SSyncNode* syncInitTest() { return syncNodeInit(); } - -void initRaftId(SSyncNode* pSyncNode) { - for (int i = 0; i < replicaNum; ++i) { - ids[i] = pSyncNode->replicasId[i]; - char* s = syncUtilRaftId2Str(&ids[i]); - printf("raftId[%d] : %s\n", i, s); - taosMemoryFree(s); - } -} +void usage(char* exe) { printf("usage: %s replicaNum myIndex \n", exe); } int main(int argc, char** argv) { - // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; - sDebugFlag = 143 + 64; - - myIndex = 0; - if (argc >= 2) { - myIndex = atoi(argv[1]); + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + if (argc != 3) { + usage(argv[0]); + exit(-1); } + int32_t replicaNum = atoi(argv[1]); + int32_t myIndex = atoi(argv[2]); - int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(replicaNum >= 1 && replicaNum <= 5); + assert(myIndex >= 0 && myIndex < replicaNum); + + init(); + int32_t ret = syncIOStart((char*)"127.0.0.1", gPorts[myIndex]); assert(ret == 0); - ret = syncEnvStart(); assert(ret == 0); - gSyncNode = syncInitTest(); - assert(gSyncNode != NULL); - syncNodeLog2((char*)"", gSyncNode); + char walPath[128]; + snprintf(walPath, sizeof(walPath), "%s_wal_replica%d_index%d", gDir, replicaNum, myIndex); + SWal* pWal = createWal(walPath, gVgId); - initRaftId(gSyncNode); + SSyncNode* pSyncNode = createSyncNode(replicaNum, myIndex, gVgId, pWal, (char*)gDir); + assert(pSyncNode != NULL); + syncNodeLog2((char*)"==syncElectTest==", pSyncNode); //--------------------------- while (1) { - char* s = syncNode2SimpleStr(gSyncNode); + char* s = syncNode2SimpleStr(pSyncNode); sTrace("%s", s); taosMemoryFree(s); taosMsleep(1000); } + syncNodeClose(pSyncNode); + walClose(pWal); + syncIOStop(); + cleanup(); return 0; } From a6b949831aba690fe9322009b3f7723a1b8f8186 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 21 Apr 2022 14:56:41 +0800 Subject: [PATCH 35/83] delete syncElectTest 1-3, add syncReplicateTest.cpp --- source/libs/sync/test/CMakeLists.txt | 42 ---- source/libs/sync/test/syncElectTest1.cpp | 129 ----------- source/libs/sync/test/syncElectTest2.cpp | 139 ------------ source/libs/sync/test/syncElectTest3.cpp | 142 ------------ source/libs/sync/test/syncReplicateTest.cpp | 239 +++++++++++--------- 5 files changed, 128 insertions(+), 563 deletions(-) delete mode 100644 source/libs/sync/test/syncElectTest1.cpp delete mode 100644 source/libs/sync/test/syncElectTest2.cpp delete mode 100644 source/libs/sync/test/syncElectTest3.cpp diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 2500cd3c7b..14d4401233 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -28,9 +28,6 @@ add_executable(syncRpcMsgTest "") add_executable(syncPingTimerTest2 "") add_executable(syncPingSelfTest "") add_executable(syncElectTest "") -add_executable(syncElectTest1 "") -add_executable(syncElectTest2 "") -add_executable(syncElectTest3 "") add_executable(syncEncodeTest "") add_executable(syncWriteTest "") add_executable(syncReplicateTest "") @@ -165,18 +162,6 @@ target_sources(syncElectTest PRIVATE "syncElectTest.cpp" ) -target_sources(syncElectTest1 - PRIVATE - "syncElectTest1.cpp" -) -target_sources(syncElectTest2 - PRIVATE - "syncElectTest2.cpp" -) -target_sources(syncElectTest3 - PRIVATE - "syncElectTest3.cpp" -) target_sources(syncEncodeTest PRIVATE "syncEncodeTest.cpp" @@ -377,21 +362,6 @@ target_include_directories(syncElectTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) -target_include_directories(syncElectTest1 - PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) -target_include_directories(syncElectTest2 - PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) -target_include_directories(syncElectTest3 - PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) target_include_directories(syncEncodeTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" @@ -574,18 +544,6 @@ target_link_libraries(syncElectTest sync gtest_main ) -target_link_libraries(syncElectTest1 - sync - gtest_main -) -target_link_libraries(syncElectTest2 - sync - gtest_main -) -target_link_libraries(syncElectTest3 - sync - gtest_main -) target_link_libraries(syncEncodeTest sync gtest_main diff --git a/source/libs/sync/test/syncElectTest1.cpp b/source/libs/sync/test/syncElectTest1.cpp deleted file mode 100644 index feea95af82..0000000000 --- a/source/libs/sync/test/syncElectTest1.cpp +++ /dev/null @@ -1,129 +0,0 @@ -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" - -void logTest() { - sTrace("--- sync log test: trace"); - sDebug("--- sync log test: debug"); - sInfo("--- sync log test: info"); - sWarn("--- sync log test: warn"); - sError("--- sync log test: error"); - sFatal("--- sync log test: fatal"); -} - -uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; -int32_t replicaNum = 1; -int32_t myIndex = 0; - -SRaftId ids[TSDB_MAX_REPLICA]; -SSyncInfo syncInfo; -SSyncFSM* pFsm; -SWal* pWal; -SSyncNode* gSyncNode; -const char* pDir = "./syncElectTest1"; - -SSyncNode* syncNodeInit() { - syncInfo.vgId = 1234; - syncInfo.rpcClient = gSyncIO->clientRpc; - syncInfo.FpSendMsg = syncIOSendMsg; - syncInfo.queue = gSyncIO->pMsgQ; - syncInfo.FpEqMsg = syncIOEqMsg; - syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "%s_sync_%d", pDir, myIndex); - - int code = walInit(); - assert(code == 0); - SWalCfg walCfg; - memset(&walCfg, 0, sizeof(SWalCfg)); - walCfg.vgId = syncInfo.vgId; - walCfg.fsyncPeriod = 1000; - walCfg.retentionPeriod = 1000; - walCfg.rollPeriod = 1000; - walCfg.retentionSize = 1000; - walCfg.segSize = 1000; - walCfg.level = TAOS_WAL_FSYNC; - - char tmpdir[128]; - snprintf(tmpdir, sizeof(tmpdir), "%s_wal_%d", pDir, myIndex); - pWal = walOpen(tmpdir, &walCfg); - assert(pWal != NULL); - - syncInfo.pWal = pWal; - - SSyncCfg* pCfg = &syncInfo.syncCfg; - pCfg->myIndex = myIndex; - pCfg->replicaNum = replicaNum; - - for (int i = 0; i < replicaNum; ++i) { - pCfg->nodeInfo[i].nodePort = ports[i]; - snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); - } - - SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); - assert(pSyncNode != NULL); - - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; - gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; - gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; - gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; - gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; - gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; - gSyncIO->pSyncNode = pSyncNode; - - syncNodeStart(pSyncNode); - - return pSyncNode; -} - -SSyncNode* syncInitTest() { return syncNodeInit(); } - -void initRaftId(SSyncNode* pSyncNode) { - for (int i = 0; i < replicaNum; ++i) { - ids[i] = pSyncNode->replicasId[i]; - char* s = syncUtilRaftId2Str(&ids[i]); - printf("raftId[%d] : %s\n", i, s); - taosMemoryFree(s); - } -} - -int main(int argc, char** argv) { - // taosInitLog((char *)"syncTest.log", 100000, 10); - tsAsyncLog = 0; - sDebugFlag = 143 + 64; - - myIndex = 0; - if (argc >= 2) { - myIndex = atoi(argv[1]); - } - - int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); - assert(ret == 0); - - ret = syncEnvStart(); - assert(ret == 0); - - gSyncNode = syncInitTest(); - assert(gSyncNode != NULL); - syncNodeLog2((char*)"", gSyncNode); - - initRaftId(gSyncNode); - - //--------------------------- - while (1) { - char* s = syncNode2SimpleStr(gSyncNode); - sTrace("%s", s); - taosMemoryFree(s); - taosMsleep(1000); - } - - return 0; -} diff --git a/source/libs/sync/test/syncElectTest2.cpp b/source/libs/sync/test/syncElectTest2.cpp deleted file mode 100644 index d569b47fb1..0000000000 --- a/source/libs/sync/test/syncElectTest2.cpp +++ /dev/null @@ -1,139 +0,0 @@ -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "tref.h" - -void logTest() { - sTrace("--- sync log test: trace"); - sDebug("--- sync log test: debug"); - sInfo("--- sync log test: info"); - sWarn("--- sync log test: warn"); - sError("--- sync log test: error"); - sFatal("--- sync log test: fatal"); -} - -uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; -int32_t replicaNum = 3; -int32_t myIndex = 0; - -SRaftId ids[TSDB_MAX_REPLICA]; -SSyncInfo syncInfo; -SSyncFSM* pFsm; -SWal* pWal; -const char* pDir = "./syncElectTest2"; - -int64_t syncNodeInit() { - syncInfo.vgId = 1234; - syncInfo.rpcClient = gSyncIO->clientRpc; - syncInfo.FpSendMsg = syncIOSendMsg; - syncInfo.queue = gSyncIO->pMsgQ; - syncInfo.FpEqMsg = syncIOEqMsg; - syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "%s_sync_%d", pDir, myIndex); - - int code = walInit(); - assert(code == 0); - SWalCfg walCfg; - memset(&walCfg, 0, sizeof(SWalCfg)); - walCfg.vgId = syncInfo.vgId; - walCfg.fsyncPeriod = 1000; - walCfg.retentionPeriod = 1000; - walCfg.rollPeriod = 1000; - walCfg.retentionSize = 1000; - walCfg.segSize = 1000; - walCfg.level = TAOS_WAL_FSYNC; - - char tmpdir[128]; - snprintf(tmpdir, sizeof(tmpdir), "%s_wal_%d", pDir, myIndex); - pWal = walOpen(tmpdir, &walCfg); - assert(pWal != NULL); - - syncInfo.pWal = pWal; - - SSyncCfg* pCfg = &syncInfo.syncCfg; - pCfg->myIndex = myIndex; - pCfg->replicaNum = replicaNum; - - for (int i = 0; i < replicaNum; ++i) { - pCfg->nodeInfo[i].nodePort = ports[i]; - snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); - } - - int64_t rid = syncOpen(&syncInfo); - assert(rid > 0); - syncStart(rid); - - SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); - assert(pSyncNode != NULL); - - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; - gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; - gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; - gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; - gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; - gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; - gSyncIO->pSyncNode = pSyncNode; - - syncNodeStart(pSyncNode); - - syncNodeRelease(pSyncNode); - - return rid; -} - -void initRaftId(SSyncNode* pSyncNode) { - for (int i = 0; i < replicaNum; ++i) { - ids[i] = pSyncNode->replicasId[i]; - char* s = syncUtilRaftId2Str(&ids[i]); - printf("raftId[%d] : %s\n", i, s); - taosMemoryFree(s); - } -} - -int main(int argc, char** argv) { - // taosInitLog((char *)"syncTest.log", 100000, 10); - tsAsyncLog = 0; - sDebugFlag = 143 + 64; - - myIndex = 0; - if (argc >= 2) { - myIndex = atoi(argv[1]); - } - - int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); - assert(ret == 0); - - ret = syncInit(); - assert(ret == 0); - - int64_t rid = syncNodeInit(); - assert(rid > 0); - - SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); - assert(pSyncNode != NULL); - - syncNodeLog2((char*)"", pSyncNode); - initRaftId(pSyncNode); - - //--------------------------- - while (1) { - sTrace( - "elect sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d", - pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, - pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS); - taosMsleep(1000); - } - - syncNodeRelease(pSyncNode); - - return 0; -} diff --git a/source/libs/sync/test/syncElectTest3.cpp b/source/libs/sync/test/syncElectTest3.cpp deleted file mode 100644 index 59e1b993d8..0000000000 --- a/source/libs/sync/test/syncElectTest3.cpp +++ /dev/null @@ -1,142 +0,0 @@ -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "tref.h" - -void logTest() { - sTrace("--- sync log test: trace"); - sDebug("--- sync log test: debug"); - sInfo("--- sync log test: info"); - sWarn("--- sync log test: warn"); - sError("--- sync log test: error"); - sFatal("--- sync log test: fatal"); -} - -uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; -int32_t replicaNum = 3; -int32_t myIndex = 0; - -SRaftId ids[TSDB_MAX_REPLICA]; -SSyncInfo syncInfo; -SSyncFSM* pFsm; -SWal* pWal; -const char* pDir = "./syncElectTest3"; - -int64_t syncNodeInit() { - syncInfo.vgId = 1234; - syncInfo.rpcClient = gSyncIO->clientRpc; - syncInfo.FpSendMsg = syncIOSendMsg; - syncInfo.queue = gSyncIO->pMsgQ; - syncInfo.FpEqMsg = syncIOEqMsg; - syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "%s_sync_%d", pDir, myIndex); - - int code = walInit(); - assert(code == 0); - SWalCfg walCfg; - memset(&walCfg, 0, sizeof(SWalCfg)); - walCfg.vgId = syncInfo.vgId; - walCfg.fsyncPeriod = 1000; - walCfg.retentionPeriod = 1000; - walCfg.rollPeriod = 1000; - walCfg.retentionSize = 1000; - walCfg.segSize = 1000; - walCfg.level = TAOS_WAL_FSYNC; - - char tmpdir[128]; - snprintf(tmpdir, sizeof(tmpdir), "%s_wal_%d", pDir, myIndex); - pWal = walOpen(tmpdir, &walCfg); - assert(pWal != NULL); - - syncInfo.pWal = pWal; - - SSyncCfg* pCfg = &syncInfo.syncCfg; - pCfg->myIndex = myIndex; - pCfg->replicaNum = replicaNum; - - for (int i = 0; i < replicaNum; ++i) { - pCfg->nodeInfo[i].nodePort = ports[i]; - snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); - } - - int64_t rid = syncOpen(&syncInfo); - assert(rid > 0); - syncStart(rid); - - SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); - assert(pSyncNode != NULL); - - pSyncNode->hbBaseLine = 500; - pSyncNode->electBaseLine = 1500; - - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; - gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; - gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; - gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; - gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; - gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; - gSyncIO->pSyncNode = pSyncNode; - - syncNodeStart(pSyncNode); - - syncNodeRelease(pSyncNode); - - return rid; -} - -void initRaftId(SSyncNode* pSyncNode) { - for (int i = 0; i < replicaNum; ++i) { - ids[i] = pSyncNode->replicasId[i]; - char* s = syncUtilRaftId2Str(&ids[i]); - printf("raftId[%d] : %s\n", i, s); - taosMemoryFree(s); - } -} - -int main(int argc, char** argv) { - // taosInitLog((char *)"syncTest.log", 100000, 10); - tsAsyncLog = 0; - sDebugFlag = 143 + 64; - - myIndex = 0; - if (argc >= 2) { - myIndex = atoi(argv[1]); - } - - int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); - assert(ret == 0); - - ret = syncInit(); - assert(ret == 0); - - int64_t rid = syncNodeInit(); - assert(rid > 0); - - SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); - assert(pSyncNode != NULL); - - syncNodeLog2((char*)"", pSyncNode); - initRaftId(pSyncNode); - - //--------------------------- - while (1) { - sTrace( - "elect sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d", - pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, - pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS); - taosMsleep(1000); - } - - syncNodeRelease(pSyncNode); - - return 0; -} diff --git a/source/libs/sync/test/syncReplicateTest.cpp b/source/libs/sync/test/syncReplicateTest.cpp index 9b5aaf5c18..14132848b4 100644 --- a/source/libs/sync/test/syncReplicateTest.cpp +++ b/source/libs/sync/test/syncReplicateTest.cpp @@ -3,11 +3,8 @@ #include "syncEnv.h" #include "syncIO.h" #include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftEntry.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" #include "syncUtil.h" +#include "wal.h" void logTest() { sTrace("--- sync log test: trace"); @@ -18,184 +15,204 @@ void logTest() { sFatal("--- sync log test: fatal"); } -uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; -int32_t replicaNum = 3; -int32_t myIndex = 0; +uint16_t gPorts[] = {7010, 7110, 7210, 7310, 7410}; +const char* gDir = "./syncReplicateTest"; +int32_t gVgId = 1234; +SyncIndex gSnapshotLastApplyIndex; -SRaftId ids[TSDB_MAX_REPLICA]; -SSyncInfo syncInfo; -SSyncFSM * pFsm; -SWal * pWal; -SSyncNode *gSyncNode; +void init() { + int code = walInit(); + assert(code == 0); -void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", - pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); - syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); + code = syncInit(); + assert(code == 0); } -void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { +void cleanup() { walCleanUp(); } + +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { + SyncIndex beginIndex = SYNC_INDEX_INVALID; + if (pFsm->FpGetSnapshot != NULL) { + SSnapshot snapshot; + pFsm->FpGetSnapshot(pFsm, &snapshot); + beginIndex = snapshot.lastApplyIndex; + } + + if (cbMeta.index > beginIndex) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); + } else { + sTrace("==callback== ==CommitCb== do not apply again %ld", cbMeta.index); + } +} + +void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); - syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } -void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { +void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); - syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } -void initFsm() { - pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); +int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { + pSnapshot->data = NULL; + pSnapshot->lastApplyIndex = gSnapshotLastApplyIndex; + pSnapshot->lastApplyTerm = 100; + return 0; +} + +SSyncFSM* createFsm() { + SSyncFSM* pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM)); pFsm->FpCommitCb = CommitCb; pFsm->FpPreCommitCb = PreCommitCb; pFsm->FpRollBackCb = RollBackCb; + pFsm->FpGetSnapshot = GetSnapshotCb; + return pFsm; } -SSyncNode *syncNodeInit() { - syncInfo.vgId = 1234; - syncInfo.rpcClient = gSyncIO->clientRpc; - syncInfo.FpSendMsg = syncIOSendMsg; - syncInfo.queue = gSyncIO->pMsgQ; - syncInfo.FpEqMsg = syncIOEqMsg; - syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "./replicate_test_%d", myIndex); - - int code = walInit(); - assert(code == 0); +SWal* createWal(char* path, int32_t vgId) { SWalCfg walCfg; memset(&walCfg, 0, sizeof(SWalCfg)); - walCfg.vgId = syncInfo.vgId; + walCfg.vgId = vgId; walCfg.fsyncPeriod = 1000; walCfg.retentionPeriod = 1000; walCfg.rollPeriod = 1000; walCfg.retentionSize = 1000; walCfg.segSize = 1000; walCfg.level = TAOS_WAL_FSYNC; - - char tmpdir[128]; - snprintf(tmpdir, sizeof(tmpdir), "./replicate_test_wal_%d", myIndex); - pWal = walOpen(tmpdir, &walCfg); + SWal* pWal = walOpen(path, &walCfg); assert(pWal != NULL); + return pWal; +} +int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* pWal, char* path) { + SSyncInfo syncInfo; + syncInfo.vgId = vgId; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = createFsm(); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s_sync_replica%d_index%d", path, replicaNum, myIndex); syncInfo.pWal = pWal; - SSyncCfg *pCfg = &syncInfo.syncCfg; + SSyncCfg* pCfg = &syncInfo.syncCfg; pCfg->myIndex = myIndex; pCfg->replicaNum = replicaNum; for (int i = 0; i < replicaNum; ++i) { - pCfg->nodeInfo[i].nodePort = ports[i]; - snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + pCfg->nodeInfo[i].nodePort = gPorts[i]; + taosGetFqdn(pCfg->nodeInfo[i].nodeFqdn); + // snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); } - SSyncNode *pSyncNode = syncNodeOpen(&syncInfo); + int64_t rid = syncOpen(&syncInfo); + assert(rid > 0); + syncStart(rid); + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); assert(pSyncNode != NULL); gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; gSyncIO->pSyncNode = pSyncNode; - return pSyncNode; + syncNodeStart(pSyncNode); + + return rid; } -SSyncNode *syncInitTest() { return syncNodeInit(); } +void usage(char* exe) { printf("usage: %s replicaNum myIndex lastApplyIndex writeRecordNum \n", exe); } -void initRaftId(SSyncNode *pSyncNode) { - for (int i = 0; i < replicaNum; ++i) { - ids[i] = pSyncNode->replicasId[i]; - char *s = syncUtilRaftId2Str(&ids[i]); - printf("raftId[%d] : %s\n", i, s); - taosMemoryFree(s); - } -} - -SRpcMsg *step0(int i) { +SRpcMsg *createRpcMsg(int i, int count, int myIndex) { SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); memset(pMsg, 0, sizeof(SRpcMsg)); pMsg->msgType = 9999; pMsg->contLen = 128; pMsg->pCont = taosMemoryMalloc(pMsg->contLen); - snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i); + snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-%ld", myIndex, i, count, taosGetTimestampMs()); return pMsg; } -SyncClientRequest *step1(const SRpcMsg *pMsg) { - SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true, 1000); - return pRetMsg; -} - -int main(int argc, char **argv) { - // taosInitLog((char *)"syncTest.log", 100000, 10); +int main(int argc, char** argv) { tsAsyncLog = 0; - sDebugFlag = 143 + 64; - void logTest(); - - myIndex = 0; - if (argc >= 2) { - myIndex = atoi(argv[1]); + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + if (argc != 5) { + usage(argv[0]); + exit(-1); } + int32_t replicaNum = atoi(argv[1]); + int32_t myIndex = atoi(argv[2]); + int32_t lastApplyIndex = atoi(argv[3]); + int32_t writeRecordNum = atoi(argv[4]); + gSnapshotLastApplyIndex = lastApplyIndex; - int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); + assert(replicaNum >= 1 && replicaNum <= 5); + assert(myIndex >= 0 && myIndex < replicaNum); + assert(lastApplyIndex >= -1); + assert(writeRecordNum >= 0); + + init(); + int32_t ret = syncIOStart((char*)"127.0.0.1", gPorts[myIndex]); assert(ret == 0); - ret = syncEnvStart(); - assert(ret == 0); + char walPath[128]; + snprintf(walPath, sizeof(walPath), "%s_wal_replica%d_index%d", gDir, replicaNum, myIndex); + SWal* pWal = createWal(walPath, gVgId); - taosRemoveDir("./wal_test"); + int64_t rid = createSyncNode(replicaNum, myIndex, gVgId, pWal, (char*)gDir); + assert(rid > 0); + syncStart(rid); - initFsm(); - - gSyncNode = syncInitTest(); - assert(gSyncNode != NULL); - syncNodeLog2((char *)"", gSyncNode); - - initRaftId(gSyncNode); - - for (int i = 0; i < 30; ++i) { - // step0 - SRpcMsg *pMsg0 = step0(i); - syncRpcMsgLog2((char *)"==step0==", pMsg0); - - // step1 - SyncClientRequest *pMsg1 = step1(pMsg0); - syncClientRequestLog2((char *)"==step1==", pMsg1); - - SyncClientRequest *pSyncClientRequest = pMsg1; - SRpcMsg rpcMsg; - syncClientRequest2RpcMsg(pSyncClientRequest, &rpcMsg); - gSyncNode->FpEqMsg(gSyncNode->queue, &rpcMsg); - - taosMsleep(1000); - sTrace( - "syncPropose sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " - "electTimerMS:%d, commitIndex:%ld", - gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->pRaftStore->currentTerm, - gSyncNode->electTimerLogicClock, gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS, - gSyncNode->commitIndex); - } + SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + //--------------------------- + int32_t alreadySend = 0; while (1) { - sTrace( - "replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " - "electTimerMS:%d, commitIndex:%ld", - gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->pRaftStore->currentTerm, - gSyncNode->electTimerLogicClock, gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS, - gSyncNode->commitIndex); + char* s = syncNode2SimpleStr(pSyncNode); + sTrace("%s", s); + + if (alreadySend < writeRecordNum) { + SRpcMsg *pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); + int32_t ret = syncPropose(rid, pRpcMsg, false); + if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { + sTrace("%s value%d write not leader", s, alreadySend); + } else { + assert(ret == 0); + sTrace("%s value%d write ok", s, alreadySend); + } + alreadySend++; + + //rpcFreeCont(pRpcMsg->pCont); + taosMemoryFree(pRpcMsg); + } + + taosMsleep(1000); + taosMemoryFree(s); taosMsleep(1000); } + syncNodeRelease(pSyncNode); + syncStop(rid); + walClose(pWal); + syncIOStop(); + cleanup(); return 0; } From 81a2f3b78c2bf8f1dc3f40f6bd351e4df5f779bb Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 21 Apr 2022 15:00:49 +0800 Subject: [PATCH 36/83] fix coredump due to dmStopUdfd called while dmStartUdfd not called --- source/dnode/mgmt/implement/src/dmHandle.c | 72 +++++++++++++--------- source/dnode/mgmt/interface/inc/dmDef.h | 19 +++--- 2 files changed, 54 insertions(+), 37 deletions(-) diff --git a/source/dnode/mgmt/implement/src/dmHandle.c b/source/dnode/mgmt/implement/src/dmHandle.c index 0ebdeccc06..61acb66086 100644 --- a/source/dnode/mgmt/implement/src/dmHandle.c +++ b/source/dnode/mgmt/implement/src/dmHandle.c @@ -216,20 +216,21 @@ static void dmStopMgmt(SMgmtWrapper *pWrapper) { dmStopStatusThread(pWrapper->pDnode); } -static int32_t dmSpawnUdfd(SDnodeData *pData); +static int32_t dmSpawnUdfd(SDnode *pDnode); void dmUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) { dInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal); uv_close((uv_handle_t*)process, NULL); - SDnodeData *pData = process->data; - if (atomic_load_8(&pData->udfdStoping) != 0) { + SDnode *pDnode = process->data; + SUdfdData *pData = &pDnode->udfdData; + if (atomic_load_8(&pData->stopping) != 0) { dDebug("udfd process exit due to stopping"); } else { - dmSpawnUdfd(pData); + dmSpawnUdfd(pDnode); } } -static int32_t dmSpawnUdfd(SDnodeData *pData) { +static int32_t dmSpawnUdfd(SDnode *pDnode) { dInfo("dnode start spawning udfd"); uv_process_options_t options = {0}; @@ -251,16 +252,17 @@ static int32_t dmSpawnUdfd(SDnodeData *pData) { char dnodeIdEnvItem[32] = {0}; char thrdPoolSizeEnvItem[32] = {0}; - snprintf(dnodeIdEnvItem, 32, "%s=%d", "DNODE_ID", pData->dnodeId); + snprintf(dnodeIdEnvItem, 32, "%s=%d", "DNODE_ID", pDnode->data.dnodeId); + SUdfdData *pData = &pDnode->udfdData; float numCpuCores = 4; taosGetCpuCores(&numCpuCores); snprintf(thrdPoolSizeEnvItem,32, "%s=%d", "UV_THREADPOOL_SIZE", (int)numCpuCores*2); char* envUdfd[] = {dnodeIdEnvItem, thrdPoolSizeEnvItem, NULL}; options.env = envUdfd; - int err = uv_spawn(&pData->udfdLoop, &pData->udfdProcess, &options); + int err = uv_spawn(&pData->loop, &pData->process, &options); - pData->udfdProcess.data = (void*)pData; + pData->process.data = (void*)pDnode; if (err != 0) { dError("can not spawn udfd. path: %s, error: %s", path, uv_strerror(err)); @@ -269,40 +271,50 @@ static int32_t dmSpawnUdfd(SDnodeData *pData) { } void dmWatchUdfd(void *args) { - SDnodeData *pData = args; - uv_loop_init(&pData->udfdLoop); - int err = dmSpawnUdfd(pData); - pData->udfdErrCode = err; - uv_barrier_wait(&pData->udfdBarrier); - if (pData->udfdErrCode == 0) { - uv_run(&pData->udfdLoop, UV_RUN_DEFAULT); + SDnode *pDnode = args; + SUdfdData *pData = &pDnode->udfdData; + uv_loop_init(&pData->loop); + int32_t err = dmSpawnUdfd(pDnode); + atomic_store_32(&pData->spawnErr, err); + uv_barrier_wait(&pData->barrier); + if (pData->spawnErr == 0) { + uv_run(&pData->loop, UV_RUN_DEFAULT); } - uv_loop_close(&pData->udfdLoop); + uv_loop_close(&pData->loop); return; } int32_t dmStartUdfd(SDnode *pDnode) { - SDnodeData *pData = &pDnode->data; - uv_barrier_init(&pData->udfdBarrier, 2); - pData->udfdStoping = 0; - uv_thread_create(&pData->udfdThread, dmWatchUdfd, pData); - uv_barrier_wait(&pData->udfdBarrier); - return pData->udfdErrCode; + SUdfdData *pData = &pDnode->udfdData; + if (pData->startCalled) { + dInfo("dnode-mgmt start udfd already called"); + return 0; + } + uv_barrier_init(&pData->barrier, 2); + pData->stopping = 0; + uv_thread_create(&pData->thread, dmWatchUdfd, pDnode); + uv_barrier_wait(&pData->barrier); + pData->startCalled = true; + pData->needCleanUp = true; + return pData->spawnErr; } int32_t dmStopUdfd(SDnode *pDnode) { - dInfo("dnode-mgmt to stop udfd. spawn err: %d", pDnode->data.udfdErrCode); - SDnodeData *pData = &pDnode->data; - if (pData->udfdErrCode != 0) { + dInfo("dnode-mgmt to stop udfd. need cleanup: %d, spawn err: %d", + pDnode->udfdData.needCleanUp, pDnode->udfdData.spawnErr); + SUdfdData *pData = &pDnode->udfdData; + if (!pData->needCleanUp) { return 0; } - atomic_store_8(&pData->udfdStoping, 1); + atomic_store_8(&pData->stopping, 1); - uv_barrier_destroy(&pData->udfdBarrier); - uv_process_kill(&pData->udfdProcess, SIGINT); - uv_thread_join(&pData->udfdThread); + uv_barrier_destroy(&pData->barrier); + if (pData->spawnErr == 0) { + uv_process_kill(&pData->process, SIGINT); + } + uv_thread_join(&pData->thread); - atomic_store_8(&pData->udfdStoping, 0); + atomic_store_8(&pData->stopping, 0); return 0; } diff --git a/source/dnode/mgmt/interface/inc/dmDef.h b/source/dnode/mgmt/interface/inc/dmDef.h index fdce59b4df..e76ac73b85 100644 --- a/source/dnode/mgmt/interface/inc/dmDef.h +++ b/source/dnode/mgmt/interface/inc/dmDef.h @@ -136,13 +136,6 @@ typedef struct { int32_t numOfDisks; int32_t supportVnodes; uint16_t serverPort; - - uv_loop_t udfdLoop; - uv_thread_t udfdThread; - uv_barrier_t udfdBarrier; - uv_process_t udfdProcess; - int udfdErrCode; - int8_t udfdStoping; } SDnodeData; typedef struct { @@ -150,6 +143,17 @@ typedef struct { char desc[TSDB_STEP_DESC_LEN]; } SStartupInfo; +typedef struct SUdfdData { + bool startCalled; + bool needCleanUp; + uv_loop_t loop; + uv_thread_t thread; + uv_barrier_t barrier; + uv_process_t process; + int spawnErr; + int8_t stopping; +} SUdfdData; + typedef struct SDnode { EDndProcType ptype; EDndNodeType ntype; @@ -158,6 +162,7 @@ typedef struct SDnode { SStartupInfo startup; SDnodeTrans trans; SDnodeData data; + SUdfdData udfdData; TdThreadMutex mutex; SMgmtWrapper wrappers[NODE_END]; } SDnode; From 49cf252495b0f1b782ab73ba7b353fa34b0d689d Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 21 Apr 2022 15:24:50 +0800 Subject: [PATCH 37/83] add syncReplicateTest.cpp --- source/libs/sync/test/syncReplicateTest.cpp | 33 ++++++++++----------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/source/libs/sync/test/syncReplicateTest.cpp b/source/libs/sync/test/syncReplicateTest.cpp index 14132848b4..0e94498a38 100644 --- a/source/libs/sync/test/syncReplicateTest.cpp +++ b/source/libs/sync/test/syncReplicateTest.cpp @@ -30,7 +30,7 @@ void init() { void cleanup() { walCleanUp(); } -void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { +void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { SyncIndex beginIndex = SYNC_INDEX_INVALID; if (pFsm->FpGetSnapshot != NULL) { SSnapshot snapshot; @@ -42,7 +42,7 @@ void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); - syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } else { sTrace("==callback== ==CommitCb== do not apply again %ld", cbMeta.index); } @@ -117,11 +117,9 @@ int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* int64_t rid = syncOpen(&syncInfo); assert(rid > 0); - syncStart(rid); SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); assert(pSyncNode != NULL); - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; @@ -131,22 +129,22 @@ int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; gSyncIO->pSyncNode = pSyncNode; - - syncNodeStart(pSyncNode); + syncNodeRelease(pSyncNode); return rid; } void usage(char* exe) { printf("usage: %s replicaNum myIndex lastApplyIndex writeRecordNum \n", exe); } -SRpcMsg *createRpcMsg(int i, int count, int myIndex) { - SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); +SRpcMsg* createRpcMsg(int i, int count, int myIndex) { + SRpcMsg* pMsg = (SRpcMsg*)taosMemoryMalloc(sizeof(SRpcMsg)); memset(pMsg, 0, sizeof(SRpcMsg)); pMsg->msgType = 9999; - pMsg->contLen = 128; - pMsg->pCont = taosMemoryMalloc(pMsg->contLen); - snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-%ld", myIndex, i, count, taosGetTimestampMs()); + pMsg->contLen = 256; + pMsg->pCont = rpcMallocCont(pMsg->contLen); + snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-%ld", myIndex, i, count, taosGetTimestampMs()); return pMsg; } @@ -180,28 +178,29 @@ int main(int argc, char** argv) { assert(rid > 0); syncStart(rid); - SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); assert(pSyncNode != NULL); //--------------------------- int32_t alreadySend = 0; while (1) { char* s = syncNode2SimpleStr(pSyncNode); - sTrace("%s", s); if (alreadySend < writeRecordNum) { - SRpcMsg *pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); - int32_t ret = syncPropose(rid, pRpcMsg, false); + SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); + int32_t ret = syncPropose(rid, pRpcMsg, false); if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { - sTrace("%s value%d write not leader", s, alreadySend); + sTrace("%s value%d write not leader", s, alreadySend); } else { assert(ret == 0); sTrace("%s value%d write ok", s, alreadySend); } alreadySend++; - //rpcFreeCont(pRpcMsg->pCont); + rpcFreeCont(pRpcMsg->pCont); taosMemoryFree(pRpcMsg); + } else { + sTrace("%s", s); } taosMsleep(1000); From 42b2b4a95ccd42b6ce9ba668fbf35d316647c7c0 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 21 Apr 2022 16:00:34 +0800 Subject: [PATCH 38/83] fix memory error --- source/dnode/mgmt/implement/src/dmHandle.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/implement/src/dmHandle.c b/source/dnode/mgmt/implement/src/dmHandle.c index 61acb66086..94c6d20834 100644 --- a/source/dnode/mgmt/implement/src/dmHandle.c +++ b/source/dnode/mgmt/implement/src/dmHandle.c @@ -270,6 +270,12 @@ static int32_t dmSpawnUdfd(SDnode *pDnode) { return err; } +static void dmUdfdCloseWalkCb(uv_handle_t* handle, void* arg) { + if (!uv_is_closing(handle)) { + uv_close(handle, NULL); + } +} + void dmWatchUdfd(void *args) { SDnode *pDnode = args; SUdfdData *pData = &pDnode->udfdData; @@ -277,10 +283,13 @@ void dmWatchUdfd(void *args) { int32_t err = dmSpawnUdfd(pDnode); atomic_store_32(&pData->spawnErr, err); uv_barrier_wait(&pData->barrier); - if (pData->spawnErr == 0) { + uv_run(&pData->loop, UV_RUN_DEFAULT); + err = uv_loop_close(&pData->loop); + while (err == UV_EBUSY) { + uv_walk(&pData->loop, dmUdfdCloseWalkCb, NULL); uv_run(&pData->loop, UV_RUN_DEFAULT); + err = uv_loop_close(&pData->loop); } - uv_loop_close(&pData->loop); return; } @@ -312,6 +321,7 @@ int32_t dmStopUdfd(SDnode *pDnode) { if (pData->spawnErr == 0) { uv_process_kill(&pData->process, SIGINT); } + uv_stop(&pData->loop); uv_thread_join(&pData->thread); atomic_store_8(&pData->stopping, 0); From 446a2ed1bfe19cab6570125b35298efa29bdc82a Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 21 Apr 2022 17:35:48 +0800 Subject: [PATCH 39/83] rm syncReplicateTest 2, 3, Load --- source/libs/sync/test/CMakeLists.txt | 42 ---- .../libs/sync/test/syncReplicateLoadTest.cpp | 187 --------------- source/libs/sync/test/syncReplicateTest2.cpp | 203 ----------------- source/libs/sync/test/syncReplicateTest3.cpp | 215 ------------------ 4 files changed, 647 deletions(-) delete mode 100644 source/libs/sync/test/syncReplicateLoadTest.cpp delete mode 100644 source/libs/sync/test/syncReplicateTest2.cpp delete mode 100644 source/libs/sync/test/syncReplicateTest3.cpp diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 14d4401233..3319a73fcf 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -31,9 +31,6 @@ add_executable(syncElectTest "") add_executable(syncEncodeTest "") add_executable(syncWriteTest "") add_executable(syncReplicateTest "") -add_executable(syncReplicateTest2 "") -add_executable(syncReplicateTest3 "") -add_executable(syncReplicateLoadTest "") add_executable(syncRefTest "") add_executable(syncLogStoreCheck "") add_executable(syncRaftCfgTest "") @@ -174,18 +171,6 @@ target_sources(syncReplicateTest PRIVATE "syncReplicateTest.cpp" ) -target_sources(syncReplicateTest2 - PRIVATE - "syncReplicateTest2.cpp" -) -target_sources(syncReplicateTest3 - PRIVATE - "syncReplicateTest3.cpp" -) -target_sources(syncReplicateLoadTest - PRIVATE - "syncReplicateLoadTest.cpp" -) target_sources(syncRefTest PRIVATE "syncRefTest.cpp" @@ -377,21 +362,6 @@ target_include_directories(syncReplicateTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) -target_include_directories(syncReplicateTest2 - PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) -target_include_directories(syncReplicateTest3 - PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) -target_include_directories(syncReplicateLoadTest - PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) target_include_directories(syncRefTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" @@ -556,18 +526,6 @@ target_link_libraries(syncReplicateTest sync gtest_main ) -target_link_libraries(syncReplicateTest2 - sync - gtest_main -) -target_link_libraries(syncReplicateTest3 - sync - gtest_main -) -target_link_libraries(syncReplicateLoadTest - sync - gtest_main -) target_link_libraries(syncRefTest sync gtest_main diff --git a/source/libs/sync/test/syncReplicateLoadTest.cpp b/source/libs/sync/test/syncReplicateLoadTest.cpp deleted file mode 100644 index f58c372445..0000000000 --- a/source/libs/sync/test/syncReplicateLoadTest.cpp +++ /dev/null @@ -1,187 +0,0 @@ -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftEntry.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" - -void logTest() { - sTrace("--- sync log test: trace"); - sDebug("--- sync log test: debug"); - sInfo("--- sync log test: info"); - sWarn("--- sync log test: warn"); - sError("--- sync log test: error"); - sFatal("--- sync log test: fatal"); -} - -uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; -int32_t replicaNum = 3; -int32_t myIndex = 0; - -SRaftId ids[TSDB_MAX_REPLICA]; -SSyncInfo syncInfo; -SSyncFSM *pFsm; -SWal * pWal; - -void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", - pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); - syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); -} - -void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), - "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, - cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); - syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); -} - -void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", - pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); - syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); -} - -void initFsm() { - pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); - pFsm->FpCommitCb = CommitCb; - pFsm->FpPreCommitCb = PreCommitCb; - pFsm->FpRollBackCb = RollBackCb; -} - -int64_t syncNodeInit() { - syncInfo.vgId = 1234; - syncInfo.rpcClient = gSyncIO->clientRpc; - syncInfo.FpSendMsg = syncIOSendMsg; - syncInfo.queue = gSyncIO->pMsgQ; - syncInfo.FpEqMsg = syncIOEqMsg; - syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "./replicate2_test_%d", myIndex); - - int code = walInit(); - assert(code == 0); - SWalCfg walCfg; - memset(&walCfg, 0, sizeof(SWalCfg)); - walCfg.vgId = syncInfo.vgId; - walCfg.fsyncPeriod = 1000; - walCfg.retentionPeriod = 1000; - walCfg.rollPeriod = 1000; - walCfg.retentionSize = 1000; - walCfg.segSize = 1000; - walCfg.level = TAOS_WAL_FSYNC; - - char tmpdir[128]; - snprintf(tmpdir, sizeof(tmpdir), "./replicate2_test_wal_%d", myIndex); - pWal = walOpen(tmpdir, &walCfg); - assert(pWal != NULL); - - syncInfo.pWal = pWal; - - SSyncCfg *pCfg = &syncInfo.syncCfg; - pCfg->myIndex = myIndex; - pCfg->replicaNum = replicaNum; - - for (int i = 0; i < replicaNum; ++i) { - pCfg->nodeInfo[i].nodePort = ports[i]; - snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); - } - - int64_t rid = syncOpen(&syncInfo); - assert(rid > 0); - syncStart(rid); - - SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); - assert(pSyncNode != NULL); - - // pSyncNode->hbBaseLine = 500; - // pSyncNode->electBaseLine = 1500; - - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; - gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; - gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; - gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; - gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; - gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; - gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; - gSyncIO->pSyncNode = pSyncNode; - - syncNodeRelease(pSyncNode); - - return rid; -} - -void initRaftId(SSyncNode *pSyncNode) { - for (int i = 0; i < replicaNum; ++i) { - ids[i] = pSyncNode->replicasId[i]; - char *s = syncUtilRaftId2Str(&ids[i]); - printf("raftId[%d] : %s\n", i, s); - taosMemoryFree(s); - } -} - -SRpcMsg *step0(int i) { - SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); - memset(pMsg, 0, sizeof(SRpcMsg)); - pMsg->msgType = 9999; - pMsg->contLen = 128; - pMsg->pCont = taosMemoryMalloc(pMsg->contLen); - snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i); - return pMsg; -} - -SyncClientRequest *step1(const SRpcMsg *pMsg) { - SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true, 1000); - return pRetMsg; -} - -int main(int argc, char **argv) { - // taosInitLog((char *)"syncTest.log", 100000, 10); - tsAsyncLog = 0; - sDebugFlag = 143 + 64; - void logTest(); - - myIndex = 0; - if (argc >= 2) { - myIndex = atoi(argv[1]); - } - - int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); - assert(ret == 0); - - initFsm(); - - ret = syncInit(); - assert(ret == 0); - - int64_t rid = syncNodeInit(); - assert(rid > 0); - - SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); - assert(pSyncNode != NULL); - - syncNodeLog2((char *)"", pSyncNode); - initRaftId(pSyncNode); - - // only load ... - - while (1) { - sTrace( - "replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " - "electTimerMS:%d, commitIndex:%ld", - pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, - pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, - pSyncNode->commitIndex); - taosMsleep(1000); - } - - return 0; -} diff --git a/source/libs/sync/test/syncReplicateTest2.cpp b/source/libs/sync/test/syncReplicateTest2.cpp deleted file mode 100644 index 1e2f823be5..0000000000 --- a/source/libs/sync/test/syncReplicateTest2.cpp +++ /dev/null @@ -1,203 +0,0 @@ -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftEntry.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" - -void logTest() { - sTrace("--- sync log test: trace"); - sDebug("--- sync log test: debug"); - sInfo("--- sync log test: info"); - sWarn("--- sync log test: warn"); - sError("--- sync log test: error"); - sFatal("--- sync log test: fatal"); -} - -uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; -int32_t replicaNum = 3; -int32_t myIndex = 0; - -SRaftId ids[TSDB_MAX_REPLICA]; -SSyncInfo syncInfo; -SSyncFSM *pFsm; -SWal * pWal; - -void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", - pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); - syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); -} - -void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), - "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, - cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); - syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); -} - -void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", - pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); - syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); -} - -void initFsm() { - pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); - pFsm->FpCommitCb = CommitCb; - pFsm->FpPreCommitCb = PreCommitCb; - pFsm->FpRollBackCb = RollBackCb; -} - -int64_t syncNodeInit() { - syncInfo.vgId = 1234; - syncInfo.rpcClient = gSyncIO->clientRpc; - syncInfo.FpSendMsg = syncIOSendMsg; - syncInfo.queue = gSyncIO->pMsgQ; - syncInfo.FpEqMsg = syncIOEqMsg; - syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "./replicate2_test_%d", myIndex); - - int code = walInit(); - assert(code == 0); - SWalCfg walCfg; - memset(&walCfg, 0, sizeof(SWalCfg)); - walCfg.vgId = syncInfo.vgId; - walCfg.fsyncPeriod = 1000; - walCfg.retentionPeriod = 1000; - walCfg.rollPeriod = 1000; - walCfg.retentionSize = 1000; - walCfg.segSize = 1000; - walCfg.level = TAOS_WAL_FSYNC; - - char tmpdir[128]; - snprintf(tmpdir, sizeof(tmpdir), "./replicate2_test_wal_%d", myIndex); - pWal = walOpen(tmpdir, &walCfg); - assert(pWal != NULL); - - syncInfo.pWal = pWal; - - SSyncCfg *pCfg = &syncInfo.syncCfg; - pCfg->myIndex = myIndex; - pCfg->replicaNum = replicaNum; - - for (int i = 0; i < replicaNum; ++i) { - pCfg->nodeInfo[i].nodePort = ports[i]; - snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); - } - - int64_t rid = syncOpen(&syncInfo); - assert(rid > 0); - syncStart(rid); - - SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); - assert(pSyncNode != NULL); - - // pSyncNode->hbBaseLine = 500; - // pSyncNode->electBaseLine = 1500; - - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; - gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; - gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; - gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; - gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; - gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; - gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; - gSyncIO->pSyncNode = pSyncNode; - - syncNodeRelease(pSyncNode); - - return rid; -} - -void initRaftId(SSyncNode *pSyncNode) { - for (int i = 0; i < replicaNum; ++i) { - ids[i] = pSyncNode->replicasId[i]; - char *s = syncUtilRaftId2Str(&ids[i]); - printf("raftId[%d] : %s\n", i, s); - taosMemoryFree(s); - } -} - -SRpcMsg *step0(int i) { - SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); - memset(pMsg, 0, sizeof(SRpcMsg)); - pMsg->msgType = 9999; - pMsg->contLen = 128; - pMsg->pCont = taosMemoryMalloc(pMsg->contLen); - snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i); - return pMsg; -} - -SyncClientRequest *step1(const SRpcMsg *pMsg) { - SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true, 1000); - return pRetMsg; -} - -int main(int argc, char **argv) { - // taosInitLog((char *)"syncTest.log", 100000, 10); - tsAsyncLog = 0; - sDebugFlag = 143 + 64; - void logTest(); - - myIndex = 0; - if (argc >= 2) { - myIndex = atoi(argv[1]); - } - - int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); - assert(ret == 0); - - initFsm(); - - ret = syncInit(); - assert(ret == 0); - - int64_t rid = syncNodeInit(); - assert(rid > 0); - - SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); - assert(pSyncNode != NULL); - - syncNodeLog2((char *)"", pSyncNode); - initRaftId(pSyncNode); - - for (int i = 0; i < 30; ++i) { - // step0 - SRpcMsg *pMsg0 = step0(i); - syncRpcMsgLog2((char *)"==step0==", pMsg0); - - syncPropose(rid, pMsg0, true); - taosMsleep(1000); - - taosMemoryFree(pMsg0); - - sTrace( - "syncPropose sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " - "electTimerMS:%d, commitIndex:%ld", - pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, - pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, - pSyncNode->commitIndex); - } - - while (1) { - sTrace( - "replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " - "electTimerMS:%d, commitIndex:%ld", - pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, - pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, - pSyncNode->commitIndex); - taosMsleep(1000); - } - - return 0; -} diff --git a/source/libs/sync/test/syncReplicateTest3.cpp b/source/libs/sync/test/syncReplicateTest3.cpp deleted file mode 100644 index e8e24b507d..0000000000 --- a/source/libs/sync/test/syncReplicateTest3.cpp +++ /dev/null @@ -1,215 +0,0 @@ -#define ALLOW_FORBID_FUNC - -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftEntry.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" - -void logTest() { - sTrace("--- sync log test: trace"); - sDebug("--- sync log test: debug"); - sInfo("--- sync log test: info"); - sWarn("--- sync log test: warn"); - sError("--- sync log test: error"); - sFatal("--- sync log test: fatal"); -} - -uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; -int32_t replicaNum = 3; -int32_t myIndex = 0; - -SRaftId ids[TSDB_MAX_REPLICA]; -SSyncInfo syncInfo; -SSyncFSM *pFsm; -SWal * pWal; - -void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", - pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); - syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); -} - -void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), - "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, - cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); - syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); -} - -void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", - pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); - syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); -} - -void initFsm() { - pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); - pFsm->FpCommitCb = CommitCb; - pFsm->FpPreCommitCb = PreCommitCb; - pFsm->FpRollBackCb = RollBackCb; -} - -int64_t syncNodeInit() { - syncInfo.vgId = 1234; - syncInfo.rpcClient = gSyncIO->clientRpc; - syncInfo.FpSendMsg = syncIOSendMsg; - syncInfo.queue = gSyncIO->pMsgQ; - syncInfo.FpEqMsg = syncIOEqMsg; - syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "./replicate2_test_%d", myIndex); - - int code = walInit(); - assert(code == 0); - SWalCfg walCfg; - memset(&walCfg, 0, sizeof(SWalCfg)); - walCfg.vgId = syncInfo.vgId; - walCfg.fsyncPeriod = 1000; - walCfg.retentionPeriod = 1000; - walCfg.rollPeriod = 1000; - walCfg.retentionSize = 1000; - walCfg.segSize = 1000; - walCfg.level = TAOS_WAL_FSYNC; - - char tmpdir[128]; - snprintf(tmpdir, sizeof(tmpdir), "./replicate2_test_wal_%d", myIndex); - pWal = walOpen(tmpdir, &walCfg); - assert(pWal != NULL); - - syncInfo.pWal = pWal; - - SSyncCfg *pCfg = &syncInfo.syncCfg; - pCfg->myIndex = myIndex; - pCfg->replicaNum = replicaNum; - - for (int i = 0; i < replicaNum; ++i) { - pCfg->nodeInfo[i].nodePort = ports[i]; - snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); - } - - int64_t rid = syncOpen(&syncInfo); - assert(rid > 0); - syncStart(rid); - - SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); - assert(pSyncNode != NULL); - - // pSyncNode->hbBaseLine = 500; - // pSyncNode->electBaseLine = 1500; - - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; - gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; - gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; - gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; - gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; - gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; - gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; - gSyncIO->pSyncNode = pSyncNode; - - syncNodeRelease(pSyncNode); - - return rid; -} - -void initRaftId(SSyncNode *pSyncNode) { - for (int i = 0; i < replicaNum; ++i) { - ids[i] = pSyncNode->replicasId[i]; - char *s = syncUtilRaftId2Str(&ids[i]); - printf("raftId[%d] : %s\n", i, s); - taosMemoryFree(s); - } -} - -SRpcMsg *step0(int i) { - SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); - memset(pMsg, 0, sizeof(SRpcMsg)); - pMsg->msgType = 9999; - pMsg->contLen = 128; - pMsg->pCont = taosMemoryMalloc(pMsg->contLen); - snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i); - return pMsg; -} - -SyncClientRequest *step1(const SRpcMsg *pMsg) { - SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true, 1000); - return pRetMsg; -} - -int main(int argc, char **argv) { - // taosInitLog((char *)"syncTest.log", 100000, 10); - tsAsyncLog = 0; - sDebugFlag = 143 + 64; - void logTest(); - - myIndex = 0; - if (argc >= 2) { - myIndex = atoi(argv[1]); - } - - int recordCount = 100; - if (argc >= 3) { - recordCount = atoi(argv[2]); - } - - int sleepMS = 10; - if (argc >= 4) { - sleepMS = atoi(argv[3]); - } - - int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); - assert(ret == 0); - - initFsm(); - - ret = syncInit(); - assert(ret == 0); - - int64_t rid = syncNodeInit(); - assert(rid > 0); - - SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); - assert(pSyncNode != NULL); - - syncNodeLog2((char *)"", pSyncNode); - initRaftId(pSyncNode); - - for (int i = 0; i < recordCount; ++i) { - // step0 - SRpcMsg *pMsg0 = step0(i); - syncRpcMsgLog2((char *)"==step0==", pMsg0); - - syncPropose(rid, pMsg0, true); - taosMsleep(sleepMS); - - taosMemoryFree(pMsg0); - - sTrace( - "syncPropose sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " - "electTimerMS:%d, commitIndex:%ld", - pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, - pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, - pSyncNode->commitIndex); - } - - while (1) { - sTrace( - "replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " - "electTimerMS:%d, commitIndex:%ld", - pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, - pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, - pSyncNode->commitIndex); - taosMsleep(1000); - } - - return 0; -} From 3278a2309bea41993a12f0e7906fa474d41c2f6b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 21 Apr 2022 17:51:33 +0800 Subject: [PATCH 40/83] enh(rpc):add auth --- include/common/tmsg.h | 8 ++++++++ source/common/src/tmsg.c | 22 ++++++++++++++++++++++ source/libs/transport/src/trans.c | 9 ++++----- source/libs/transport/src/transCli.c | 11 ++--------- source/libs/transport/src/transSrv.c | 5 ----- 5 files changed, 36 insertions(+), 19 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 809390a9d2..b9a561d790 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -326,6 +326,13 @@ int32_t tDecodeSEpSet(SCoder* pDecoder, SEpSet* pEp); int32_t taosEncodeSEpSet(void** buf, const SEpSet* pEp); void* taosDecodeSEpSet(const void* buf, SEpSet* pEp); +typedef struct { + SEpSet epSet; +} SMEpSet; + +int32_t tSerializeSMEpSet(void* buf, int32_t bufLen, SMEpSet* pReq); +int32_t tDeserializeSMEpSet(void* buf, int32_t buflen, SMEpSet* pReq); + typedef struct { int8_t connType; int32_t pid; @@ -2690,6 +2697,7 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p } return buf; } + #pragma pack(pop) #ifdef __cplusplus diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 3622390f45..e0ae67597e 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -828,6 +828,28 @@ void tFreeSMAltertbReq(SMAltertbReq *pReq) { taosArrayDestroy(pReq->pFields); pReq->pFields = NULL; } +int32_t tSerializeSMEpSet(void *buf, int32_t bufLen, SMEpSet *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + if (tEncodeSEpSet(&encoder, &pReq->epSet) < 0) { + return -1; + } + tEndEncode(&encoder); + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} +int32_t tDeserializeSMEpSet(void *buf, int32_t bufLen, SMEpSet *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + if (tDecodeSEpSet(&decoder, &pReq->epSet) < 0) { + return -1; + } + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; +} int32_t tSerializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq) { SCoder encoder = {0}; diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index ebb90338cd..5f9af2eb48 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -100,11 +100,10 @@ void rpcSendRedirectRsp(void* thandle, const SEpSet* pEpSet) { SRpcMsg rpcMsg; memset(&rpcMsg, 0, sizeof(rpcMsg)); - rpcMsg.contLen = sizeof(SEpSet); - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - if (rpcMsg.pCont == NULL) return; - - memcpy(rpcMsg.pCont, pEpSet, sizeof(SEpSet)); + SMEpSet msg = {.epSet = *pEpSet}; + int32_t len = tSerializeSMEpSet(NULL, 0, &msg); + rpcMsg.pCont = rpcMallocCont(len); + tSerializeSMEpSet(rpcMsg.pCont, len, &msg); rpcMsg.code = TSDB_CODE_RPC_REDIRECT; rpcMsg.handle = thandle; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index b81310b90b..b2d0e7f020 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -31,12 +31,8 @@ typedef struct SCliConn { int hThrdIdx; STransCtx ctx; - bool broken; // link broken or not - ConnStatus status; // - int release; // 1: release - // spi configure - char spi; - char secured; + bool broken; // link broken or not + ConnStatus status; // char* ip; uint32_t port; @@ -44,7 +40,6 @@ typedef struct SCliConn { // debug and log info struct sockaddr_in addr; struct sockaddr_in locaddr; - } SCliConn; typedef struct SCliMsg { @@ -303,8 +298,6 @@ void cliHandleResp(SCliConn* conn) { TMSG_INFO(pHead->msgType), taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), taosInetNtoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port), transMsg.contLen); - conn->secured = pHead->secured; - if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) { tTrace("except, server continue send while cli ignore it"); // transUnrefCliHandle(conn); diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index ec66f3e8df..b1b2781859 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -30,7 +30,6 @@ typedef struct SSrvConn { uv_timer_t pTimer; queue queue; - int ref; int persist; // persist connection or not SConnBuffer readBuf; // read buf, int inType; @@ -692,8 +691,6 @@ static void uvDestroyConn(uv_handle_t* handle) { if (thrd->quit && QUEUE_IS_EMPTY(&thrd->conn)) { tTrace("work thread quit"); uv_walk(thrd->loop, uvWalkCb, NULL); - // uv_loop_close(thrd->loop); - // uv_stop(thrd->loop); } } @@ -756,8 +753,6 @@ void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd) { thrd->quit = true; if (QUEUE_IS_EMPTY(&thrd->conn)) { uv_walk(thrd->loop, uvWalkCb, NULL); - // uv_loop_close(thrd->loop); - // uv_stop(thrd->loop); } else { destroyAllConn(thrd); } From ad398bb626add41792e644e06941d0867d03a023 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 21 Apr 2022 18:00:12 +0800 Subject: [PATCH 41/83] run ci test again --- include/os/osEnv.h | 1 + source/dnode/mgmt/implement/src/dmHandle.c | 6 ++++-- source/os/src/osEnv.c | 1 + source/os/src/osProc.c | 2 +- tests/script/test.sh | 4 ++-- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/include/os/osEnv.h b/include/os/osEnv.h index 14d50858b7..a3f92a0b29 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -34,6 +34,7 @@ extern int64_t tsOpenMax; extern int64_t tsStreamMax; extern float tsNumOfCores; extern int64_t tsTotalMemoryKB; +extern char* tsProcPath; extern char configDir[]; extern char tsDataDir[]; diff --git a/source/dnode/mgmt/implement/src/dmHandle.c b/source/dnode/mgmt/implement/src/dmHandle.c index 94c6d20834..c59ff1521a 100644 --- a/source/dnode/mgmt/implement/src/dmHandle.c +++ b/source/dnode/mgmt/implement/src/dmHandle.c @@ -234,7 +234,10 @@ static int32_t dmSpawnUdfd(SDnode *pDnode) { dInfo("dnode start spawning udfd"); uv_process_options_t options = {0}; - char path[] = "udfd"; + char path[PATH_MAX] = {0}; + strncpy(path, tsProcPath, strlen(tsProcPath)); + char* dirName = taosDirName(path); + strcat(path, "/udfd"); char* argsUdfd[] = {path, "-c", configDir, NULL}; options.args = argsUdfd; options.file = path; @@ -261,7 +264,6 @@ static int32_t dmSpawnUdfd(SDnode *pDnode) { options.env = envUdfd; int err = uv_spawn(&pData->loop, &pData->process, &options); - pData->process.data = (void*)pDnode; if (err != 0) { diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 22884298ef..e24ac41f20 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -37,6 +37,7 @@ int64_t tsOpenMax = 0; int64_t tsStreamMax = 0; float tsNumOfCores = 0; int64_t tsTotalMemoryKB = 0; +char* tsProcPath = NULL; void osDefaultInit() { taosSeedRand(taosSafeRand()); diff --git a/source/os/src/osProc.c b/source/os/src/osProc.c index b6de638ac2..d569582256 100644 --- a/source/os/src/osProc.c +++ b/source/os/src/osProc.c @@ -17,7 +17,7 @@ #define _DEFAULT_SOURCE #include "os.h" -static char *tsProcPath = NULL; +char *tsProcPath = NULL; int32_t taosNewProc(char **args) { int32_t pid = fork(); diff --git a/tests/script/test.sh b/tests/script/test.sh index 14dc43beaf..e4191da0a9 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -131,8 +131,8 @@ if [ -n "$FILE_NAME" ]; then FLAG="-v" fi - echo valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG - valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG + echo valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --child-silent-after-fork=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG + valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --child-silent-after-fork=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG else if [[ $MULTIPROCESS -eq 1 ]];then echo "ExcuteCmd(multiprocess):" $PROGRAM -m -c $CFG_DIR -f $FILE_NAME From 219c8dbf377937c65a08c09072b360be972ed6c1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 21 Apr 2022 19:33:48 +0800 Subject: [PATCH 42/83] enh(rpc):add auth --- include/libs/transport/trpc.h | 20 ++++++++++++-------- source/libs/transport/inc/transportInt.h | 13 +++++++------ source/libs/transport/src/trans.c | 1 + source/libs/transport/src/transCli.c | 18 ++++++++++++++++-- 4 files changed, 36 insertions(+), 16 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 3e2f596784..ab26cfc155 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -54,12 +54,13 @@ typedef struct { uint16_t clientPort; SRpcMsg rpcMsg; int32_t rspLen; - void *pRsp; - void *pNode; + void * pRsp; + void * pNode; } SNodeMsg; typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *); typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); +typedef int (*RpcRfp)(void *parent, SRpcMsg *, SEpSet *); typedef struct SRpcInit { uint16_t localPort; // local port @@ -80,22 +81,25 @@ typedef struct SRpcInit { RpcCfp cfp; // call back to retrieve the client auth info, for server app only - RpcAfp afp;; + RpcAfp afp; + + // user defined retry func + RpcRfp rfp; void *parent; } SRpcInit; typedef struct { - void *val; + void *val; int32_t (*clone)(void *src, void **dst); - void (*freeFunc)(const void *arg); + void (*freeFunc)(const void *arg); } SRpcCtxVal; typedef struct { - int32_t msgType; - void *val; + int32_t msgType; + void * val; int32_t (*clone)(void *src, void **dst); - void (*freeFunc)(const void *arg); + void (*freeFunc)(const void *arg); } SRpcBrokenlinkVal; typedef struct { diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index ad50948a02..eaca9b0fc7 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -63,13 +63,14 @@ typedef struct { void (*cfp)(void* parent, SRpcMsg*, SEpSet*); int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); + int (*retry)(void* parent, SRpcMsg*, SEpSet*); - int32_t refCount; - void* parent; - void* idPool; // handle to ID pool - void* tmrCtrl; // handle to timer - SHashObj* hash; // handle returned by hash utility - void* tcphandle; // returned handle from TCP initialization + int32_t refCount; + void* parent; + void* idPool; // handle to ID pool + void* tmrCtrl; // handle to timer + SHashObj* hash; // handle returned by hash utility + void* tcphandle; // returned handle from TCP initialization TdThreadMutex mutex; } SRpcInfo; diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 5f9af2eb48..fa517d6d61 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -39,6 +39,7 @@ void* rpcOpen(const SRpcInit* pInit) { // register callback handle pRpc->cfp = pInit->cfp; pRpc->afp = pInit->afp; + pRpc->retry = pInit->rfp; if (pInit->connType == TAOS_CONN_SERVER) { pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index b2d0e7f020..46eb040468 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -97,6 +97,8 @@ static void cliSendCb(uv_write_t* req, int status); static void cliConnCb(uv_connect_t* req, int status); static void cliAsyncCb(uv_async_t* handle); +static void cliAppCb(SCliConn* pConn, STransMsg* pMsg); + static SCliConn* cliCreateConn(SCliThrdObj* thrd); static void cliDestroyConn(SCliConn* pConn, bool clear /*clear tcp handle or not*/); static void cliDestroy(uv_handle_t* handle); @@ -311,7 +313,8 @@ void cliHandleResp(SCliConn* conn) { if (pCtx == NULL || pCtx->pSem == NULL) { tTrace("%s cli conn %p handle resp", pTransInst->label, conn); - (pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); + cliAppCb(conn, &transMsg); + //(pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); } else { tTrace("%s cli conn(sync) %p handle resp", pTransInst->label, conn); memcpy((char*)pCtx->pRsp, (char*)&transMsg, sizeof(transMsg)); @@ -377,7 +380,8 @@ void cliHandleExcept(SCliConn* pConn) { once = true; continue; } - (pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); + cliAppCb(pConn, &transMsg); + //(pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); } else { tTrace("%s cli conn(sync) %p handle except", pTransInst->label, pConn); memcpy((char*)(pCtx->pRsp), (char*)(&transMsg), sizeof(transMsg)); @@ -877,6 +881,16 @@ int cliRBChoseIdx(STrans* pTransInst) { } return index % pTransInst->numOfThreads; } +void cliAppCb(SCliConn* pConn, STransMsg* transMsg) { + SCliThrdObj* pThrd = pConn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; + + if (transMsg->code == TSDB_CODE_RPC_REDIRECT && pTransInst->retry != NULL) { + // impl retry + } else { + (*pTransInst->cfp)(pTransInst->parent, transMsg, NULL); + } +} void transCloseClient(void* arg) { SCliObj* cli = arg; From fa14caffcd3643f8d7139b404e2f31eb46328f73 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 21 Apr 2022 20:28:37 +0800 Subject: [PATCH 43/83] enh(rpc):add auth --- source/libs/transport/src/transCli.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 46eb040468..b43b8a1e0c 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -886,9 +886,11 @@ void cliAppCb(SCliConn* pConn, STransMsg* transMsg) { STrans* pTransInst = pThrd->pTransInst; if (transMsg->code == TSDB_CODE_RPC_REDIRECT && pTransInst->retry != NULL) { - // impl retry + SMEpSet emsg = {0}; + tDeserializeSMEpSet(transMsg->pCont, transMsg->contLen, &emsg); + pTransInst->retry(pTransInst, transMsg, &(emsg.epSet)); } else { - (*pTransInst->cfp)(pTransInst->parent, transMsg, NULL); + pTransInst->cfp(pTransInst->parent, transMsg, NULL); } } From 0f22bf3ef989e2e17aaf291ce8b606ab883b4cc6 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 21 Apr 2022 21:24:27 +0800 Subject: [PATCH 44/83] enh(rpc):add auth --- source/common/src/tmsg.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 90283f1e8c..3d5b6397ca 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -847,9 +847,9 @@ void tFreeSMAltertbReq(SMAltertbReq *pReq) { int32_t tSerializeSMEpSet(void *buf, int32_t bufLen, SMEpSet *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); - if (tEncodeSEpSet(&encoder, &pReq->epSet) < 0) { - return -1; - } + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeSEpSet(&encoder, &pReq->epSet) < 0) return -1; + tEndEncode(&encoder); int32_t tlen = encoder.pos; tCoderClear(&encoder); @@ -858,9 +858,8 @@ int32_t tSerializeSMEpSet(void *buf, int32_t bufLen, SMEpSet *pReq) { int32_t tDeserializeSMEpSet(void *buf, int32_t bufLen, SMEpSet *pReq) { SCoder decoder = {0}; tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); - if (tDecodeSEpSet(&decoder, &pReq->epSet) < 0) { - return -1; - } + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeSEpSet(&decoder, &pReq->epSet) < 0) return -1; tEndDecode(&decoder); tCoderClear(&decoder); From eefb0a2049f3698ab8d3816d3fdc2fadb8e28205 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 21 Apr 2022 22:03:29 +0800 Subject: [PATCH 45/83] enh(rpc):add auth --- source/libs/transport/src/transSrv.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index b1b2781859..59a30051ef 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -846,10 +846,8 @@ void transRefSrvHandle(void* handle) { if (handle == NULL) { return; } - SSrvConn* conn = handle; - int ref = T_REF_INC((SSrvConn*)handle); - UNUSED(ref); + tDebug("server conn %p ref count: %d", handle, ref); } void transUnrefSrvHandle(void* handle) { From 7c54b699778ca67ad4b6ed4dc5666f317240bf18 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 22 Apr 2022 08:13:45 +0800 Subject: [PATCH 46/83] sigkill to kill taosd causes udfd to exit --- source/dnode/mgmt/implement/src/dmHandle.c | 12 +++-- source/dnode/mgmt/interface/inc/dmDef.h | 1 + source/libs/function/src/udfd.c | 61 +++++++++++++++++----- 3 files changed, 56 insertions(+), 18 deletions(-) diff --git a/source/dnode/mgmt/implement/src/dmHandle.c b/source/dnode/mgmt/implement/src/dmHandle.c index c59ff1521a..7b7ab7fa9a 100644 --- a/source/dnode/mgmt/implement/src/dmHandle.c +++ b/source/dnode/mgmt/implement/src/dmHandle.c @@ -226,6 +226,7 @@ void dmUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) { if (atomic_load_8(&pData->stopping) != 0) { dDebug("udfd process exit due to stopping"); } else { + uv_close((uv_handle_t*)&pData->ctrlPipe, NULL); dmSpawnUdfd(pDnode); } } @@ -243,20 +244,21 @@ static int32_t dmSpawnUdfd(SDnode *pDnode) { options.file = path; options.exit_cb = dmUdfdExit; + SUdfdData *pData = &pDnode->udfdData; + uv_pipe_init(&pData->loop, &pData->ctrlPipe, 1); - options.stdio_count = 3; uv_stdio_container_t child_stdio[3]; - child_stdio[0].flags = UV_IGNORE; - child_stdio[1].flags = UV_INHERIT_FD; - child_stdio[1].data.fd = 1; + child_stdio[0].flags = UV_CREATE_PIPE | UV_READABLE_PIPE; + child_stdio[0].data.stream = (uv_stream_t*) &pData->ctrlPipe; + child_stdio[1].flags = UV_IGNORE; child_stdio[2].flags = UV_INHERIT_FD; child_stdio[2].data.fd = 2; + options.stdio_count = 3; options.stdio = child_stdio; char dnodeIdEnvItem[32] = {0}; char thrdPoolSizeEnvItem[32] = {0}; snprintf(dnodeIdEnvItem, 32, "%s=%d", "DNODE_ID", pDnode->data.dnodeId); - SUdfdData *pData = &pDnode->udfdData; float numCpuCores = 4; taosGetCpuCores(&numCpuCores); snprintf(thrdPoolSizeEnvItem,32, "%s=%d", "UV_THREADPOOL_SIZE", (int)numCpuCores*2); diff --git a/source/dnode/mgmt/interface/inc/dmDef.h b/source/dnode/mgmt/interface/inc/dmDef.h index e76ac73b85..4f4a2ed349 100644 --- a/source/dnode/mgmt/interface/inc/dmDef.h +++ b/source/dnode/mgmt/interface/inc/dmDef.h @@ -152,6 +152,7 @@ typedef struct SUdfdData { uv_process_t process; int spawnErr; int8_t stopping; + uv_pipe_t ctrlPipe; } SUdfdData; typedef struct SDnode { diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 1dd0871ae9..d6e7a43666 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -27,7 +27,10 @@ typedef struct SUdfdContext { uv_loop_t *loop; + uv_pipe_t ctrlPipe; + uv_signal_t intrSignal; char listenPipeName[UDF_LISTEN_PIPE_NAME_LEN]; + uv_pipe_t listeningPipe; void *clientRpc; uv_mutex_t udfsMutex; @@ -380,10 +383,12 @@ void udfdOnNewConnection(uv_stream_t *server, int status) { } } -void removeListeningPipe(int sig) { +void udfdIntrSignalHandler(uv_signal_t *handle, int signum) { + fnInfo("udfd signal received: %d\n", signum); uv_fs_t req; - uv_fs_unlink(global.loop, &req, "udf.sock", NULL); - exit(0); + uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL); + uv_signal_stop(handle); + uv_stop(global.loop); } void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { return; } @@ -492,37 +497,67 @@ static int32_t udfdInitLog() { return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, 0); } +void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf) { + buf->base = taosMemoryMalloc(suggested_size); + buf->len = suggested_size; +} + +void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf) { + if (nread < 0) { + fnError("udfd ctrl pipe read error. %s", uv_err_name(nread)); + uv_close((uv_handle_t*)q, NULL); + uv_stop(global.loop); + return; + } + fnError("udfd ctrl pipe read %zu bytes", nread); + taosMemoryFree(buf->base); +} + +static int32_t removeListeningPipe() { + uv_fs_t req; + int err = uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL); + uv_fs_req_cleanup(&req); + return err; +} + static int32_t udfdUvInit() { uv_loop_t* loop = taosMemoryMalloc(sizeof(uv_loop_t)); if (loop) { uv_loop_init(loop); } global.loop = loop; + + uv_pipe_init(global.loop, &global.ctrlPipe, 1); + uv_pipe_open(&global.ctrlPipe, 0); + uv_read_start((uv_stream_t*)&global.ctrlPipe, udfdCtrlAllocBufCb, udfdCtrlReadCb); + char dnodeId[8] = {0}; size_t dnodeIdSize; - uv_os_getenv("DNODE_ID", dnodeId, &dnodeIdSize); + int32_t err = uv_os_getenv("DNODE_ID", dnodeId, &dnodeIdSize); + if (err != 0) { + dnodeId[0] = '1'; + } char listenPipeName[32] = {0}; snprintf(listenPipeName, sizeof(listenPipeName), "%s%s", UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId); strcpy(global.listenPipeName, listenPipeName); - uv_fs_t req; - uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL); + removeListeningPipe(); - uv_pipe_t server; - uv_pipe_init(global.loop, &server, 0); + uv_pipe_init(global.loop, &global.listeningPipe, 0); - signal(SIGINT, removeListeningPipe); + uv_signal_init(global.loop, &global.intrSignal); + uv_signal_start(&global.intrSignal, udfdIntrSignalHandler, SIGINT); int r; fnInfo("bind to pipe %s", global.listenPipeName); - if ((r = uv_pipe_bind(&server, listenPipeName))) { + if ((r = uv_pipe_bind(&global.listeningPipe, listenPipeName))) { fnError("Bind error %s", uv_err_name(r)); - removeListeningPipe(0); + removeListeningPipe(); return -1; } - if ((r = uv_listen((uv_stream_t *)&server, 128, udfdOnNewConnection))) { + if ((r = uv_listen((uv_stream_t *)&global.listeningPipe, 128, udfdOnNewConnection))) { fnError("Listen error %s", uv_err_name(r)); - removeListeningPipe(0); + removeListeningPipe(); return -2; } return 0; From a01f1a4e1d5752ab7c0e8f97d2ba8901eba4cd3b Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 22 Apr 2022 08:45:38 +0800 Subject: [PATCH 47/83] fix unit test case that causes core dump --- source/dnode/mgmt/implement/src/dmHandle.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/implement/src/dmHandle.c b/source/dnode/mgmt/implement/src/dmHandle.c index 7b7ab7fa9a..376f589acd 100644 --- a/source/dnode/mgmt/implement/src/dmHandle.c +++ b/source/dnode/mgmt/implement/src/dmHandle.c @@ -236,8 +236,12 @@ static int32_t dmSpawnUdfd(SDnode *pDnode) { uv_process_options_t options = {0}; char path[PATH_MAX] = {0}; - strncpy(path, tsProcPath, strlen(tsProcPath)); - char* dirName = taosDirName(path); + if (tsProcPath == NULL) { + path[0] = '.'; + } else { + strncpy(path, tsProcPath, strlen(tsProcPath)); + taosDirName(path); + } strcat(path, "/udfd"); char* argsUdfd[] = {path, "-c", configDir, NULL}; options.args = argsUdfd; From 31528bbe55324d8b52da5821de388009a6eaf53f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 22 Apr 2022 09:16:00 +0800 Subject: [PATCH 48/83] feature/qnode --- source/libs/parser/src/parInsert.c | 10 ++-- source/libs/parser/src/parTranslater.c | 3 ++ source/libs/scalar/inc/filterInt.h | 6 ++- source/libs/scalar/src/filter.c | 52 +++++++++++++++++-- .../libs/scalar/test/filter/filterTests.cpp | 38 ++++++++++++-- 5 files changed, 94 insertions(+), 15 deletions(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index f21a738032..cdccf0b1a1 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1407,11 +1407,7 @@ int32_t qBindStmtSingleColValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, } SSchema* pColSchema = &pSchema[spd->boundColumns[colIdx] - 1]; - - if (bind->buffer_type != pColSchema->type) { - return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); - } - + if (bind->num != rowNum) { return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); } @@ -1426,6 +1422,10 @@ int32_t qBindStmtSingleColValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, CHECK_CODE(MemRowAppend(&pBuf, NULL, 0, ¶m)); } else { + if (bind->buffer_type != pColSchema->type) { + return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); + } + int32_t colLen = pColSchema->bytes; if (IS_VAR_DATA_TYPE(pColSchema->type)) { colLen = bind->length[r]; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 322dc004e2..6f1531bd5f 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -547,6 +547,9 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { TSDB_DATA_TYPE_BLOB == rdt.type) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); } + if (OP_TYPE_IN == pOp->opType || OP_TYPE_NOT_IN == pOp->opType) { + ((SExprNode*)pOp->pRight)->resType = ((SExprNode*)pOp->pLeft)->resType; + } pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; } else if (nodesIsJsonOp(pOp)){ diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index 2b9e3c7f39..99c33d3165 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -215,8 +215,10 @@ typedef struct SFilterPCtx { } SFilterPCtx; typedef struct SFltTreeStat { - int32_t code; - bool scalarMode; + int32_t code; + int8_t precision; + bool scalarMode; + SFilterInfo* info; } SFltTreeStat; typedef struct SFltScalarCtx { diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 23047c2930..e24162bccb 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3364,6 +3364,12 @@ int32_t filterGetTimeRangeImpl(SFilterInfo *info, STimeWindow *win, bool * filterGetRangeRes(prev, &tra); win->skey = tra.s; win->ekey = tra.e; + if (FILTER_GET_FLAG(tra.sflag, RANGE_FLG_EXCLUDE)) { + win->skey++; + } + if (FILTER_GET_FLAG(tra.eflag, RANGE_FLG_EXCLUDE)) { + win->ekey--; + } } filterFreeRangeCtx(prev); @@ -3492,7 +3498,40 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } - if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { + if (QUERY_NODE_VALUE == nodeType(*pNode)) { + if (!FILTER_GET_FLAG(stat->info->options, FLT_OPTION_TIMESTAMP)) { + return DEAL_RES_CONTINUE; + } + + SValueNode *valueNode = (SValueNode *)*pNode; + if (TSDB_DATA_TYPE_BINARY != valueNode->node.resType.type) { + return DEAL_RES_CONTINUE; + } + +#if 0 + if (stat->precision < 0) { + //TODO + return DEAL_RES_CONTINUE; + } + + char *timeStr = valueNode->datum.p; + if (taosParseTime(valueNode->datum.p, &valueNode->datum.i, valueNode->node.resType.bytes, stat->precision, tsDaylight) != + TSDB_CODE_SUCCESS) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); + } + TODO +#else + return DEAL_RES_CONTINUE; +#endif + } + + if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + SColumnNode *colNode = (SColumnNode *)*pNode; + stat->precision = colNode->node.resType.precision; + return DEAL_RES_CONTINUE; + } + + if (QUERY_NODE_NODE_LIST == nodeType(*pNode)) { return DEAL_RES_CONTINUE; } @@ -3656,16 +3695,19 @@ int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options) info = *pInfo; info->options = options; - SFltTreeStat stat1 = {0}; - FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat1)); + SFltTreeStat stat = {0}; + stat.precision = -1; + stat.info = info; + + FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat)); - info->scalarMode = stat1.scalarMode; + info->scalarMode = stat.scalarMode; if (!info->scalarMode) { FLT_ERR_JRET(fltInitFromNode(pNode, info, options)); } else { info->sclCtx.node = pNode; - FLT_ERR_JRET(fltOptimizeNodes(info, &info->sclCtx.node, &stat1)); + FLT_ERR_JRET(fltOptimizeNodes(info, &info->sclCtx.node, &stat)); } return code; diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 42998aba00..9670dc0180 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -242,7 +242,7 @@ TEST(timerangeTest, greater) { int32_t code = filterGetTimeRange(opNode1, &win, &isStrict); ASSERT_EQ(code, 0); ASSERT_EQ(isStrict, true); - ASSERT_EQ(win.skey, tsmall); + ASSERT_EQ(win.skey, tsmall+1); ASSERT_EQ(win.ekey, INT64_MAX); //filterFreeInfo(filter); nodesDestroyNode(opNode1); @@ -265,6 +265,37 @@ TEST(timerangeTest, greater_and_lower) { flttMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); + //SFilterInfo *filter = NULL; + //int32_t code = filterInitFromNode(logicNode, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP); + //ASSERT_EQ(code, 0); + STimeWindow win = {0}; + bool isStrict = false; + int32_t code = filterGetTimeRange(logicNode, &win, &isStrict); + ASSERT_EQ(isStrict, true); + ASSERT_EQ(code, 0); + ASSERT_EQ(win.skey, tsmall+1); + ASSERT_EQ(win.ekey, tbig-1); + //filterFreeInfo(filter); + nodesDestroyNode(logicNode); +} + +TEST(timerangeTest, greater_equal_and_lower_equal) { + SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL; + bool eRes[5] = {false, false, true, true, true}; + SScalarParam res = {0}; + int64_t tsmall = 222, tbig = 333; + flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); + flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pcol, pval); + flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); + flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tbig); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pcol, pval); + SNode *list[2] = {0}; + list[0] = opNode1; + list[1] = opNode2; + + flttMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); + //SFilterInfo *filter = NULL; //int32_t code = filterInitFromNode(logicNode, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP); //ASSERT_EQ(code, 0); @@ -279,6 +310,7 @@ TEST(timerangeTest, greater_and_lower) { nodesDestroyNode(logicNode); } + TEST(timerangeTest, greater_and_lower_not_strict) { SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode1 = NULL, *logicNode2 = NULL; bool eRes[5] = {false, false, true, true, true}; @@ -321,8 +353,8 @@ TEST(timerangeTest, greater_and_lower_not_strict) { int32_t code = filterGetTimeRange(logicNode1, &win, &isStrict); ASSERT_EQ(isStrict, false); ASSERT_EQ(code, 0); - ASSERT_EQ(win.skey, tsmall1); - ASSERT_EQ(win.ekey, tbig2); + ASSERT_EQ(win.skey, tsmall1+1); + ASSERT_EQ(win.ekey, tbig2-1); //filterFreeInfo(filter); nodesDestroyNode(logicNode1); } From fc2452162171a0584aa108e233475c02860c5fd9 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Fri, 22 Apr 2022 09:54:27 +0800 Subject: [PATCH 49/83] fix(os): windows compile 3.0. --- cmake/cmake.define | 13 +- cmake/cmake.options | 25 +- cmake/msvcregex_CMakeLists.txt.in | 13 + cmake/pthread_CMakeLists.txt.in | 2 +- cmake/wcwidth_CMakeLists.txt.in | 13 + cmake/wingetopt_CMakeLists.txt.in | 13 + contrib/CMakeLists.txt | 95 ++++- contrib/test/craft/CMakeLists.txt | 9 +- contrib/test/tdev/src/main.c | 72 ++-- example/CMakeLists.txt | 1 + example/src/demoapi.c | 40 ++- include/common/trow.h | 2 +- include/dnode/bnode/bnode.h | 3 + include/libs/qcom/query.h | 4 + include/os/os.h | 10 +- include/os/osAtomic.h | 8 +- include/os/osDef.h | 2 +- include/os/osDir.h | 6 +- include/os/osFile.h | 1 - include/os/osLz4.h | 2 +- include/os/osMath.h | 3 +- include/os/osSocket.h | 114 +++--- include/os/osString.h | 2 +- include/os/osThread.h | 13 +- include/os/osTime.h | 8 +- include/util/tencode.h | 6 +- include/util/tjson.h | 4 + source/common/CMakeLists.txt | 4 +- source/common/test/commonTests.cpp | 2 +- source/dnode/mgmt/exe/dmMain.c | 4 +- source/dnode/vnode/src/meta/metaIdx.c | 3 + source/dnode/vnode/src/meta/metaTDBImpl.c | 4 +- source/dnode/vnode/src/tq/tqMetaStore.c | 6 +- source/libs/catalog/inc/catalogInt.h | 4 + source/libs/catalog/test/catalogTests.cpp | 117 +++++-- source/libs/executor/src/groupoperator.c | 8 +- source/libs/executor/src/tsimplehash.c | 6 +- source/libs/executor/test/executorTests.cpp | 6 +- source/libs/function/CMakeLists.txt | 3 + source/libs/function/inc/tudfInt.h | 3 + source/libs/function/inc/udfc.h | 3 + source/libs/index/inc/indexFstRegex.h | 3 + source/libs/index/src/indexTfile.c | 2 +- source/libs/index/test/indexTests.cc | 6 +- source/libs/index/test/jsonUT.cc | 2 +- source/libs/parser/inc/parToken.h | 1 - source/libs/parser/src/parAstParser.c | 1 + source/libs/parser/src/parInsert.c | 1 + source/libs/parser/src/parTranslater.c | 14 +- source/libs/parser/src/parser.c | 1 + source/libs/parser/src/sql.c | 1 + source/libs/parser/test/CMakeLists.txt | 8 + source/libs/parser/test/mockCatalog.cpp | 4 +- .../libs/parser/test/mockCatalogService.cpp | 12 +- source/libs/parser/test/parserInsertTest.cpp | 1 + source/libs/parser/test/parserTestMain.cpp | 8 +- source/libs/planner/test/CMakeLists.txt | 9 + source/libs/planner/test/planTestMain.cpp | 2 +- source/libs/planner/test/planTestUtil.cpp | 1 + source/libs/qworker/inc/qworkerInt.h | 3 + source/libs/qworker/test/qworkerTests.cpp | 84 ++++- source/libs/scalar/src/sclfunc.c | 10 +- .../libs/scalar/test/filter/filterTests.cpp | 41 ++- .../libs/scalar/test/scalar/scalarTests.cpp | 3 + source/libs/scheduler/inc/schedulerInt.h | 12 + source/libs/scheduler/test/schedulerTests.cpp | 36 +- source/libs/sync/src/syncEnv.c | 2 +- source/libs/tdb/CMakeLists.txt | 2 +- source/libs/tdb/src/db/tdbBtree.c | 9 +- source/libs/tdb/src/db/tdbPCache.c | 15 +- source/libs/tdb/src/db/tdbPage.c | 13 +- source/libs/tdb/src/db/tdbPager.c | 4 +- source/libs/tdb/src/inc/tdbPage.h | 30 +- source/libs/tdb/src/inc/tdbUtil.h | 28 +- source/libs/tfs/src/tfs.c | 6 +- source/libs/transport/test/rclient.c | 2 - source/libs/transport/test/syncClient.c | 2 - source/libs/transport/test/transportTests.cpp | 12 +- source/libs/wal/src/walMeta.c | 4 +- source/os/CMakeLists.txt | 14 +- source/os/src/osAtomic.c | 328 +++++++++++------- source/os/src/osDir.c | 170 ++++++--- source/os/src/osEnv.c | 4 +- source/os/src/osFile.c | 209 ++++------- source/os/src/osLocale.c | 4 +- source/os/src/osLz4.c | 2 +- source/os/src/osMemory.c | 6 +- source/os/src/osProc.c | 20 +- source/os/src/osRand.c | 23 +- source/os/src/osSemaphore.c | 2 +- source/os/src/osShm.c | 12 + source/os/src/osSignal.c | 4 +- source/os/src/osSleep.c | 15 +- source/os/src/osSocket.c | 51 ++- source/os/src/osString.c | 49 +++ source/os/src/osStrptime.c | 4 +- source/os/src/osSysinfo.c | 74 ++-- source/os/src/osSystem.c | 25 +- source/os/src/osThread.c | 21 +- source/os/src/osTime.c | 41 ++- source/os/src/osTimer.c | 6 +- source/os/src/osTimezone.c | 7 +- source/util/CMakeLists.txt | 2 +- source/util/src/tcache.c | 3 +- source/util/src/tconfig.c | 2 +- source/util/src/thash.c | 2 +- source/util/src/tlog.c | 1 + source/util/src/tprocess.c | 6 +- source/util/src/tskiplist.c | 2 +- source/util/src/ttimer.c | 2 +- source/util/test/freelistTest.cpp | 2 +- source/util/test/procTest.cpp | 104 +++--- source/util/test/trefTest.c | 6 +- tests/test/c/tmqDemo.c | 9 +- tests/test/c/tmqSim.c | 7 +- tools/shell/src/shellEngine.c | 1 - tools/shell/src/shellMain.c | 16 + 117 files changed, 1533 insertions(+), 775 deletions(-) create mode 100644 cmake/msvcregex_CMakeLists.txt.in create mode 100644 cmake/wcwidth_CMakeLists.txt.in create mode 100644 cmake/wingetopt_CMakeLists.txt.in diff --git a/cmake/cmake.define b/cmake/cmake.define index c985ba1cc7..c9a188600a 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -1,5 +1,7 @@ cmake_minimum_required(VERSION 3.16) +set(CMAKE_VERBOSE_MAKEFILE ON) + #set output directory SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib) SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin) @@ -43,11 +45,14 @@ ENDIF () IF (TD_WINDOWS) MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}") SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) - SET(COMMON_FLAGS "/nologo /WX /wd4018 /wd4999 /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") + SET(COMMON_FLAGS "/W3 /D_WIN32") - IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) - SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") - ENDIF () + # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) + # SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") + # ENDIF () + + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMMON_FLAGS}") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON_FLAGS}") ELSE () IF (${SANITIZER} MATCHES "true") diff --git a/cmake/cmake.options b/cmake/cmake.options index b51f096185..f32c5acdd1 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -25,9 +25,28 @@ IF(${TD_WINDOWS}) ON ) -ENDIF () + MESSAGE("build msvcregex Win32") + option( + BUILD_MSVCREGEX + "If build msvcregex on Windows" + ON + ) -IF(${TD_LINUX} MATCHES TRUE) + MESSAGE("build wcwidth Win32") + option( + BUILD_WCWIDTH + "If build wcwidth on Windows" + ON + ) + + MESSAGE("build wingetopt Win32") + option( + BUILD_WINGETOPT + "If build wingetopt on Windows" + ON + ) + +ENDIF () option( BUILD_TEST @@ -35,8 +54,6 @@ option( ON ) -ENDIF () - option( BUILD_WITH_LEVELDB "If build with leveldb" diff --git a/cmake/msvcregex_CMakeLists.txt.in b/cmake/msvcregex_CMakeLists.txt.in new file mode 100644 index 0000000000..686ac4cb5d --- /dev/null +++ b/cmake/msvcregex_CMakeLists.txt.in @@ -0,0 +1,13 @@ + +# msvcregex +ExternalProject_Add(msvcregex + GIT_REPOSITORY https://gitee.com/l0km/libgnurx-msvc.git + GIT_TAG master + SOURCE_DIR "${TD_CONTRIB_DIR}/msvcregex" + BINARY_DIR "" + #BUILD_IN_SOURCE TRUE + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) \ No newline at end of file diff --git a/cmake/pthread_CMakeLists.txt.in b/cmake/pthread_CMakeLists.txt.in index 2553df6500..774f5dd972 100644 --- a/cmake/pthread_CMakeLists.txt.in +++ b/cmake/pthread_CMakeLists.txt.in @@ -3,7 +3,7 @@ ExternalProject_Add(pthread GIT_REPOSITORY https://github.com/GerHobbelt/pthread-win32 GIT_TAG v3.0.3.1 - SOURCE_DIR "${TD_CONTRIB_DIR}/pthread-win32" + SOURCE_DIR "${TD_CONTRIB_DIR}/pthread" BINARY_DIR "" #BUILD_IN_SOURCE TRUE CONFIGURE_COMMAND "" diff --git a/cmake/wcwidth_CMakeLists.txt.in b/cmake/wcwidth_CMakeLists.txt.in new file mode 100644 index 0000000000..234ccbf915 --- /dev/null +++ b/cmake/wcwidth_CMakeLists.txt.in @@ -0,0 +1,13 @@ + +# wcwidth +ExternalProject_Add(wcwidth + GIT_REPOSITORY https://github.com/fumiyas/wcwidth-cjk.git + GIT_TAG master + SOURCE_DIR "${TD_CONTRIB_DIR}/wcwidth" + BINARY_DIR "" + #BUILD_IN_SOURCE TRUE + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) \ No newline at end of file diff --git a/cmake/wingetopt_CMakeLists.txt.in b/cmake/wingetopt_CMakeLists.txt.in new file mode 100644 index 0000000000..b93925c170 --- /dev/null +++ b/cmake/wingetopt_CMakeLists.txt.in @@ -0,0 +1,13 @@ + +# wingetopt +ExternalProject_Add(wingetopt + GIT_REPOSITORY https://github.com/alex85k/wingetopt.git + GIT_TAG master + SOURCE_DIR "${TD_CONTRIB_DIR}/wingetopt" + BINARY_DIR "" + #BUILD_IN_SOURCE TRUE + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) \ No newline at end of file diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 1ddc765c5c..19923a5ad6 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -14,9 +14,24 @@ if(${BUILD_PTHREAD}) cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif() -# gnu regex -if(${BUILD_GNUREGEX}) - cat("${TD_SUPPORT_DIR}/gnuregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +# iconv +if(${BUILD_WITH_ICONV}) + cat("${TD_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif() + +# msvc regex +if(${BUILD_MSVCREGEX}) + cat("${TD_SUPPORT_DIR}/msvcregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif() + +# wcwidth +if(${BUILD_WCWIDTH}) + cat("${TD_SUPPORT_DIR}/wcwidth_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif() + +# wingetopt +if(${BUILD_WINGETOPT}) + cat("${TD_SUPPORT_DIR}/wingetopt_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif() # googletest @@ -99,8 +114,27 @@ if(${BUILD_TEST}) target_include_directories( gtest PUBLIC $ - PUBLIC $ ) + if(${TD_WINDOWS}) + target_include_directories( + gtest + PUBLIC $ + ) + endif(${TD_WINDOWS}) + if(${TD_LINUX}) + target_include_directories( + gtest + PUBLIC $ + ) + endif(${TD_LINUX}) + if(${TD_DARWIN}) + target_include_directories( + gtest + PUBLIC $ + ) + endif(${TD_DARWIN}) + + endif(${BUILD_TEST}) # cJson @@ -182,6 +216,53 @@ if(${BUILD_WITH_NURAFT}) add_subdirectory(nuraft) endif(${BUILD_WITH_NURAFT}) +# pthread +if(${BUILD_PTHREAD}) + set(CMAKE_BUILD_TYPE release) + add_definitions(-DPTW32_STATIC_LIB) + add_subdirectory(pthread) + set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread) + add_library(pthread STATIC IMPORTED GLOBAL) + SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib) +endif() + +# iconv +if(${BUILD_WITH_ICONV}) + add_subdirectory(iconv) +endif(${BUILD_WITH_ICONV}) + +# wingetopt +if(${BUILD_WINGETOPT}) + add_subdirectory(wingetopt) +endif(${BUILD_WINGETOPT}) + +# msvcregex +if(${BUILD_MSVCREGEX}) + add_library(msvcregex STATIC "") + target_sources(msvcregex + PRIVATE "msvcregex/regex.c" + ) + target_include_directories(msvcregex + PRIVATE "msvcregex" + ) + target_link_libraries(msvcregex + INTERFACE Shell32 + ) + SET_TARGET_PROPERTIES(msvcregex PROPERTIES OUTPUT_NAME msvcregex) +endif(${BUILD_MSVCREGEX}) + +# msvcregex +if(${BUILD_WCWIDTH}) + add_library(wcwidth STATIC "") + target_sources(wcwidth + PRIVATE "wcwidth/wcwidth.c" + ) + target_include_directories(wcwidth + PRIVATE "wcwidth" + ) + SET_TARGET_PROPERTIES(wcwidth PROPERTIES OUTPUT_NAME wcwidth) +endif(${BUILD_WCWIDTH}) + # CRAFT if(${BUILD_WITH_CRAFT}) add_library(craft STATIC IMPORTED GLOBAL) @@ -238,8 +319,12 @@ if(${BUILD_WITH_SQLITE}) target_link_libraries(sqlite INTERFACE m INTERFACE pthread - INTERFACE dl ) + if(NOT TD_WINDOWS) + target_link_libraries(sqlite + INTERFACE dl + ) + endif(NOT TD_WINDOWS) endif(${BUILD_WITH_SQLITE}) # pthread diff --git a/contrib/test/craft/CMakeLists.txt b/contrib/test/craft/CMakeLists.txt index e0f6ae64bd..ec8b44b673 100644 --- a/contrib/test/craft/CMakeLists.txt +++ b/contrib/test/craft/CMakeLists.txt @@ -1,2 +1,9 @@ add_executable(simulate_vnode "simulate_vnode.c") -target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a) \ No newline at end of file +target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a) +if(${BUILD_WINGETOPT}) + target_link_libraries(simulate_vnode PUBLIC wingetopt) + target_include_directories( + simulate_vnode + PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src" + ) +endif() \ No newline at end of file diff --git a/contrib/test/tdev/src/main.c b/contrib/test/tdev/src/main.c index 5e1de83e88..e40040ce97 100644 --- a/contrib/test/tdev/src/main.c +++ b/contrib/test/tdev/src/main.c @@ -6,43 +6,39 @@ #define POINTER_SHIFT(ptr, s) ((void *)(((char *)ptr) + (s))) #define POINTER_DISTANCE(pa, pb) ((char *)(pb) - (char *)(pa)) -#define tPutA(buf, val) \ - ({ \ - memcpy(buf, &val, sizeof(val)); \ - POINTER_SHIFT(buf, sizeof(val)); \ - }) +static inline void tPutA(void **buf, uint64_t val) { + memcpy(buf, &val, sizeof(val)); + *buf = POINTER_SHIFT(buf, sizeof(val)); +} -#define tPutB(buf, val) \ - ({ \ - ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; \ - ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; \ - ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; \ - ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; \ - ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; \ - ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; \ - ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff; \ - ((uint8_t *)buf)[0] = (val)&0xff; \ - POINTER_SHIFT(buf, sizeof(val)); \ - }) +static inline void tPutB(void **buf, uint64_t val) { + ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; + ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; + ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; + ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; + ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; + ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; + ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff; + ((uint8_t *)buf)[0] = (val)&0xff; + *buf = POINTER_SHIFT(buf, sizeof(val)); +} -#define tPutC(buf, val) \ - ({ \ - if (buf) { \ - ((uint64_t *)buf)[0] = (val); \ - POINTER_SHIFT(buf, sizeof(val)); \ - } \ - NULL; \ - }) +static inline void tPutC(void **buf, uint64_t val) { + if (buf) { + ((uint64_t *)buf)[0] = (val); + POINTER_SHIFT(buf, sizeof(val)); + } + *buf = NULL; +} -#define tPutD(buf, val) \ - ({ \ - uint64_t tmp = val; \ - for (size_t i = 0; i < sizeof(val); i++) { \ - ((uint8_t *)buf)[i] = tmp & 0xff; \ - tmp >>= 8; \ - } \ - POINTER_SHIFT(buf, sizeof(val)); \ - }) +static inline void tPutD(void **buf, uint64_t val) { + uint64_t tmp = val; + for (size_t i = 0; i < sizeof(val); i++) { + ((uint8_t *)buf)[i] = tmp & 0xff; + tmp >>= 8; + } + *buf = POINTER_SHIFT(buf, sizeof(val)); +} static inline void tPutE(void **buf, uint64_t val) { if (buf) { @@ -61,7 +57,7 @@ static void func(T t) { switch (t) { case A: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - pBuf = tPutA(pBuf, val); + tPutA(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } @@ -69,7 +65,7 @@ static void func(T t) { break; case B: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - pBuf = tPutB(pBuf, val); + tPutB(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } @@ -77,7 +73,7 @@ static void func(T t) { break; case C: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - pBuf = tPutC(pBuf, val); + tPutC(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } @@ -85,7 +81,7 @@ static void func(T t) { break; case D: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - pBuf = tPutD(pBuf, val); + tPutD(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index fe5f8cf7ed..80059dde4d 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -39,6 +39,7 @@ target_include_directories(tstream target_include_directories(demoapi PUBLIC "${TD_SOURCE_DIR}/include/client" + PUBLIC "${TD_SOURCE_DIR}/include/os" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) diff --git a/example/src/demoapi.c b/example/src/demoapi.c index afa9e19d4d..c3a6d26776 100644 --- a/example/src/demoapi.c +++ b/example/src/demoapi.c @@ -4,10 +4,12 @@ #include #include #include -#include +// #include #include +#ifndef WINDOWS #include - +#endif +#include "osSleep.h" #include "taos.h" #define debugPrint(fmt, ...) \ @@ -32,6 +34,7 @@ int64_t g_num_of_tb = 2; int64_t g_num_of_rec = 3; +#ifndef WINDOWS static struct argp_option options[] = { {"tables", 't', "NUMBER", 0, "Number of child tables, default is 10000."}, {"records", 'n', "NUMBER", 0, @@ -61,16 +64,16 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { } static struct argp argp = {options, parse_opt, "", ""}; - +#endif static void prepare_data(TAOS* taos) { TAOS_RES *res; res = taos_query(taos, "drop database if exists test;"); taos_free_result(res); - usleep(100000); + taosMsleep(100); res = taos_query(taos, "create database test;"); taos_free_result(res); - usleep(100000); + taosMsleep(100); taos_select_db(taos, "test"); char command[1024] = {0}; @@ -87,18 +90,17 @@ static void prepare_data(TAOS* taos) { taos_free_result(res); for (int64_t i = 0; i < g_num_of_tb; i ++) { + // sprintf(command, "create table t%"PRId64" using meters " + // "tags(%"PRId64", '%s', '%s', '%s');", + // i, i, (i%2)?"beijing":"shanghai", + // (i%2)?"朝阳区":"黄浦区", + // (i%2)?"长安街":"中山路"); sprintf(command, "create table t%"PRId64" using meters " - "tags(%"PRId64", '%s', '%s', '%s');", - i, i, (i%2)?"beijing":"shanghai", - (i%2)?"朝阳区":"黄浦区", - (i%2)?"长安街":"中山路"); -/* sprintf(command, "create table t%"PRId64" using meters " "tags(%"PRId64", '%s', '%s', '%s');", i, i, (i%2)?"beijing":"shanghai", (i%2)?"chaoyang":"huangpu", (i%2?"changan street":"jianguo rd")); - */ res = taos_query(taos, command); if ((res) && (0 == taos_errno(res))) { okPrint("t%" PRId64 " created\n", i); @@ -117,7 +119,8 @@ static void prepare_data(TAOS* taos) { "'%c%d', '%s%c%d', '%c%d')", i, 1650000000000+j, (float)j, j, 'a'+(int)j%25, rand(), - "涛思", 'z' - (int)j%25, rand(), + // "涛思", 'z' - (int)j%25, rand(), + "TAOS", 'z' - (int)j%25, rand(), 'b' - (int)j%25, rand() ); res = taos_query(taos, command); @@ -196,7 +199,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d, row: %"PRId64", " "value: %"PRId64"\n", f, c, - *(int64_t*)(row[f]+c*sizeof(int64_t))); + *(int64_t*)((char*)(row[f])+c*sizeof(int64_t))); } break; @@ -208,7 +211,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d, row: %"PRId64", " "value: %d\n", f, c, - *(int32_t*)(row[f]+c*sizeof(int32_t))); + *(int32_t*)((char*)(row[f])+c*sizeof(int32_t))); } break; @@ -220,7 +223,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d, row: %"PRId64", " "value: %f\n", f, c, - *(float*)(row[f]+c*sizeof(float))); + *(float*)((char*)(row[f])+c*sizeof(float))); } break; @@ -235,9 +238,9 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { if (offsets) { for (int c = 0; c < rows; c++) { if (offsets[c] != -1) { - int length = *(int16_t*)(row[f] + offsets[c]); + int length = *(int16_t*)((char*)(row[f]) + offsets[c]); char *buf = calloc(1, length + 1); - strncpy(buf, (char *)(row[f] + offsets[c] + 2), length); + strncpy(buf, (char *)((char*)(row[f]) + offsets[c] + 2), length); printf("row: %d, col: %d, offset: %d, length: %d, content: %s\n", c, f, offsets[c], length, buf); free(buf); @@ -314,8 +317,9 @@ int main(int argc, char *argv[]) { const char* host = "127.0.0.1"; const char* user = "root"; const char* passwd = "taosdata"; - +#ifndef WINDOWS argp_parse(&argp, argc, argv, 0, 0, NULL); +#endif TAOS* taos = taos_connect(host, user, passwd, "", 0); if (taos == NULL) { printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos)); diff --git a/include/common/trow.h b/include/common/trow.h index 7b0c009a11..bc5413f9ce 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -1359,7 +1359,7 @@ static void tdSRowPrint(STSRow *row, STSchema *pSchema) { printf(">>>"); for (int i = 0; i < pSchema->numOfCols; ++i) { STColumn *stCol = pSchema->columns + i; - SCellVal sVal = {.valType = 255, .val = NULL}; + SCellVal sVal = { 255, NULL}; if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) { break; } diff --git a/include/dnode/bnode/bnode.h b/include/dnode/bnode/bnode.h index 528cea8828..d5958d432b 100644 --- a/include/dnode/bnode/bnode.h +++ b/include/dnode/bnode/bnode.h @@ -26,6 +26,9 @@ extern "C" { typedef struct SBnode SBnode; typedef struct { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif } SBnodeLoad; typedef struct { diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index bb550e75e8..2b86b61cdd 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -51,6 +51,10 @@ typedef struct STableComInfo { typedef struct SIndexMeta { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif + } SIndexMeta; diff --git a/include/os/os.h b/include/os/os.h index 3ea94d0094..86abcc15f5 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -22,7 +22,6 @@ extern "C" { #include #include -#include #include @@ -51,6 +50,15 @@ extern "C" { #include #include #endif +#else + +#include +#ifndef TD_USE_WINSOCK +#include +#else +#include +#endif + #endif diff --git a/include/os/osAtomic.h b/include/os/osAtomic.h index 5d45ced3ed..e2a122a0fe 100644 --- a/include/os/osAtomic.h +++ b/include/os/osAtomic.h @@ -63,22 +63,22 @@ int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val); int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val); -void* atomic_add_fetch_ptr(void *ptr, int32_t val); +void* atomic_add_fetch_ptr(void *ptr, void *val); int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val); -void* atomic_fetch_add_ptr(void *ptr, int32_t val); +void* atomic_fetch_add_ptr(void *ptr, void *val); int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val); int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val); -void* atomic_sub_fetch_ptr(void *ptr, int32_t val); +void* atomic_sub_fetch_ptr(void *ptr, void *val); int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val); -void* atomic_fetch_sub_ptr(void *ptr, int32_t val); +void* atomic_fetch_sub_ptr(void *ptr, void *val); int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val); diff --git a/include/os/osDef.h b/include/os/osDef.h index 492df1047c..8da0c2c00c 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -191,7 +191,7 @@ extern "C" { #define threadlocal __declspec( thread ) #endif -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #define PRIzu "ld" #else #define PRIzu "zu" diff --git a/include/os/osDir.h b/include/os/osDir.h index e7da54bf54..b549acde37 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -36,12 +36,12 @@ typedef struct TdDirEntry *TdDirEntryPtr; void taosRemoveDir(const char *dirname); -bool taosDirExist(char *dirname); +bool taosDirExist(const char *dirname); int32_t taosMkDir(const char *dirname); int32_t taosMulMkDir(const char *dirname); void taosRemoveOldFiles(const char *dirname, int32_t keepDays); int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen); -int32_t taosRealPath(char *dirname, int32_t maxlen); +int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen); bool taosIsDir(const char *dirname); char* taosDirName(char *dirname); char* taosDirEntryBaseName(char *dirname); @@ -50,7 +50,7 @@ TdDirPtr taosOpenDir(const char *dirname); TdDirEntryPtr taosReadDir(TdDirPtr pDir); bool taosDirEntryIsDir(TdDirEntryPtr pDirEntry); char* taosGetDirEntryName(TdDirEntryPtr pDirEntry); -int32_t taosCloseDir(TdDirPtr pDir); +int32_t taosCloseDir(TdDirPtr *ppDir); #ifdef __cplusplus } diff --git a/include/os/osFile.h b/include/os/osFile.h index 36ca6fb8bb..b364d233ef 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -93,7 +93,6 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size); -void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length); bool taosValidFile(TdFilePtr pFile); int32_t taosGetErrorFile(TdFilePtr pFile); diff --git a/include/os/osLz4.h b/include/os/osLz4.h index 09aaa553ac..5bb33242d4 100644 --- a/include/os/osLz4.h +++ b/include/os/osLz4.h @@ -20,7 +20,7 @@ extern "C" { #endif -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS int32_t BUILDIN_CLZL(uint64_t val); int32_t BUILDIN_CLZ(uint32_t val); int32_t BUILDIN_CTZL(uint64_t val); diff --git a/include/os/osMath.h b/include/os/osMath.h index 0b5e4bd93c..cabb821844 100644 --- a/include/os/osMath.h +++ b/include/os/osMath.h @@ -23,7 +23,7 @@ extern "C" { #define TPOW2(x) ((x) * (x)) #define TABS(x) ((x) > 0 ? (x) : -(x)) -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #define TSWAP(a, b, c) \ do { \ @@ -33,6 +33,7 @@ extern "C" { } while (0) #define TMAX(a, b) (((a) > (b)) ? (a) : (b)) #define TMIN(a, b) (((a) < (b)) ? (a) : (b)) + #define TRANGE(aa, bb, cc) ((aa) = TMAX((aa), (bb)),(aa) = TMIN((aa), (cc))) #else diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 7af8dd37bf..a47239089a 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -19,32 +19,53 @@ // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC - #define socket SOCKET_FUNC_TAOS_FORBID - #define bind BIND_FUNC_TAOS_FORBID - #define listen LISTEN_FUNC_TAOS_FORBID - #define accept ACCEPT_FUNC_TAOS_FORBID - #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID - #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID - #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID - #define inet_addr INET_ADDR_FUNC_TAOS_FORBID - #define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID + #define socket SOCKET_FUNC_TAOS_FORBID + #define bind BIND_FUNC_TAOS_FORBID + #define listen LISTEN_FUNC_TAOS_FORBID + #define accept ACCEPT_FUNC_TAOS_FORBID + #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID + #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID + #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID + #define inet_addr INET_ADDR_FUNC_TAOS_FORBID + #define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID #endif #if defined(WINDOWS) - #include "winsock2.h" - #include - #include - #include -#else - #include - #include + #if BYTE_ORDER == LITTLE_ENDIAN + #include + #define htobe16(x) _byteswap_ushort(x) + #define htole16(x) (x) + #define be16toh(x) _byteswap_ushort(x) + #define le16toh(x) (x) + + #define htobe32(x) _byteswap_ulong(x) + #define htole32(x) (x) + #define be32toh(x) _byteswap_ulong(x) + #define le32toh(x) (x) + + #define htobe64(x) _byteswap_uint64(x) + #define htole64(x) (x) + #define be64toh(x) _byteswap_uint64(x) + #define le64toh(x) (x) + #else + #error byte order not supported + #endif - #if defined(_TD_DARWIN_64) - #include - #else - #include - #include - #endif + #define __BYTE_ORDER BYTE_ORDER + #define __BIG_ENDIAN BIG_ENDIAN + #define __LITTLE_ENDIAN LITTLE_ENDIAN + #define __PDP_ENDIAN PDP_ENDIAN + +#else + #include + #include + + #if defined(_TD_DARWIN_64) + #include + #else + #include + #include + #endif #endif #ifdef __cplusplus @@ -52,14 +73,10 @@ extern "C" { #endif #if defined(WINDOWS) - #define htobe64 htonll -#endif - -#if defined(WINDOWS) + typedef int socklen_t; #define TAOS_EPOLL_WAIT_TIME 100 typedef SOCKET eventfd_t; #define eventfd(a, b) -1 - typedef SOCKET EpollFd; #define EpollClose(pollFd) epoll_close(pollFd) #ifndef EPOLLWAKEUP #define EPOLLWAKEUP (1u << 29) @@ -79,31 +96,29 @@ extern "C" { #if defined(_TD_DARWIN_64) // #define htobe64 htonll -# include +#include -# define htobe16(x) OSSwapHostToBigInt16(x) -# define htole16(x) OSSwapHostToLittleInt16(x) -# define be16toh(x) OSSwapBigToHostInt16(x) -# define le16toh(x) OSSwapLittleToHostInt16(x) +#define htobe16(x) OSSwapHostToBigInt16(x) +#define htole16(x) OSSwapHostToLittleInt16(x) +#define be16toh(x) OSSwapBigToHostInt16(x) +#define le16toh(x) OSSwapLittleToHostInt16(x) -# define htobe32(x) OSSwapHostToBigInt32(x) -# define htole32(x) OSSwapHostToLittleInt32(x) -# define be32toh(x) OSSwapBigToHostInt32(x) -# define le32toh(x) OSSwapLittleToHostInt32(x) +#define htobe32(x) OSSwapHostToBigInt32(x) +#define htole32(x) OSSwapHostToLittleInt32(x) +#define be32toh(x) OSSwapBigToHostInt32(x) +#define le32toh(x) OSSwapLittleToHostInt32(x) -# define htobe64(x) OSSwapHostToBigInt64(x) -# define htole64(x) OSSwapHostToLittleInt64(x) -# define be64toh(x) OSSwapBigToHostInt64(x) -# define le64toh(x) OSSwapLittleToHostInt64(x) +#define htobe64(x) OSSwapHostToBigInt64(x) +#define htole64(x) OSSwapHostToLittleInt64(x) +#define be64toh(x) OSSwapBigToHostInt64(x) +#define le64toh(x) OSSwapLittleToHostInt64(x) -# define __BYTE_ORDER BYTE_ORDER -# define __BIG_ENDIAN BIG_ENDIAN -# define __LITTLE_ENDIAN LITTLE_ENDIAN -# define __PDP_ENDIAN PDP_ENDIAN +#define __BYTE_ORDER BYTE_ORDER +#define __BIG_ENDIAN BIG_ENDIAN +#define __LITTLE_ENDIAN LITTLE_ENDIAN +#define __PDP_ENDIAN PDP_ENDIAN #endif -#define TAOS_EPOLL_WAIT_TIME 500 - typedef int32_t SocketFd; typedef SocketFd EpollFd; @@ -122,7 +137,7 @@ typedef struct TdEpoll *TdEpollPtr; int32_t taosSendto(TdSocketPtr pSocket, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen); int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len); int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len); -int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen); +int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, int *addrLen); int32_t taosCloseSocketNoCheck1(SocketFd fd); int32_t taosCloseSocket(TdSocketPtr *ppSocket); int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer); @@ -139,14 +154,15 @@ int32_t taosWriteMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes); int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes); int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes); int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len); +void taosWinSocketInit(); TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port); int32_t taosKeepTcpAlive(TdSocketPtr pSocket); -TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, socklen_t *addrLen); +TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen); -int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, socklen_t *addrLen); +int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, int *addrLen); void taosBlockSIGPIPE(); uint32_t taosGetIpv4FromFqdn(const char *); diff --git a/include/os/osString.h b/include/os/osString.h index 66d69a849c..a4100652c3 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -39,7 +39,7 @@ typedef int32_t TdUcs4; #define wchar_t WCHAR_T_TYPE_TAOS_FORBID #endif -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #define tstrdup(str) _strdup(str) #else #define tstrdup(str) strdup(str) diff --git a/include/os/osThread.h b/include/os/osThread.h index 18c1b290ab..6252a0cb60 100644 --- a/include/os/osThread.h +++ b/include/os/osThread.h @@ -22,9 +22,12 @@ extern "C" { #endif +#ifndef WINDOWS #ifndef __USE_XOPEN2K +#define TD_USE_SPINLOCK_AS_MUTEX typedef pthread_mutex_t pthread_spinlock_t; #endif +#endif typedef pthread_t TdThread; typedef pthread_spinlock_t TdThreadSpinlock; @@ -41,6 +44,13 @@ typedef pthread_key_t TdThreadKey; #define taosThreadCleanupPush pthread_cleanup_push #define taosThreadCleanupPop pthread_cleanup_pop + +#ifdef WINDOWS +#define TD_PTHREAD_MUTEX_INITIALIZER (TdThreadMutex)(-1) +#else +#define TD_PTHREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER +#endif + // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC @@ -218,8 +228,7 @@ int32_t taosThreadSpinLock(TdThreadSpinlock * lock); int32_t taosThreadSpinTrylock(TdThreadSpinlock * lock); int32_t taosThreadSpinUnlock(TdThreadSpinlock * lock); void taosThreadTestCancel(void); -int32_t taosThreadSigMask(int32_t how, sigset_t const *set, sigset_t * oset); -int32_t taosThreadSigWait(const sigset_t * set, int32_t *sig); +void taosThreadClear(TdThread *thread); #ifdef __cplusplus } diff --git a/include/os/osTime.h b/include/os/osTime.h index fd431f6df8..b9e407cbcf 100644 --- a/include/os/osTime.h +++ b/include/os/osTime.h @@ -20,8 +20,6 @@ extern "C" { #endif -#include - // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC @@ -34,7 +32,7 @@ extern "C" { #define mktime MKTIME_FUNC_TAOS_FORBID #endif -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #define CLOCK_REALTIME 0 @@ -58,6 +56,8 @@ extern "C" { int32_t taosGetTimeOfDay(struct timeval *tv); +int32_t taosClockGetTime(int clock_id, struct timespec *pTS); + //@return timestamp in second int32_t taosGetTimestampSec(); @@ -78,7 +78,7 @@ static FORCE_INLINE int64_t taosGetTimestampUs() { //@return timestamp in nanosecond static FORCE_INLINE int64_t taosGetTimestampNs() { struct timespec systemTime = {0}; - clock_gettime(CLOCK_REALTIME, &systemTime); + taosClockGetTime(CLOCK_REALTIME, &systemTime); return (int64_t)systemTime.tv_sec * 1000000000L + (int64_t)systemTime.tv_nsec; } diff --git a/include/util/tencode.h b/include/util/tencode.h index 7c877ae428..11357ba533 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -236,7 +236,8 @@ static FORCE_INLINE int32_t tEncodeFloat(SCoder* pEncoder, float val) { union { uint32_t ui; float f; - } v = {.f = val}; + } v; + v.f = val; return tEncodeU32(pEncoder, v.ui); } @@ -245,7 +246,8 @@ static FORCE_INLINE int32_t tEncodeDouble(SCoder* pEncoder, double val) { union { uint64_t ui; double d; - } v = {.d = val}; + } v; + v.d = val; return tEncodeU64(pEncoder, v.ui); } diff --git a/include/util/tjson.h b/include/util/tjson.h index 28bcb3cfc6..286a7e65fc 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -22,6 +22,9 @@ extern "C" { #endif +#ifdef WINDOWS +#define tjsonGetNumberValue(pJson, pName, val) -1 +#else #define tjsonGetNumberValue(pJson, pName, val) \ ({ \ uint64_t _tmp = 0; \ @@ -29,6 +32,7 @@ extern "C" { val = _tmp; \ _code; \ }) +#endif typedef void SJson; diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index 63a04a8813..e01b113a04 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -5,8 +5,8 @@ target_include_directories( PUBLIC "${TD_SOURCE_DIR}/include/common" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" IF(${TD_WINDOWS}) - PRIVATE "${TD_SOURCE_DIR}/contrib/pthread-win32" - PRIVATE "${TD_SOURCE_DIR}/contrib/gnuregex" + PRIVATE "${TD_SOURCE_DIR}/contrib/pthread" + PRIVATE "${TD_SOURCE_DIR}/contrib/msvcregex" ENDIF () ) target_link_libraries( diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index d60261dfca..fea6c4c891 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -167,7 +167,7 @@ TEST(testCase, Datablock_test) { printf("the second row of binary:%s, length:%d\n", (char*)varDataVal(pData), varDataLen(pData)); SArray* pOrderInfo = taosArrayInit(3, sizeof(SBlockOrderInfo)); - SBlockOrderInfo order = {.nullFirst = true, .order = TSDB_ORDER_ASC, .slotId = 0}; + SBlockOrderInfo order = { true, TSDB_ORDER_ASC, 0, NULL }; taosArrayPush(pOrderInfo, &order); blockDataSort(b, pOrderInfo); diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 3f5c22dd84..7e4505ec95 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -40,10 +40,12 @@ static void dmSetSignalHandle() { taosSetSignal(SIGTERM, dmStopDnode); taosSetSignal(SIGHUP, dmStopDnode); taosSetSignal(SIGINT, dmStopDnode); - taosSetSignal(SIGTSTP, dmStopDnode); taosSetSignal(SIGABRT, dmStopDnode); taosSetSignal(SIGBREAK, dmStopDnode); +#ifndef WINDOWS + taosSetSignal(SIGTSTP, dmStopDnode); taosSetSignal(SIGQUIT, dmStopDnode); +#endif if (!tsMultiProcess) { } else if (global.ntype == DNODE || global.ntype == NODE_END) { diff --git a/source/dnode/vnode/src/meta/metaIdx.c b/source/dnode/vnode/src/meta/metaIdx.c index 9a566f788c..36057ae806 100644 --- a/source/dnode/vnode/src/meta/metaIdx.c +++ b/source/dnode/vnode/src/meta/metaIdx.c @@ -23,6 +23,9 @@ struct SMetaIdx { SIndex *pIdx; #endif /* data */ +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif }; int metaOpenIdx(SMeta *pMeta) { diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 149a7ca3cb..8389286596 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -45,10 +45,12 @@ struct SMetaDB { #endif }; -typedef struct __attribute__((__packed__)) { +#pragma pack(push,1) +typedef struct { tb_uid_t uid; int32_t sver; } SSchemaDbKey; +#pragma pack(pop) typedef struct { char *name; diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index 809fd7cb06..72469b74ac 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -86,7 +86,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F } strcpy(pMeta->dirPath, path); - char name[pathLen + 10]; + char *name = taosMemoryMalloc(pathLen + 10) ; strcpy(name, path); if (!taosDirExist(name) && taosMkDir(name) != 0) { @@ -99,6 +99,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F terrno = TAOS_SYSTEM_ERROR(errno); tqError("failed to open file:%s since %s ", name, terrstr()); // free memory + taosMemoryFree(name); return NULL; } @@ -106,6 +107,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F pMeta->unpersistHead = taosMemoryCalloc(1, sizeof(STqMetaList)); if (pMeta->unpersistHead == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + taosMemoryFree(name); return NULL; } pMeta->unpersistHead->unpersistNext = pMeta->unpersistHead->unpersistPrev = pMeta->unpersistHead; @@ -116,8 +118,10 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); tqError("failed to open file:%s since %s", name, terrstr()); + taosMemoryFree(name); return NULL; } + taosMemoryFree(name); pMeta->pFile = pFile; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 8938084724..918892b786 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -104,6 +104,10 @@ typedef struct SCatalog { typedef struct SCtgApiStat { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif + } SCtgApiStat; typedef struct SCtgRuntimeStat { diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 9a6356acbd..1045acbe93 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -24,6 +24,9 @@ #pragma GCC diagnostic ignored "-Wformat" #include +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif #include "os.h" #include "tglobal.h" #include "catalog.h" @@ -155,11 +158,11 @@ int32_t ctgTestGetVgNumFromVgVersion(int32_t vgVersion) { } void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { - SName cn = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName cn = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(cn.dbname, "db1"); strcpy(cn.tname, ctgTestCTablename); - SName sn = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName sn = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(sn.dbname, "db1"); strcpy(sn.tname, ctgTestSTablename); @@ -577,9 +580,16 @@ void ctgTestSetRspDbVgroups() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspDbVgroups); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspDbVgroups); } @@ -590,9 +600,16 @@ void ctgTestSetRspTableMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspTableMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspTableMeta); } @@ -603,9 +620,16 @@ void ctgTestSetRspCTableMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspCTableMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspCTableMeta); } @@ -616,9 +640,16 @@ void ctgTestSetRspSTableMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspSTableMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspSTableMeta); } @@ -629,9 +660,16 @@ void ctgTestSetRspMultiSTableMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspMultiSTableMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspMultiSTableMeta); } @@ -642,9 +680,16 @@ void ctgTestSetRspByIdx() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspByIdx); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspByIdx); } @@ -656,9 +701,16 @@ void ctgTestSetRspDbVgroupsAndNormalMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndNormalMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspDbVgroupsAndNormalMeta); } @@ -669,9 +721,16 @@ void ctgTestSetRspDbVgroupsAndChildMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndChildMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspDbVgroupsAndChildMeta); } @@ -682,9 +741,16 @@ void ctgTestSetRspDbVgroupsAndSuperMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndSuperMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspDbVgroupsAndSuperMeta); } @@ -695,9 +761,16 @@ void ctgTestSetRspDbVgroupsAndMultiSuperMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndMultiSuperMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspDbVgroupsAndMultiSuperMeta); } @@ -789,7 +862,7 @@ void *ctgTestGetCtableMetaThread(void *param) { STableMeta *tbMeta = NULL; bool inCache = false; - SName cn = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName cn = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(cn.dbname, "db1"); strcpy(cn.tname, ctgTestCTablename); @@ -871,7 +944,7 @@ TEST(tableMeta, normalTable) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -978,7 +1051,7 @@ TEST(tableMeta, childTableCase) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestCTablename); @@ -1085,7 +1158,7 @@ TEST(tableMeta, superTableCase) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestSTablename); @@ -1210,7 +1283,7 @@ TEST(tableMeta, rmStbMeta) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestSTablename); @@ -1280,7 +1353,7 @@ TEST(tableMeta, updateStbMeta) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestSTablename); @@ -1378,7 +1451,7 @@ TEST(refreshGetMeta, normal2normal) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -1457,7 +1530,7 @@ TEST(refreshGetMeta, normal2notexist) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -1529,7 +1602,7 @@ TEST(refreshGetMeta, normal2child) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); ctgTestCurrentCTableName = ctgTestTablename; @@ -1613,7 +1686,7 @@ TEST(refreshGetMeta, stable2child) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); ctgTestCurrentSTableName = ctgTestTablename; @@ -1699,7 +1772,7 @@ TEST(refreshGetMeta, stable2stable) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); ctgTestCurrentSTableName = ctgTestTablename; @@ -1786,7 +1859,7 @@ TEST(refreshGetMeta, child2stable) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); ctgTestCurrentCTableName = ctgTestTablename; @@ -1871,7 +1944,7 @@ TEST(tableDistVgroup, normalTable) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -1913,7 +1986,7 @@ TEST(tableDistVgroup, childTableCase) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestCTablename); @@ -1956,7 +2029,7 @@ TEST(tableDistVgroup, superTableCase) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestSTablename); @@ -2006,7 +2079,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -2091,7 +2164,7 @@ TEST(multiThread, getSetRmSameDbVgroup) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -2143,7 +2216,7 @@ TEST(multiThread, getSetRmDiffDbVgroup) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -2195,7 +2268,7 @@ TEST(multiThread, ctableMeta) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -2247,7 +2320,7 @@ TEST(rentTest, allRent) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); for (int32_t i = 1; i <= 10; ++i) { diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 9f75f97632..e51284bfef 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -410,8 +410,8 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { int32_t contentLen = 0; if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - int32_t* offset = pPage + startOffset; - columnLen = pPage + startOffset + sizeof(int32_t) * pInfo->rowCapacity; + int32_t* offset = (int32_t*)((char*)pPage + startOffset); + columnLen = (char*)pPage + startOffset + sizeof(int32_t) * pInfo->rowCapacity; char* data = (char*)(columnLen + sizeof(int32_t)); if (colDataIsNull_s(pColInfoData, j)) { @@ -424,8 +424,8 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { contentLen = varDataTLen(src); } } else { - char* bitmap = pPage + startOffset; - columnLen = pPage + startOffset + BitmapLen(pInfo->rowCapacity); + char* bitmap = (char*)pPage + startOffset; + columnLen = (char*)pPage + startOffset + BitmapLen(pInfo->rowCapacity); char* data = (char*) columnLen + sizeof(int32_t); bool isNull = colDataIsNull_f(pColInfoData->nullbitmap, j); diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index fe370d93b2..8f7ff04d92 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -48,7 +48,7 @@ struct SSHashObj { }; static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { - int32_t len = MIN(length, HASH_MAX_CAPACITY); + int32_t len = (length < HASH_MAX_CAPACITY ? length : HASH_MAX_CAPACITY); int32_t i = 4; while (i < len) i = (i << 1u); @@ -127,7 +127,7 @@ static void taosHashTableResize(SSHashObj *pHashObj) { } size_t inc = newCapacity - pHashObj->capacity; - memset(pNewEntryList + pHashObj->capacity * sizeof(void*), 0, inc); + memset((char*)pNewEntryList + pHashObj->capacity * sizeof(void*), 0, inc); pHashObj->hashList = pNewEntryList; pHashObj->capacity = newCapacity; @@ -301,7 +301,7 @@ size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj) { void *tSimpleHashGetKey(const SSHashObj* pHashObj, void *data, size_t* keyLen) { int32_t offset = offsetof(SHNode, data); - SHNode *node = data - offset; + SHNode *node = ((SHNode*)(char*)data - offset); if (keyLen != NULL) { *keyLen = pHashObj->keyLen; } diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 870a0b71fc..543df30686 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -102,7 +102,7 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { } else if (pInfo->type == data_asc) { v = ++pInfo->startVal; } else if (pInfo->type == data_rand) { - v = random(); + v = taosRand(); } colDataAppend(pColInfo, i, reinterpret_cast(&v), false); @@ -172,7 +172,7 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { } else if (pInfo->type == data_asc) { v = ++pInfo->startVal; } else if (pInfo->type == data_rand) { - v = random(); + v = taosRand(); } colDataAppend(pColInfo1, i, reinterpret_cast(&v), false); @@ -938,7 +938,7 @@ TEST(testCase, build_executor_tree_Test) { SExecTaskInfo* pTaskInfo = nullptr; DataSinkHandle sinkHandle = nullptr; - SReadHandle handle = {.reader = reinterpret_cast(0x1), .meta = reinterpret_cast(0x1)}; + SReadHandle handle = { reinterpret_cast(0x1), reinterpret_cast(0x1), NULL }; struct SSubplan *plan = NULL; int32_t code = qStringToSubplan(msg, &plan); diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index eddc094285..98f1209ad9 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -47,6 +47,9 @@ target_include_directories( "${TD_SOURCE_DIR}/include/os" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) +target_link_libraries( + udf1 PUBLIC os +) #SET(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/build/bin) add_executable(udfd src/udfd.c) diff --git a/source/libs/function/inc/tudfInt.h b/source/libs/function/inc/tudfInt.h index 496f486e05..2c16afbd0d 100644 --- a/source/libs/function/inc/tudfInt.h +++ b/source/libs/function/inc/tudfInt.h @@ -69,6 +69,9 @@ typedef struct SUdfTeardownRequest { typedef struct SUdfTeardownResponse { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif } SUdfTeardownResponse; typedef struct SUdfRequest { diff --git a/source/libs/function/inc/udfc.h b/source/libs/function/inc/udfc.h index 1cf8c67686..a693e476e8 100644 --- a/source/libs/function/inc/udfc.h +++ b/source/libs/function/inc/udfc.h @@ -84,6 +84,9 @@ typedef struct SUdfTeardownRequest { typedef struct SUdfTeardownResponse { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif } SUdfTeardownResponse; typedef struct SUdfRequest { diff --git a/source/libs/index/inc/indexFstRegex.h b/source/libs/index/inc/indexFstRegex.h index 8fb5455336..2bf9c9b791 100644 --- a/source/libs/index/inc/indexFstRegex.h +++ b/source/libs/index/inc/indexFstRegex.h @@ -31,6 +31,9 @@ extern "C" { typedef enum { MATCH, JUMP, SPLIT, RANGE } InstType; typedef struct MatchValue { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif } MatchValue; typedef struct JumpValue { uint32_t step; diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 058c33b725..554b8092a2 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -748,7 +748,7 @@ static SArray* tfileGetFileList(const char* path) { sprintf(buf, "%s/%s", path, file); taosArrayPush(files, &buf); } - taosCloseDir(pDir); + taosCloseDir(&pDir); taosArraySort(files, tfileCompare); tfileRmExpireFile(files); diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index ee9da50ab2..0162b754ee 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -485,7 +485,7 @@ TEST_F(IndexTFileEnv, test_tfile_write) { std::string colVal("ab"); SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; + SIndexTermQuery query = { term, QUERY_TERM}; SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); fObj->Get(&query, result); @@ -625,7 +625,7 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colVal("v3"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; + SIndexTermQuery query = { term, QUERY_TERM }; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); STermValueType valType; @@ -640,7 +640,7 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colVal("v2"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; + SIndexTermQuery query = { term, QUERY_TERM }; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); STermValueType valType; diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index 9b0452425a..f789d23136 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -108,7 +108,7 @@ TEST_F(JsonEnv, testWriteMillonData) { { std::string colName("voltagefdadfa"); std::string colVal("abxxxxxxxxxxxx"); - for (uint i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { colVal[i % colVal.size()] = '0' + i % 128; SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); diff --git a/source/libs/parser/inc/parToken.h b/source/libs/parser/inc/parToken.h index 1122bafd12..d8a2c22fd8 100644 --- a/source/libs/parser/inc/parToken.h +++ b/source/libs/parser/inc/parToken.h @@ -20,7 +20,6 @@ extern "C" { #endif -#include "os.h" #include "ttokendef.h" // used to denote the minimum unite in sql parsing diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 2eeff04ea2..fcc15b197f 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "os.h" #include "parInt.h" #include "parAst.h" diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index f21a738032..045f54cdb1 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "os.h" #include "parInsertData.h" #include "parInt.h" #include "parToken.h" diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 8506cfc33d..40860aaaf8 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -49,12 +49,14 @@ static bool afterGroupBy(ESqlClause clause) { return clause > SQL_CLAUSE_GROUP_B static bool beforeHaving(ESqlClause clause) { return clause < SQL_CLAUSE_HAVING; } -#define generateDealNodeErrMsg(pCxt, code, ...) \ - ({ \ - generateSyntaxErrMsg(&pCxt->msgBuf, code, ##__VA_ARGS__); \ - pCxt->errCode = code; \ - DEAL_RES_ERROR; \ - }) +enum EDealRes generateDealNodeErrMsg(STranslateContext* pCxt, int32_t code, ...) { + va_list ap; + va_start(ap, code); + generateSyntaxErrMsg(&pCxt->msgBuf, code, ap); + va_end(ap); + pCxt->errCode = code; + return DEAL_RES_ERROR; +} static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { size_t currTotalLevel = taosArrayGetSize(pCxt->pNsLevel); diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index e2e2207c70..f17322063a 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "os.h" #include "parser.h" #include "parInt.h" diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index b38e4092e3..bf23058f7c 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -32,6 +32,7 @@ #include #include +#include "os.h" #include "functionMgt.h" #include "nodes.h" #include "parToken.h" diff --git a/source/libs/parser/test/CMakeLists.txt b/source/libs/parser/test/CMakeLists.txt index 869232002a..96de3fb735 100644 --- a/source/libs/parser/test/CMakeLists.txt +++ b/source/libs/parser/test/CMakeLists.txt @@ -17,3 +17,11 @@ TARGET_LINK_LIBRARIES( parserTest PUBLIC os util common nodes parser catalog transport gtest function planner qcom ) + +if(${BUILD_WINGETOPT}) + target_include_directories( + parserTest + PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src" + ) + target_link_libraries(parserTest PUBLIC wingetopt) +endif() \ No newline at end of file diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 5723b93f8b..0f3e2fe414 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -12,7 +12,6 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - #include #include "stub.h" @@ -23,6 +22,9 @@ #pragma GCC diagnostic pop +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif #include "mockCatalog.h" namespace { diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 3ef0eaed42..d36448b686 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -40,13 +40,11 @@ public: virtual TableBuilder& setVgid(int16_t vgid) { schema()->vgId = vgid; - SVgroupInfo vgroup = {.vgId = vgid, .hashBegin = 0, .hashEnd = 0, }; - - vgroup.epSet.eps[0] = (SEp){"dnode_1", 6030}; - vgroup.epSet.eps[1] = (SEp){"dnode_2", 6030}; - vgroup.epSet.eps[2] = (SEp){"dnode_3", 6030}; + SVgroupInfo vgroup = { vgid, 0, 0, {0}, 0}; + addEpIntoEpSet(&vgroup.epSet, "dnode_1", 6030); + addEpIntoEpSet(&vgroup.epSet, "dnode_2", 6030); + addEpIntoEpSet(&vgroup.epSet, "dnode_3", 6030); vgroup.epSet.inUse = 0; - vgroup.epSet.numOfEps = 3; meta_->vgs.emplace_back(vgroup); return *this; @@ -148,7 +146,7 @@ public: meta_[db][tbname]->schema->uid = id_++; meta_[db][tbname]->schema->tableType = TSDB_CHILD_TABLE; - SVgroupInfo vgroup = {.vgId = vgid, .hashBegin = 0, .hashEnd = 0,}; + SVgroupInfo vgroup = { vgid, 0, 0, {0}, 0}; addEpIntoEpSet(&vgroup.epSet, "dnode_1", 6030); addEpIntoEpSet(&vgroup.epSet, "dnode_2", 6030); addEpIntoEpSet(&vgroup.epSet, "dnode_3", 6030); diff --git a/source/libs/parser/test/parserInsertTest.cpp b/source/libs/parser/test/parserInsertTest.cpp index d292fcf8b0..4aec141b55 100644 --- a/source/libs/parser/test/parserInsertTest.cpp +++ b/source/libs/parser/test/parserInsertTest.cpp @@ -15,6 +15,7 @@ #include +#include "os.h" #include "parInt.h" using namespace std; diff --git a/source/libs/parser/test/parserTestMain.cpp b/source/libs/parser/test/parserTestMain.cpp index 8cd69418b9..95faadfc34 100644 --- a/source/libs/parser/test/parserTestMain.cpp +++ b/source/libs/parser/test/parserTestMain.cpp @@ -14,14 +14,20 @@ */ #include +#include +#include #include #include -#include "mockCatalog.h" +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif +#include "os.h" #include "parserTestUtil.h" #include "parToken.h" #include "functionMgt.h" +#include "mockCatalog.h" bool g_isDump = false; diff --git a/source/libs/planner/test/CMakeLists.txt b/source/libs/planner/test/CMakeLists.txt index e068cd8698..f0c59b969f 100644 --- a/source/libs/planner/test/CMakeLists.txt +++ b/source/libs/planner/test/CMakeLists.txt @@ -22,3 +22,12 @@ TARGET_INCLUDE_DIRECTORIES( PRIVATE "${TD_SOURCE_DIR}/source/libs/planner/inc" PRIVATE "${TD_SOURCE_DIR}/source/libs/parser/test" ) + + +if(${BUILD_WINGETOPT}) + target_include_directories( + plannerTest + PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src" + ) + target_link_libraries(plannerTest PUBLIC wingetopt) +endif() \ No newline at end of file diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index 2878dc7954..84cf75c5be 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -16,7 +16,7 @@ #include #include - +#include "getopt.h" #include "mockCatalog.h" #include "planTestUtil.h" diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index 222ad9780b..f124970875 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include #include "planTestUtil.h" #include diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index d62f9f04b8..cc82349fee 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -95,6 +95,9 @@ typedef struct SQWPhaseInput { } SQWPhaseInput; typedef struct SQWPhaseOutput { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif } SQWPhaseOutput; diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 7179fa6984..5a57a47df8 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -27,6 +27,9 @@ #pragma GCC diagnostic ignored "-Wpointer-arith" #include +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif #include "os.h" #include "tglobal.h" @@ -277,8 +280,8 @@ int32_t qwtCreateExecTask(void* tsdb, int32_t vgId, uint64_t taskId, struct SSub qwtTestSinkMaxBlockNum = taosRand() % 100 + 1; qwtTestSinkQueryEnd = false; - *pTaskInfo = (qTaskInfo_t)qwtTestCaseIdx+1; - *handle = (DataSinkHandle)qwtTestCaseIdx+2; + *pTaskInfo = (qTaskInfo_t)((char*)qwtTestCaseIdx+1); + *handle = (DataSinkHandle)((char*)qwtTestCaseIdx+2); ++qwtTestCaseIdx; @@ -435,9 +438,16 @@ void stubSetStringToPlan() { static Stub stub; stub.set(qStringToSubplan, qwtStringToPlan); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("qStringToSubplan", result); +#endif +#ifdef LINUX AddrAny any("libplanner.so"); std::map result; any.get_global_func_addr_dynsym("^qStringToSubplan$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtStringToPlan); } @@ -448,9 +458,16 @@ void stubSetExecTask() { static Stub stub; stub.set(qExecTask, qwtExecTask); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("qExecTask", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^qExecTask$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtExecTask); } @@ -463,9 +480,16 @@ void stubSetCreateExecTask() { static Stub stub; stub.set(qCreateExecTask, qwtCreateExecTask); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("qCreateExecTask", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^qCreateExecTask$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtCreateExecTask); } @@ -476,9 +500,16 @@ void stubSetAsyncKillTask() { static Stub stub; stub.set(qAsyncKillTask, qwtKillTask); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("qAsyncKillTask", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^qAsyncKillTask$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtKillTask); } @@ -489,9 +520,16 @@ void stubSetDestroyTask() { static Stub stub; stub.set(qDestroyTask, qwtDestroyTask); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("qDestroyTask", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^qDestroyTask$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtDestroyTask); } @@ -503,9 +541,16 @@ void stubSetDestroyDataSinker() { static Stub stub; stub.set(dsDestroyDataSinker, qwtDestroyDataSinker); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("dsDestroyDataSinker", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^dsDestroyDataSinker$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtDestroyDataSinker); } @@ -516,9 +561,16 @@ void stubSetGetDataLength() { static Stub stub; stub.set(dsGetDataLength, qwtGetDataLength); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("dsGetDataLength", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^dsGetDataLength$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtGetDataLength); } @@ -529,9 +581,16 @@ void stubSetEndPut() { static Stub stub; stub.set(dsEndPut, qwtEndPut); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("dsEndPut", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^dsEndPut$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtEndPut); } @@ -542,9 +601,16 @@ void stubSetPutDataBlock() { static Stub stub; stub.set(dsPutDataBlock, qwtPutDataBlock); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("dsPutDataBlock", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^dsPutDataBlock$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtPutDataBlock); } @@ -555,9 +621,16 @@ void stubSetRpcSendResponse() { static Stub stub; stub.set(rpcSendResponse, qwtRpcSendResponse); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendResponse", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendResponse$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtRpcSendResponse); } @@ -568,9 +641,16 @@ void stubSetGetDataBlock() { static Stub stub; stub.set(dsGetDataBlock, qwtGetDataBlock); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("dsGetDataBlock", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^dsGetDataBlock$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtGetDataBlock); } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 0559792740..53e108bb17 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -621,13 +621,13 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu if (subPos > 0) { startPosBytes = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? subPos - 1 : (subPos - 1) * TSDB_NCHAR_SIZE; - startPosBytes = MIN(startPosBytes, len); + startPosBytes = TMIN(startPosBytes, len); } else { startPosBytes = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? len + subPos : len + subPos * TSDB_NCHAR_SIZE; - startPosBytes = MAX(startPosBytes, 0); + startPosBytes = TMAX(startPosBytes, 0); } - int32_t resLen = MIN(subLen, len - startPosBytes); + int32_t resLen = TMIN(subLen, len - startPosBytes); if (resLen > 0) { memcpy(varDataVal(output), varDataVal(input) + startPosBytes, resLen); } @@ -716,7 +716,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int32_t len = sprintf(varDataVal(output), "%.*s", (int32_t)(outputLen - VARSTR_HEADER_SIZE), *(int8_t *)input ? "true" : "false"); varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_BINARY) { - int32_t len = MIN(varDataLen(input), outputLen - VARSTR_HEADER_SIZE); + int32_t len = TMIN(varDataLen(input), outputLen - VARSTR_HEADER_SIZE); len = sprintf(varDataVal(output), "%.*s", len, varDataVal(input)); varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { @@ -750,7 +750,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = MIN(outputLen, varDataLen(input) + VARSTR_HEADER_SIZE); + int32_t len = TMIN(outputLen, varDataLen(input) + VARSTR_HEADER_SIZE); memcpy(output, input, len); varDataSetLen(output, len - VARSTR_HEADER_SIZE); } else { diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 42998aba00..c5191947e0 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -27,6 +27,9 @@ #pragma GCC diagnostic ignored "-Wpointer-arith" #include +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif #include "os.h" #include "tglobal.h" @@ -364,7 +367,7 @@ TEST(columnTest, smallint_column_greater_double_value) { keep = filterRangeExecute(filter, &stat, 1, rowNum); ASSERT_EQ(keep, true); - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -419,7 +422,7 @@ TEST(columnTest, int_column_greater_smallint_value) { keep = filterRangeExecute(filter, &stat, 1, rowNum); ASSERT_EQ(keep, false); - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -463,7 +466,7 @@ TEST(columnTest, int_column_in_double_list) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -527,7 +530,7 @@ TEST(columnTest, binary_column_in_binary_list) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -575,7 +578,7 @@ TEST(columnTest, binary_column_like_binary) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -622,7 +625,7 @@ TEST(columnTest, binary_column_is_null) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -669,7 +672,7 @@ TEST(columnTest, binary_column_is_not_null) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -706,7 +709,7 @@ TEST(opTest, smallint_column_greater_int_column) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -744,7 +747,7 @@ TEST(opTest, smallint_value_add_int_column) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -787,7 +790,7 @@ TEST(opTest, bigint_column_multi_binary_column) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -830,7 +833,7 @@ TEST(opTest, smallint_column_and_binary_column) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -868,7 +871,7 @@ TEST(opTest, smallint_column_or_float_column) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -906,7 +909,7 @@ TEST(opTest, smallint_column_or_double_value) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -950,7 +953,7 @@ TEST(opTest, binary_column_is_true) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1021,7 +1024,7 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1090,7 +1093,7 @@ TEST(filterModelogicTest, same_column_and_or_and) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1159,7 +1162,7 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1228,7 +1231,7 @@ TEST(filterModelogicTest, same_column_or_and_or) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1299,7 +1302,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index de76e7e274..f6140a75d3 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -27,6 +27,9 @@ #pragma GCC diagnostic ignored "-Wpointer-arith" #include +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif #include "os.h" #include "tglobal.h" diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index d1def1bef1..0f6961018c 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -52,14 +52,26 @@ typedef struct SSchHbTrans { typedef struct SSchApiStat { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif + } SSchApiStat; typedef struct SSchRuntimeStat { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif + } SSchRuntimeStat; typedef struct SSchJobStat { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif + } SSchJobStat; typedef struct SSchedulerStat { diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 5d9322a575..fc0e05aaf1 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -25,7 +25,9 @@ #pragma GCC diagnostic ignored "-Wformat" #include - +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif #include "os.h" #include "tglobal.h" @@ -275,9 +277,16 @@ void schtSetPlanToString() { static Stub stub; stub.set(qSubPlanToString, schtPlanToString); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("qSubPlanToString", result); +#endif +#ifdef LINUX AddrAny any("libplanner.so"); std::map result; any.get_global_func_addr_dynsym("^qSubPlanToString$", result); +#endif for (const auto& f : result) { stub.set(f.second, schtPlanToString); } @@ -288,9 +297,16 @@ void schtSetExecNode() { static Stub stub; stub.set(qSetSubplanExecutionNode, schtExecNode); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("qSetSubplanExecutionNode", result); +#endif +#ifdef LINUX AddrAny any("libplanner.so"); std::map result; any.get_global_func_addr_dynsym("^qSetSubplanExecutionNode$", result); +#endif for (const auto& f : result) { stub.set(f.second, schtExecNode); } @@ -301,9 +317,16 @@ void schtSetRpcSendRequest() { static Stub stub; stub.set(rpcSendRequest, schtRpcSendRequest); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRequest", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRequest$", result); +#endif for (const auto& f : result) { stub.set(f.second, schtRpcSendRequest); } @@ -324,9 +347,16 @@ void schtSetAsyncSendMsgToServer() { static Stub stub; stub.set(asyncSendMsgToServer, schtAsyncSendMsgToServer); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("asyncSendMsgToServer", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^asyncSendMsgToServer$", result); +#endif for (const auto& f : result) { stub.set(f.second, schtAsyncSendMsgToServer); } @@ -382,6 +412,7 @@ void *schtCreateFetchRspThread(void *param) { schReleaseJob(job); assert(code == 0); + return NULL; } @@ -413,6 +444,7 @@ void *schtFetchRspThread(void *aa) { assert(code == 0 || code); } + return NULL; } void schtFreeQueryJob(int32_t freeThread) { @@ -595,6 +627,7 @@ void* schtRunJobThread(void *aa) { schedulerDestroy(); + return NULL; } void* schtFreeJobThread(void *aa) { @@ -602,6 +635,7 @@ void* schtFreeJobThread(void *aa) { taosUsleep(taosRand() % 100); schtFreeQueryJob(1); } + return NULL; } diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index 726ebdae07..86fec5f484 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -14,7 +14,7 @@ */ #include "syncEnv.h" -#include +// #include SSyncEnv *gSyncEnv = NULL; diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 3a62b19c10..01490030f2 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -1,5 +1,5 @@ # tdb -add_library(tdb "") +add_library(tdb STATIC "") target_sources(tdb PRIVATE "src/db/tdbPCache.c" diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 8a95635ddc..0dfebe9820 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -42,14 +42,17 @@ struct SBTree { ASSERT(TDB_FLAG_IS(flags, TDB_BTREE_ROOT) || TDB_FLAG_IS(flags, TDB_BTREE_LEAF) || \ TDB_FLAG_IS(flags, TDB_BTREE_ROOT | TDB_BTREE_LEAF) || TDB_FLAG_IS(flags, 0)) -typedef struct __attribute__((__packed__)) { + +#pragma pack(push,1) +typedef struct { TDB_BTREE_PAGE_COMMON_HDR } SLeafHdr; -typedef struct __attribute__((__packed__)) { - TDB_BTREE_PAGE_COMMON_HDR; +typedef struct { + TDB_BTREE_PAGE_COMMON_HDR SPgno pgno; // right-most child } SIntHdr; +#pragma pack(pop) typedef struct { u8 flags; diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 86272b3cad..ce753ad930 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -28,11 +28,10 @@ struct SPCache { SPage lru; }; -#define PCACHE_PAGE_HASH(pPgid) \ - ({ \ - u32 *t = (u32 *)((pPgid)->fileid); \ - t[0] + t[1] + t[2] + t[3] + t[4] + t[5] + (pPgid)->pgno; \ - }) +static inline int tdbPCachePageHash(const SPgid *pPgid) { + u32 *t = (u32 *)((pPgid)->fileid); + return t[0] + t[1] + t[2] + t[3] + t[4] + t[5] + (pPgid)->pgno; +} #define PAGE_IS_PINNED(pPage) ((pPage)->pLruNext == NULL) static int tdbPCacheOpenImpl(SPCache *pCache); @@ -130,7 +129,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) SPage *pPage; // 1. Search the hash table - pPage = pCache->pgHash[PCACHE_PAGE_HASH(pPgid) % pCache->nHash]; + pPage = pCache->pgHash[tdbPCachePageHash(pPgid) % pCache->nHash]; while (pPage) { if (TDB_IS_SAME_PAGE(&(pPage->pgid), pPgid)) break; pPage = pPage->pHashNext; @@ -218,7 +217,7 @@ static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) { SPage **ppPage; int h; - h = PCACHE_PAGE_HASH(&(pPage->pgid)); + h = tdbPCachePageHash(&(pPage->pgid)); for (ppPage = &(pCache->pgHash[h % pCache->nHash]); *ppPage != pPage; ppPage = &((*ppPage)->pHashNext)) ; ASSERT(*ppPage == pPage); @@ -230,7 +229,7 @@ static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) { static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage) { int h; - h = PCACHE_PAGE_HASH(&(pPage->pgid)) % pCache->nHash; + h = tdbPCachePageHash(&(pPage->pgid)) % pCache->nHash; pPage->pHashNext = pCache->pgHash[h]; pCache->pgHash[h] = pPage; diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index ea75f7fbeb..05939579d8 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -435,17 +435,20 @@ static int tdbPageDefragment(SPage *pPage) { } /* ---------------------------------------------------------------------------------------------------------- */ -typedef struct __attribute__((__packed__)) { + +#pragma pack(push,1) +typedef struct { u16 cellNum; u16 cellBody; u16 cellFree; u16 nFree; } SPageHdr; -typedef struct __attribute__((__packed__)) { +typedef struct { u16 szCell; u16 nxOffset; } SFreeCell; +#pragma pack(pop) // cellNum static inline int getPageCellNum(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellNum; } @@ -517,17 +520,19 @@ SPageMethods pageMethods = { setPageFreeCellInfo // setFreeCellInfo }; -typedef struct __attribute__((__packed__)) { +#pragma pack(push,1) +typedef struct { u8 cellNum[3]; u8 cellBody[3]; u8 cellFree[3]; u8 nFree[3]; } SPageHdrL; -typedef struct __attribute__((__packed__)) { +typedef struct { u8 szCell[3]; u8 nxOffset[3]; } SFreeCellL; +#pragma pack(pop) // cellNum static inline int getLPageCellNum(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum); } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 7672ccf34e..8835b7f759 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -15,13 +15,15 @@ #include "tdbInt.h" -typedef struct __attribute__((__packed__)) { +#pragma pack(push,1) +typedef struct { u8 hdrString[16]; u16 pageSize; SPgno freePage; u32 nFreePages; u8 reserved[102]; } SFileHdr; +#pragma pack(pop) TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct"); diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 77a1ee5f6e..3707a922ee 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -47,10 +47,13 @@ typedef struct { void (*setFreeCellInfo)(SCell *pCell, int szCell, int nxOffset); } SPageMethods; +#pragma pack(push,1) + // Page footer -typedef struct __attribute__((__packed__)) { +typedef struct { u8 cksm[4]; } SPageFtr; +#pragma pack(pop) struct SPage { tdb_spinlock_t lock; @@ -80,22 +83,23 @@ struct SPage { #define P_LOCK_BUSY 1 #define P_LOCK_FAIL -1 +static inline int tdbTryLockPage(tdb_spinlock_t *pLock) { + int ret; + if (tdbSpinlockTrylock(pLock) == 0) { + ret = P_LOCK_SUCC; + } else if (errno == EBUSY) { + ret = P_LOCK_BUSY; + } else { + ret = P_LOCK_FAIL; + } + return ret; +} + #define TDB_INIT_PAGE_LOCK(pPage) tdbSpinlockInit(&((pPage)->lock), 0) #define TDB_DESTROY_PAGE_LOCK(pPage) tdbSpinlockDestroy(&((pPage)->lock)) #define TDB_LOCK_PAGE(pPage) tdbSpinlockLock(&((pPage)->lock)) #define TDB_UNLOCK_PAGE(pPage) tdbSpinlockUnlock(&((pPage)->lock)) -#define TDB_TRY_LOCK_PAGE(pPage) \ - ({ \ - int ret; \ - if (tdbSpinlockTrylock(&((pPage)->lock)) == 0) { \ - ret = P_LOCK_SUCC; \ - } else if (errno == EBUSY) { \ - ret = P_LOCK_BUSY; \ - } else { \ - ret = P_LOCK_FAIL; \ - } \ - ret; \ - }) +#define TDB_TRY_LOCK_PAGE(pPage) tdbTryLockPage(&((pPage)->lock)) // APIs #define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index 7085e71567..cffd3f2746 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -31,20 +31,20 @@ extern "C" { int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique); int tdbGetFileSize(tdb_fd_t fd, int szPage, SPgno *size); -#define TDB_REALLOC(PTR, SIZE) \ - ({ \ - void *nPtr; \ - if ((PTR) == NULL || ((int *)(PTR))[-1] < (SIZE)) { \ - nPtr = tdbOsRealloc((PTR) ? (char *)(PTR) - sizeof(int) : NULL, (SIZE) + sizeof(int)); \ - if (nPtr) { \ - ((int *)nPtr)[0] = (SIZE); \ - nPtr = (char *)nPtr + sizeof(int); \ - } \ - } else { \ - nPtr = (PTR); \ - } \ - nPtr; \ - }) +static inline void *tdbRealloc(void *ptr, size_t size) { + void *nPtr; + if ((ptr) == NULL || ((int *)(ptr))[-1] < (size)) { + nPtr = tdbOsRealloc((ptr) ? (char *)(ptr) - sizeof(int) : NULL, (size) + sizeof(int)); + if (nPtr) { + ((int *)nPtr)[0] = (size); + nPtr = (char *)nPtr + sizeof(int); + } + } else { + nPtr = (ptr); + } + return nPtr; +} +#define TDB_REALLOC(PTR, SIZE) tdbRealloc(PTR, SIZE) #define TDB_FREE(PTR) \ do { \ diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index e50045d30b..18a3a28bab 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -366,7 +366,7 @@ const STfsFile *tfsReaddir(STfsDir *pTfsDir) { void tfsClosedir(STfsDir *pTfsDir) { if (pTfsDir) { if (pTfsDir->pDir != NULL) { - taosCloseDir(pTfsDir->pDir); + taosCloseDir(&pTfsDir->pDir); pTfsDir->pDir = NULL; } taosMemoryFree(pTfsDir); @@ -455,7 +455,7 @@ static int32_t tfsFormatDir(char *idir, char *odir) { } char tmp[PATH_MAX] = {0}; - if (realpath(wep.we_wordv[0], tmp) == NULL) { + if (taosRealPath(wep.we_wordv[0], tmp, PATH_MAX) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); wordfree(&wep); return -1; @@ -499,7 +499,7 @@ static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pTfsDir) { char adir[TMPNAME_LEN * 2] = "\0"; if (pTfsDir->pDir != NULL) { - taosCloseDir(pTfsDir->pDir); + taosCloseDir(&pTfsDir->pDir); pTfsDir->pDir = NULL; } diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index cdf0908167..2b1c4fa623 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -12,8 +12,6 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include - #include #include "os.h" #include "rpcLog.h" diff --git a/source/libs/transport/test/syncClient.c b/source/libs/transport/test/syncClient.c index 27a468a86f..3f1a7805b4 100644 --- a/source/libs/transport/test/syncClient.c +++ b/source/libs/transport/test/syncClient.c @@ -12,8 +12,6 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include - #include #include "os.h" #include "rpcLog.h" diff --git a/source/libs/transport/test/transportTests.cpp b/source/libs/transport/test/transportTests.cpp index 2488dda3f6..35009c7686 100644 --- a/source/libs/transport/test/transportTests.cpp +++ b/source/libs/transport/test/transportTests.cpp @@ -156,14 +156,14 @@ TEST_F(TransCtxEnv, mergeTest) { STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); transCtxInit(src); { - STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = (void (*)(const void*))taosMemoryFree}; + STransCtxVal val1 = { NULL, NULL, (void (*)(const void*))taosMemoryFree}; val1.val = taosMemoryMalloc(12); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; } { - STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = (void (*)(const void*))taosMemoryFree}; + STransCtxVal val1 = { NULL, NULL, (void (*)(const void*))taosMemoryFree}; val1.val = taosMemoryMalloc(12); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; @@ -176,14 +176,14 @@ TEST_F(TransCtxEnv, mergeTest) { STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); transCtxInit(src); { - STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = (void (*)(const void*))taosMemoryFree}; + STransCtxVal val1 = { NULL, NULL, (void (*)(const void*))taosMemoryFree}; val1.val = taosMemoryMalloc(12); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; } { - STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = (void (*)(const void*))taosMemoryFree}; + STransCtxVal val1 = { NULL, NULL, (void (*)(const void*))taosMemoryFree}; val1.val = taosMemoryMalloc(12); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; @@ -198,7 +198,7 @@ TEST_F(TransCtxEnv, mergeTest) { STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); transCtxInit(src); { - STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = (void (*)(const void*))taosMemoryFree}; + STransCtxVal val1 = { NULL, NULL, (void (*)(const void*))taosMemoryFree}; val1.val = taosMemoryCalloc(1, 11); memcpy(val1.val, val.c_str(), val.size()); @@ -206,7 +206,7 @@ TEST_F(TransCtxEnv, mergeTest) { key++; } { - STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = (void (*)(const void*))taosMemoryFree}; + STransCtxVal val1 = { NULL, NULL, (void (*)(const void*))taosMemoryFree}; val1.val = taosMemoryCalloc(1, 11); memcpy(val1.val, val.c_str(), val.size()); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 9f8fe8d84d..3e6663f464 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -145,7 +145,7 @@ int walCheckAndRepairMeta(SWal* pWal) { } } - taosCloseDir(pDir); + taosCloseDir(&pDir); regfree(&logRegPattern); regfree(&idxRegPattern); @@ -351,7 +351,7 @@ static int walFindCurMetaVer(SWal* pWal) { break; } } - taosCloseDir(pDir); + taosCloseDir(&pDir); regfree(&walMetaRegexPattern); return metaVer; } diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index 261ec312ad..d709bbbcc2 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -6,7 +6,8 @@ target_include_directories( PUBLIC "${TD_SOURCE_DIR}/include" PUBLIC "${TD_SOURCE_DIR}/include/util" PUBLIC "${TD_SOURCE_DIR}/contrib/pthread" - PUBLIC "${TD_SOURCE_DIR}/contrib/gnuregex" + PUBLIC "${TD_SOURCE_DIR}/contrib/iconv" + PUBLIC "${TD_SOURCE_DIR}/contrib/msvcregex" ) # iconv find_path(IconvApiIncludes iconv.h PATHS) @@ -17,5 +18,14 @@ if(USE_TD_MEMORY) add_definitions(-DUSE_TD_MEMORY) endif () target_link_libraries( - os pthread dl rt m + os pthread ) +if(NOT TD_WINDOWS) + target_link_libraries( + os dl m rt + ) +else() + target_link_libraries( + os ws2_32 iconv msvcregex wcwidth winmm + ) +endif(NOT TD_WINDOWS) diff --git a/source/os/src/osAtomic.c b/source/os/src/osAtomic.c index 8b452947d3..0fe946bf68 100644 --- a/source/os/src/osAtomic.c +++ b/source/os/src/osAtomic.c @@ -16,78 +16,101 @@ #define ALLOW_FORBID_FUNC #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS // add -char interlocked_add_fetch_8(char volatile* ptr, char val) { +int8_t interlocked_add_fetch_8(int8_t volatile* ptr, int8_t val) { return _InterlockedExchangeAdd8(ptr, val) + val; } -short interlocked_add_fetch_16(short volatile* ptr, short val) { +int16_t interlocked_add_fetch_16(int16_t volatile* ptr, int16_t val) { return _InterlockedExchangeAdd16(ptr, val) + val; } -long interlocked_add_fetch_32(long volatile* ptr, long val) { +int32_t interlocked_add_fetch_32(int32_t volatile* ptr, int32_t val) { return _InterlockedExchangeAdd(ptr, val) + val; } -__int64 interlocked_add_fetch_64(__int64 volatile* ptr, __int64 val) { +int64_t interlocked_add_fetch_64(int64_t volatile* ptr, int64_t val) { return InterlockedExchangeAdd64(ptr, val) + val; } -char interlocked_and_fetch_8(char volatile* ptr, char val) { +void* interlocked_add_fetch_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)(_InterlockedExchangeAdd((int32_t volatile*)(ptr), (int32_t)val) + (int32_t)val); +#else + return (void*)(InterlockedExchangeAdd64((int64_t volatile*)(ptr), (int64_t)val) + (int64_t)val); +#endif +} + +int8_t interlocked_and_fetch_8(int8_t volatile* ptr, int8_t val) { return _InterlockedAnd8(ptr, val) & val; } -short interlocked_and_fetch_16(short volatile* ptr, short val) { +int16_t interlocked_and_fetch_16(int16_t volatile* ptr, int16_t val) { return _InterlockedAnd16(ptr, val) & val; } -long interlocked_and_fetch_32(long volatile* ptr, long val) { +int32_t interlocked_and_fetch_32(int32_t volatile* ptr, int32_t val) { return _InterlockedAnd(ptr, val) & val; } - -__int64 interlocked_and_fetch_64(__int64 volatile* ptr, __int64 val) { -#ifndef _M_IX86 - return _InterlockedAnd64(ptr, val) & val; -#else - __int64 old, res; +int64_t interlocked_and_fetch_64(int64_t volatile* ptr, int64_t val) { +#ifdef _TD_WINDOWS_32 + int64_t old, res; do { old = *ptr; res = old & val; } while (_InterlockedCompareExchange64(ptr, res, old) != old); return res; +#else + return _InterlockedAnd64(ptr, val) & val; #endif } -__int64 interlocked_fetch_and_64(__int64 volatile* ptr, __int64 val) { -#ifdef _M_IX86 - __int64 old; +void* interlocked_and_fetch_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)interlocked_and_fetch_32((int32_t volatile*)ptr, (int32_t)val); +#else + return (void*)interlocked_and_fetch_64((int64_t volatile*)ptr, (int64_t)val); +#endif +} + +int64_t interlocked_fetch_and_64(int64_t volatile* ptr, int64_t val) { +#ifdef _TD_WINDOWS_32 + int64_t old; do { old = *ptr; } while (_InterlockedCompareExchange64(ptr, old & val, old) != old); return old; #else - return _InterlockedAnd64((__int64 volatile*)(ptr), (__int64)(val)); + return _InterlockedAnd64((int64_t volatile*)(ptr), (int64_t)(val)); #endif } -char interlocked_or_fetch_8(char volatile* ptr, char val) { +void* interlocked_fetch_and_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)_InterlockedAnd((int32_t volatile*)(ptr), (int32_t)(val)); +#else + return (void*)_InterlockedAnd64((int64_t volatile*)(ptr), (int64_t)(val)); +#endif +} + +int8_t interlocked_or_fetch_8(int8_t volatile* ptr, int8_t val) { return _InterlockedOr8(ptr, val) | val; } -short interlocked_or_fetch_16(short volatile* ptr, short val) { +int16_t interlocked_or_fetch_16(int16_t volatile* ptr, int16_t val) { return _InterlockedOr16(ptr, val) | val; } -long interlocked_or_fetch_32(long volatile* ptr, long val) { +int32_t interlocked_or_fetch_32(int32_t volatile* ptr, int32_t val) { return _InterlockedOr(ptr, val) | val; } -__int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val) { -#ifdef _M_IX86 - __int64 old, res; +int64_t interlocked_or_fetch_64(int64_t volatile* ptr, int64_t val) { +#ifdef _TD_WINDOWS_32 + int64_t old, res; do { old = *ptr; res = old | val; @@ -98,33 +121,49 @@ __int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val) { #endif } -__int64 interlocked_fetch_or_64(__int64 volatile* ptr, __int64 val) { -#ifdef _M_IX86 - __int64 old; +void* interlocked_or_fetch_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)interlocked_or_fetch_32((int32_t volatile*)ptr, (int32_t)val); +#else + return (void*)interlocked_or_fetch_64((int64_t volatile*)ptr, (int64_t)val); +#endif +} + +int64_t interlocked_fetch_or_64(int64_t volatile* ptr, int64_t val) { +#ifdef _TD_WINDOWS_32 + int64_t old; do { old = *ptr; } while(_InterlockedCompareExchange64(ptr, old | val, old) != old); return old; #else - return _InterlockedOr64((__int64 volatile*)(ptr), (__int64)(val)); + return _InterlockedOr64((int64_t volatile*)(ptr), (int64_t)(val)); #endif } -char interlocked_xor_fetch_8(char volatile* ptr, char val) { +void* interlocked_fetch_or_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)_InterlockedOr((int32_t volatile*)(ptr), (int32_t)(val)); +#else + return (void*)interlocked_fetch_or_64((int64_t volatile*)(ptr), (int64_t)(val)); +#endif +} + +int8_t interlocked_xor_fetch_8(int8_t volatile* ptr, int8_t val) { return _InterlockedXor8(ptr, val) ^ val; } -short interlocked_xor_fetch_16(short volatile* ptr, short val) { +int16_t interlocked_xor_fetch_16(int16_t volatile* ptr, int16_t val) { return _InterlockedXor16(ptr, val) ^ val; } -long interlocked_xor_fetch_32(long volatile* ptr, long val) { +int32_t interlocked_xor_fetch_32(int32_t volatile* ptr, int32_t val) { return _InterlockedXor(ptr, val) ^ val; } -__int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val) { -#ifdef _M_IX86 - __int64 old, res; +int64_t interlocked_xor_fetch_64(int64_t volatile* ptr, int64_t val) { +#ifdef _TD_WINDOWS_32 + int64_t old, res; do { old = *ptr; res = old ^ val; @@ -135,15 +174,62 @@ __int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val) { #endif } -__int64 interlocked_fetch_xor_64(__int64 volatile* ptr, __int64 val) { -#ifdef _M_IX86 - __int64 old; +void* interlocked_xor_fetch_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)interlocked_xor_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); +#else + return (void*)interlocked_xor_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); +#endif +} + +int64_t interlocked_fetch_xor_64(int64_t volatile* ptr, int64_t val) { +#ifdef _TD_WINDOWS_32 + int64_t old; do { old = *ptr; } while (_InterlockedCompareExchange64(ptr, old ^ val, old) != old); return old; #else - return _InterlockedXor64((__int64 volatile*)(ptr), (__int64)(val)); + return _InterlockedXor64((int64_t volatile*)(ptr), (int64_t)(val)); +#endif +} + +void* interlocked_fetch_xor_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)_InterlockedXor((int32_t volatile*)(ptr), (int32_t)(val)); +#else + return (void*)interlocked_fetch_xor_64((int64_t volatile*)(ptr), (int64_t)(val)); +#endif +} + +int32_t interlocked_sub_fetch_32(int32_t volatile* ptr, int32_t val) { + return interlocked_add_fetch_32(ptr, -val); +} + +int64_t interlocked_sub_fetch_64(int64_t volatile* ptr, int64_t val) { + return interlocked_add_fetch_64(ptr, -val); +} + +void* interlocked_sub_fetch_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)interlocked_sub_fetch_32((int32_t volatile*)ptr, (int32_t)val); +#else + return (void*)interlocked_add_fetch_64((int64_t volatile*)ptr, (int64_t)val); +#endif +} +int32_t interlocked_fetch_sub_32(int32_t volatile* ptr, int32_t val) { + return _InterlockedExchangeAdd(ptr, -val); +} + +int64_t interlocked_fetch_sub_64(int64_t volatile* ptr, int64_t val) { + return _InterlockedExchangeAdd64(ptr, -val); +} + +void* interlocked_fetch_sub_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)interlocked_fetch_sub_32((int32_t volatile*)ptr, (int32_t)val); +#else + return (void*)interlocked_fetch_sub_64((int64_t volatile*)ptr, (int64_t)val); #endif } @@ -189,7 +275,7 @@ int64_t atomic_exchange_64_impl(int64_t* ptr, int64_t val ) { int8_t atomic_load_8(int8_t volatile *ptr) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return (*(int8_t volatile*)(ptr)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), 0); @@ -199,7 +285,7 @@ int8_t atomic_load_8(int8_t volatile *ptr) { } int16_t atomic_load_16(int16_t volatile *ptr) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return (*(int16_t volatile*)(ptr)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), 0); @@ -209,7 +295,7 @@ int16_t atomic_load_16(int16_t volatile *ptr) { } int32_t atomic_load_32(int32_t volatile *ptr) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return (*(int32_t volatile*)(ptr)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), 0); @@ -219,7 +305,7 @@ int32_t atomic_load_32(int32_t volatile *ptr) { } int64_t atomic_load_64(int64_t volatile *ptr) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return (*(int64_t volatile*)(ptr)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), 0); @@ -229,7 +315,7 @@ int64_t atomic_load_64(int64_t volatile *ptr) { } void* atomic_load_ptr(void *ptr) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return (*(void* volatile*)(ptr)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), 0); @@ -239,7 +325,7 @@ void* atomic_load_ptr(void *ptr) { } void atomic_store_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS ((*(int8_t volatile*)(ptr)) = (int8_t)(val)); #elif defined(_TD_NINGSI_60) (*(ptr)=(val)); @@ -249,7 +335,7 @@ void atomic_store_8(int8_t volatile *ptr, int8_t val) { } void atomic_store_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS ((*(int16_t volatile*)(ptr)) = (int16_t)(val)); #elif defined(_TD_NINGSI_60) (*(ptr)=(val)); @@ -259,7 +345,7 @@ void atomic_store_16(int16_t volatile *ptr, int16_t val) { } void atomic_store_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS ((*(int32_t volatile*)(ptr)) = (int32_t)(val)); #elif defined(_TD_NINGSI_60) (*(ptr)=(val)); @@ -269,7 +355,7 @@ void atomic_store_32(int32_t volatile *ptr, int32_t val) { } void atomic_store_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS ((*(int64_t volatile*)(ptr)) = (int64_t)(val)); #elif defined(_TD_NINGSI_60) (*(ptr)=(val)); @@ -279,7 +365,7 @@ void atomic_store_64(int64_t volatile *ptr, int64_t val) { } void atomic_store_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS ((*(void* volatile*)(ptr)) = (void*)(val)); #elif defined(_TD_NINGSI_60) (*(ptr)=(val)); @@ -289,7 +375,7 @@ void atomic_store_ptr(void *ptr, void *val) { } int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchange8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return atomic_exchange_8_impl((int8_t*)ptr, (int8_t)val); @@ -299,7 +385,7 @@ int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchange16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return atomic_exchange_16_impl((int16_t*)ptr, (int16_t)val); @@ -309,7 +395,7 @@ int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchange((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return atomic_exchange_32_impl((int32_t*)ptr, (int32_t)val); @@ -319,7 +405,7 @@ int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchange64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return atomic_exchange_64_impl((int64_t*)ptr, (int64_t)val); @@ -329,7 +415,7 @@ int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val) { } void* atomic_exchange_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #ifdef _WIN64 return _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)); #else @@ -343,7 +429,7 @@ void* atomic_exchange_ptr(void *ptr, void *val) { } int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedCompareExchange8((int8_t volatile*)(ptr), (int8_t)(newval), (int8_t)(oldval)); #elif defined(_TD_NINGSI_60) return __sync_val_compare_and_swap(ptr, oldval, newval); @@ -353,7 +439,7 @@ int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t } int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedCompareExchange16((int16_t volatile*)(ptr), (int16_t)(newval), (int16_t)(oldval)); #elif defined(_TD_NINGSI_60) return __sync_val_compare_and_swap(ptr, oldval, newval); @@ -363,7 +449,7 @@ int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, in } int32_t atomic_val_compare_exchange_32(int32_t volatile *ptr, int32_t oldval, int32_t newval) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedCompareExchange((int32_t volatile*)(ptr), (int32_t)(newval), (int32_t)(oldval)); #elif defined(_TD_NINGSI_60) return __sync_val_compare_and_swap(ptr, oldval, newval); @@ -373,7 +459,7 @@ int32_t atomic_val_compare_exchange_32(int32_t volatile *ptr, int32_t oldval, in } int64_t atomic_val_compare_exchange_64(int64_t volatile *ptr, int64_t oldval, int64_t newval) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedCompareExchange64((int64_t volatile*)(ptr), (int64_t)(newval), (int64_t)(oldval)); #elif defined(_TD_NINGSI_60) return __sync_val_compare_and_swap(ptr, oldval, newval); @@ -383,7 +469,7 @@ int64_t atomic_val_compare_exchange_64(int64_t volatile *ptr, int64_t oldval, in } void* atomic_val_compare_exchange_ptr(void *ptr, void *oldval, void *newval) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedCompareExchangePointer((void* volatile*)(ptr), (void*)(newval), (void*)(oldval)); #elif defined(_TD_NINGSI_60) return __sync_val_compare_and_swap(ptr, oldval, newval); @@ -393,7 +479,7 @@ void* atomic_val_compare_exchange_ptr(void *ptr, void *oldval, void *newval) { } int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_add_fetch_8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_add_and_fetch((ptr), (val)); @@ -403,7 +489,7 @@ int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_add_fetch_16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_add_and_fetch((ptr), (val)); @@ -413,7 +499,7 @@ int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_add_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_add_and_fetch((ptr), (val)); @@ -423,7 +509,7 @@ int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_add_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_add_and_fetch((ptr), (val)); @@ -432,18 +518,18 @@ int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val) { #endif } -void* atomic_add_fetch_ptr(void *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +void* atomic_add_fetch_ptr(void *ptr, void *val) { +#ifdef WINDOWS return interlocked_add_fetch_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_add_and_fetch((ptr), (val)); #else - return __atomic_add_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST); + return __atomic_add_fetch((void **)(ptr), (size_t)(val), __ATOMIC_SEQ_CST); #endif } int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchangeAdd8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), (val)); @@ -453,7 +539,7 @@ int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchangeAdd16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), (val)); @@ -463,7 +549,7 @@ int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchangeAdd((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), (val)); @@ -473,7 +559,7 @@ int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchangeAdd64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), (val)); @@ -482,18 +568,18 @@ int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val) { #endif } -void* atomic_fetch_add_ptr(void *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return _InterlockedExchangeAddptr((void* volatile*)(ptr), (void*)(val)); +void* atomic_fetch_add_ptr(void *ptr, void *val) { +#ifdef WINDOWS + return _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), (val)); #else - return __atomic_fetch_add((void **)(ptr), (val), __ATOMIC_SEQ_CST); + return __atomic_fetch_add((void **)(ptr), (size_t)(val), __ATOMIC_SEQ_CST); #endif } int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_add_fetch_8((int8_t volatile*)(ptr), -(int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_sub_and_fetch((ptr), (val)); @@ -503,7 +589,7 @@ int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_add_fetch_16((int16_t volatile*)(ptr), -(int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_sub_and_fetch((ptr), (val)); @@ -513,8 +599,8 @@ int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return interlocked_add_fetch_32((int32_t volatile*)(ptr), -(int32_t)(val)); +#ifdef WINDOWS + return interlocked_sub_fetch_32(ptr, val); #elif defined(_TD_NINGSI_60) return __sync_sub_and_fetch((ptr), (val)); #else @@ -523,8 +609,8 @@ int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return interlocked_add_fetch_64((int64_t volatile*)(ptr), -(int64_t)(val)); +#ifdef WINDOWS + return interlocked_sub_fetch_64(ptr, val); #elif defined(_TD_NINGSI_60) return __sync_sub_and_fetch((ptr), (val)); #else @@ -532,9 +618,9 @@ int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val) { #endif } -void* atomic_sub_fetch_ptr(void *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return interlocked_add_fetch_ptr((void* volatile*)(ptr), -(void*)(val)); +void* atomic_sub_fetch_ptr(void *ptr, void* val) { +#ifdef WINDOWS + return interlocked_sub_fetch_ptr(ptr, val); #elif defined(_TD_NINGSI_60) return __sync_sub_and_fetch((ptr), (val)); #else @@ -543,7 +629,7 @@ void* atomic_sub_fetch_ptr(void *ptr, int32_t val) { } int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchangeAdd8((int8_t volatile*)(ptr), -(int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_sub((ptr), (val)); @@ -553,7 +639,7 @@ int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchangeAdd16((int16_t volatile*)(ptr), -(int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_sub((ptr), (val)); @@ -563,17 +649,17 @@ int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return _InterlockedExchangeAdd((int32_t volatile*)(ptr), -(int32_t)(val)); +#ifdef WINDOWS + return interlocked_fetch_sub_32(ptr, val); #elif defined(_TD_NINGSI_60) - return __sync_fetch_and_sub((ptr), (val)); + return __sync_fetch_and_sub(ptr, val); #else return __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST); #endif } int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchangeAdd64((int64_t volatile*)(ptr), -(int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_sub((ptr), (val)); @@ -582,9 +668,9 @@ int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val) { #endif } -void* atomic_fetch_sub_ptr(void *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return _InterlockedExchangeAddptr((void* volatile*)(ptr), -(void*)(val)); +void* atomic_fetch_sub_ptr(void *ptr, void* val) { +#ifdef WINDOWS + return interlocked_fetch_sub_ptr(ptr, val); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_sub((ptr), (val)); #else @@ -593,7 +679,7 @@ void* atomic_fetch_sub_ptr(void *ptr, int32_t val) { } int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_and_fetch_8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_and_and_fetch((ptr), (val)); @@ -603,7 +689,7 @@ int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_and_fetch_16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_and_and_fetch((ptr), (val)); @@ -613,7 +699,7 @@ int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_and_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_and_and_fetch((ptr), (val)); @@ -623,7 +709,7 @@ int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_and_fetch_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_and_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_and_and_fetch((ptr), (val)); @@ -633,7 +719,7 @@ int64_t atomic_and_fetch_64(int64_t volatile *ptr, int64_t val) { } void* atomic_and_fetch_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_and_fetch_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_and_and_fetch((ptr), (val)); @@ -643,7 +729,7 @@ void* atomic_and_fetch_ptr(void *ptr, void *val) { } int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedAnd8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_and((ptr), (val)); @@ -653,7 +739,7 @@ int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_fetch_and_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedAnd16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_and((ptr), (val)); @@ -663,7 +749,7 @@ int16_t atomic_fetch_and_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_fetch_and_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedAnd((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_and((ptr), (val)); @@ -673,7 +759,7 @@ int32_t atomic_fetch_and_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_fetch_and_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_fetch_and_64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_and((ptr), (val)); @@ -683,7 +769,7 @@ int64_t atomic_fetch_and_64(int64_t volatile *ptr, int64_t val) { } void* atomic_fetch_and_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_fetch_and_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_and((ptr), (val)); @@ -693,7 +779,7 @@ void* atomic_fetch_and_ptr(void *ptr, void *val) { } int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_or_fetch_8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_or_and_fetch((ptr), (val)); @@ -703,7 +789,7 @@ int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_or_fetch_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_or_fetch_16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_or_and_fetch((ptr), (val)); @@ -713,7 +799,7 @@ int16_t atomic_or_fetch_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_or_fetch_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_or_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_or_and_fetch((ptr), (val)); @@ -723,7 +809,7 @@ int32_t atomic_or_fetch_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_or_fetch_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_or_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_or_and_fetch((ptr), (val)); @@ -733,7 +819,7 @@ int64_t atomic_or_fetch_64(int64_t volatile *ptr, int64_t val) { } void* atomic_or_fetch_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_or_fetch_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_or_and_fetch((ptr), (val)); @@ -743,7 +829,7 @@ void* atomic_or_fetch_ptr(void *ptr, void *val) { } int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedOr8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_or((ptr), (val)); @@ -753,7 +839,7 @@ int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_fetch_or_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedOr16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_or((ptr), (val)); @@ -763,7 +849,7 @@ int16_t atomic_fetch_or_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_fetch_or_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedOr((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_or((ptr), (val)); @@ -773,7 +859,7 @@ int32_t atomic_fetch_or_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_fetch_or_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_fetch_or_64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_or((ptr), (val)); @@ -783,7 +869,7 @@ int64_t atomic_fetch_or_64(int64_t volatile *ptr, int64_t val) { } void* atomic_fetch_or_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_fetch_or_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_or((ptr), (val)); @@ -793,7 +879,7 @@ void* atomic_fetch_or_ptr(void *ptr, void *val) { } int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_xor_fetch_8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_xor_and_fetch((ptr), (val)); @@ -803,7 +889,7 @@ int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_xor_fetch_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_xor_fetch_16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_xor_and_fetch((ptr), (val)); @@ -813,7 +899,7 @@ int16_t atomic_xor_fetch_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_xor_fetch_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_xor_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_xor_and_fetch((ptr), (val)); @@ -823,7 +909,7 @@ int32_t atomic_xor_fetch_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_xor_fetch_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_xor_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_xor_and_fetch((ptr), (val)); @@ -833,7 +919,7 @@ int64_t atomic_xor_fetch_64(int64_t volatile *ptr, int64_t val) { } void* atomic_xor_fetch_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_xor_fetch_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_xor_and_fetch((ptr), (val)); @@ -843,7 +929,7 @@ void* atomic_xor_fetch_ptr(void *ptr, void *val) { } int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedXor8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_xor((ptr), (val)); @@ -853,7 +939,7 @@ int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_fetch_xor_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedXor16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_xor((ptr), (val)); @@ -863,7 +949,7 @@ int16_t atomic_fetch_xor_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedXor((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_xor((ptr), (val)); @@ -873,7 +959,7 @@ int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_fetch_xor_64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_xor((ptr), (val)); @@ -883,7 +969,7 @@ int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val) { } void* atomic_fetch_xor_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_fetch_xor_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_xor((ptr), (val)); diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index a955fb3b0a..46fa2ecd3a 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -17,19 +17,40 @@ #define ALLOW_FORBID_FUNC #include "os.h" -#include "osString.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) -/* - * windows implementation - */ +#ifdef WINDOWS -// todo +#include + +typedef struct TdDirEntry { + WIN32_FIND_DATA findFileData; +} TdDirEntry; + + +typedef struct TdDir { + TdDirEntry dirEntry; + HANDLE hFind; +} TdDir; + +int wordexp(char *words, wordexp_t *pwordexp, int flags) { + pwordexp->we_offs = 0; + pwordexp->we_wordc = 1; + pwordexp->we_wordv[0] = pwordexp->wordPos; + + memset(pwordexp->wordPos, 0, 1025); + if (_fullpath(pwordexp->wordPos, words, 1024) == NULL) { + pwordexp->we_wordv[0] = words; + printf("failed to parse relative path:%s to abs path", words); + return -1; + } + + printf("parse relative path:%s to abs path:%s", words, pwordexp->wordPos); + return 0; +} + +void wordfree(wordexp_t *pwordexp) {} #else -/* - * linux implementation - */ #include #include @@ -39,19 +60,21 @@ typedef struct dirent dirent; typedef struct DIR TdDir; -typedef struct dirent TdDirent; +typedef struct dirent TdDirEntry; + +#endif void taosRemoveDir(const char *dirname) { - DIR *dir = opendir(dirname); - if (dir == NULL) return; + TdDirPtr pDir = taosOpenDir(dirname); + if (pDir == NULL) return; - struct dirent *de = NULL; - while ((de = readdir(dir)) != NULL) { - if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; + TdDirEntryPtr de = NULL; + while ((de = taosReadDir(pDir)) != NULL) { + if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue; char filename[1024]; - snprintf(filename, sizeof(filename), "%s/%s", dirname, de->d_name); - if (de->d_type & DT_DIR) { + snprintf(filename, sizeof(filename), "%s/%s", dirname, taosGetDirEntryName(de)); + if (taosDirEntryIsDir(de)) { taosRemoveDir(filename); } else { (void)taosRemoveFile(filename); @@ -59,13 +82,14 @@ void taosRemoveDir(const char *dirname) { } } - closedir(dir); + taosCloseDir(&pDir); rmdir(dirname); //printf("dir:%s is removed\n", dirname); + return; } -bool taosDirExist(char *dirname) { return taosCheckExistFile(dirname); } +bool taosDirExist(const char *dirname) { return taosCheckExistFile(dirname); } int32_t taosMkDir(const char *dirname) { int32_t code = mkdir(dirname, 0755); @@ -118,18 +142,18 @@ int32_t taosMulMkDir(const char *dirname) { } void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { - DIR *dir = opendir(dirname); - if (dir == NULL) return; + TdDirPtr pDir = taosOpenDir(dirname); + if (pDir == NULL) return; - int64_t sec = taosGetTimestampSec(); - struct dirent *de = NULL; + int64_t sec = taosGetTimestampSec(); + TdDirEntryPtr de = NULL; - while ((de = readdir(dir)) != NULL) { - if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; + while ((de = taosReadDir(pDir)) != NULL) { + if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue; char filename[1024]; - snprintf(filename, sizeof(filename), "%s/%s", dirname, de->d_name); - if (de->d_type & DT_DIR) { + snprintf(filename, sizeof(filename), "%s/%s", dirname, taosGetDirEntryName(de)); + if (taosDirEntryIsDir(de)) { continue; } else { int32_t len = (int32_t)strlen(filename); @@ -156,7 +180,7 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { } } - closedir(dir); + taosCloseDir(&pDir); rmdir(dirname); } @@ -177,65 +201,131 @@ int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { return 0; } -int32_t taosRealPath(char *dirname, int32_t maxlen) { +int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen) { char tmp[PATH_MAX] = {0}; +#ifdef WINDOWS + if (_fullpath(dirname, tmp, maxlen) != NULL) { +#else if (realpath(dirname, tmp) != NULL) { - strncpy(dirname, tmp, maxlen); +#endif + if (realPath == NULL) { + strncpy(dirname, tmp, maxlen); + } else { + strncpy(realPath, tmp, maxlen); + } + return 0; } - return 0; + return -1; } bool taosIsDir(const char *dirname) { - DIR *dir = opendir(dirname); - if (dir != NULL) { - closedir(dir); + TdDirPtr pDir = taosOpenDir(dirname); + if (pDir != NULL) { + taosCloseDir(&pDir); return true; } return false; } char* taosDirName(char *name) { +#ifdef WINDOWS + char Drive1[MAX_PATH], Dir1[MAX_PATH]; + _splitpath(name, Drive1, Dir1, NULL, NULL); + size_t dirNameLen = strlen(Drive1) + strlen(Dir1); + if (dirNameLen > 0) { + name[dirNameLen] = 0; + } + return name; +#else return dirname(name); +#endif } char* taosDirEntryBaseName(char *name) { - return basename(name); +#ifdef WINDOWS + char Filename1[MAX_PATH], Ext1[MAX_PATH]; + _splitpath(name, NULL, NULL, Filename1, Ext1); + return name + (strlen(name) - strlen(Filename1) - strlen(Ext1)); +#else + return (char*)basename(name); +#endif } TdDirPtr taosOpenDir(const char *dirname) { if (dirname == NULL) { return NULL; } + +#ifdef WINDOWS + char szFind[MAX_PATH]; //这是要找的 + HANDLE hFind; + + TdDirPtr pDir = taosMemoryMalloc(sizeof(TdDir)); + + strcpy(szFind, dirname); + strcat(szFind, "\\*.*"); //利用通配符找这个目录下的所以文件，包括目录 + + pDir->hFind = FindFirstFile(szFind, &(pDir->dirEntry.findFileData)); + if (INVALID_HANDLE_VALUE == pDir->hFind) { + taosMemoryFree(pDir); + return NULL; + } + return pDir; +#else return (TdDirPtr)opendir(dirname); +#endif + } TdDirEntryPtr taosReadDir(TdDirPtr pDir) { if (pDir == NULL) { return NULL; } +#ifdef WINDOWS + if (!FindNextFile(pDir->hFind, &(pDir->dirEntry.findFileData))) { + return NULL; + } + return (TdDirEntryPtr)&(pDir->dirEntry.findFileData); +#else return (TdDirEntryPtr)readdir((DIR*)pDir); +#endif } bool taosDirEntryIsDir(TdDirEntryPtr pDirEntry) { if (pDirEntry == NULL) { return false; } +#ifdef WINDOWS + return (pDirEntry->findFileData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; +#else return (((dirent*)pDirEntry)->d_type & DT_DIR) != 0; +#endif } char* taosGetDirEntryName(TdDirEntryPtr pDirEntry) { if (pDirEntry == NULL) { return NULL; } +#ifdef WINDOWS + return pDirEntry->findFileData.cFileName; +#else return ((dirent*)pDirEntry)->d_name; +#endif } -int32_t taosCloseDir(TdDirPtr pDir) { - if (pDir == NULL) { +int32_t taosCloseDir(TdDirPtr *ppDir) { + if (ppDir == NULL || *ppDir == NULL) { return -1; } - return closedir((DIR*)pDir); -} - +#ifdef WINDOWS + FindClose((*ppDir)->hFind); + taosMemoryFree(*ppDir); + *ppDir = NULL; + return 0; +#else + closedir((DIR*)*ppDir); + *ppDir = NULL; + return 0; #endif +} diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index e24ac41f20..6746025f78 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -16,8 +16,6 @@ #define _DEFAULT_SOURCE #include "osEnv.h" -extern void taosWinSocketInit(); - char configDir[PATH_MAX] = {0}; char tsDataDir[PATH_MAX] = {0}; char tsLogDir[PATH_MAX] = {0}; @@ -51,7 +49,7 @@ void osDefaultInit() { tsNumOfCores = 2; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS taosWinSocketInit(); const char *tmpDir = getenv("tmp"); diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 4f2f99dec7..786e58a6d2 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -16,8 +16,11 @@ #include "os.h" #include "osSemaphore.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #include +#define F_OK 0 +#define W_OK 2 +#define R_OK 4 #if defined(_MSDOS) #define open _open @@ -33,6 +36,8 @@ extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */ #endif /* defined(_UTF8_SOURCE) */ #endif /* defined(_WIN32) */ +#define _SEND_FILE_STEP_ 1000 + #else #include #include @@ -66,7 +71,7 @@ typedef struct TdFile { #define FILE_WITH_LOCK 1 void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS const char *tdengineTmpFileNamePrefix = "tdengine-"; char tmpPath[PATH_MAX]; @@ -117,7 +122,7 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, cha } int64_t taosCopyFile(const char *from, const char *to) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else char buffer[4096]; @@ -160,7 +165,7 @@ _err: int32_t taosRemoveFile(const char *path) { return remove(path); } int32_t taosRenameFile(const char *oldName, const char *newName) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); if (code < 0) { // printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); @@ -178,7 +183,7 @@ int32_t taosRenameFile(const char *oldName, const char *newName) { } int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else struct stat fileStat; @@ -199,7 +204,7 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { #endif } int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else struct stat fileStat; @@ -223,7 +228,7 @@ int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno) { void autoDelFileListAdd(const char *path) { return; } TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return NULL; #else int fd = -1; @@ -285,7 +290,7 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { } int64_t taosCloseFile(TdFilePtr *ppFile) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else if (ppFile == NULL || *ppFile == NULL) { @@ -365,8 +370,15 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + assert(pFile->fd >= 0); // Please check if you have closed the file. +#ifdef WINDOWS + size_t pos = lseek(pFile->fd, 0, SEEK_CUR); + lseek(pFile->fd, (long)offset, SEEK_SET); + int64_t ret = read(pFile->fd, buf, count); + lseek(pFile->fd, pos, SEEK_SET); +#else int64_t ret = pread(pFile->fd, buf, count, offset); +#endif #if FILE_WITH_LOCK taosThreadRwlockUnlock(&(pFile->rwlock)); #endif @@ -423,7 +435,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { } int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else if (pFile == NULL) { @@ -450,7 +462,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { } int32_t taosLockFile(TdFilePtr pFile) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else if (pFile == NULL) { @@ -463,7 +475,7 @@ int32_t taosLockFile(TdFilePtr pFile) { } int32_t taosUnLockFile(TdFilePtr pFile) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else if (pFile == NULL) { @@ -476,10 +488,10 @@ int32_t taosUnLockFile(TdFilePtr pFile) { } int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS if (pFile->fd < 0) { errno = EBADF; - uError("%s\n", "fd arg was negative"); + printf("%s\n", "fd arg was negative"); return -1; } @@ -489,7 +501,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { li_0.QuadPart = (int64_t)0; BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT); if (!cur) { - uError("SetFilePointerEx Error getting current position in file.\n"); + printf("SetFilePointerEx Error getting current position in file.\n"); return -1; } @@ -498,7 +510,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN); if (cur2 == 0) { int error = GetLastError(); - uError("SetFilePointerEx GetLastError is: %d\n", error); + printf("SetFilePointerEx GetLastError is: %d\n", error); switch (error) { case ERROR_INVALID_HANDLE: errno = EBADF; @@ -512,7 +524,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { if (!SetEndOfFile(h)) { int error = GetLastError(); - uError("SetEndOfFile GetLastError is:%d", error); + printf("SetEndOfFile GetLastError is:%d", error); switch (error) { case ERROR_INVALID_HANDLE: errno = EBADF; @@ -536,10 +548,10 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { } int32_t taosFsyncFile(TdFilePtr pFile) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS if (pFile->fd < 0) { errno = EBADF; - uError("%s\n", "fd arg was negative"); + printf("%s\n", "fd arg was negative"); return -1; } @@ -558,84 +570,51 @@ int32_t taosFsyncFile(TdFilePtr pFile) { #endif } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) { + if (pFileOut == NULL || pFileIn == NULL) { + return 0; + } + assert(pFileIn->fd >= 0 && pFileOut->fd >= 0); -#define _SEND_FILE_STEP_ 1000 +#ifdef WINDOWS -int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) { - fseek(in_file, (int32_t)(*offset), 0); + lseek(pFileIn->fd, (int32_t)(*offset), 0); int64_t writeLen = 0; uint8_t buffer[_SEND_FILE_STEP_] = {0}; - for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { - size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file); + for (int64_t len = 0; len < (size - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { + size_t rlen = read(pFileIn->fd, (void *)buffer, _SEND_FILE_STEP_); if (rlen <= 0) { return writeLen; } else if (rlen < _SEND_FILE_STEP_) { - fwrite(buffer, 1, rlen, out_file); + write(pFileOut->fd, (void *)buffer, (uint32_t)rlen); return (int64_t)(writeLen + rlen); } else { - fwrite(buffer, 1, _SEND_FILE_STEP_, in_file); + write(pFileOut->fd, (void *)buffer, _SEND_FILE_STEP_); writeLen += _SEND_FILE_STEP_; } } - int64_t remain = count - writeLen; + int64_t remain = size - writeLen; if (remain > 0) { - size_t rlen = fread(buffer, 1, (size_t)remain, in_file); + size_t rlen = read(pFileIn->fd, (void *)buffer, (size_t)remain); if (rlen <= 0) { return writeLen; } else { - fwrite(buffer, 1, (size_t)remain, out_file); + write(pFileOut->fd, (void *)buffer, (uint32_t)remain); writeLen += remain; } } - return writeLen; -} - -int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) { - if (offset != NULL) lseek(sfd, (int32_t)(*offset), 0); - - int64_t writeLen = 0; - uint8_t buffer[_SEND_FILE_STEP_] = {0}; - - for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { - int32_t rlen = (int32_t)read(sfd, buffer, _SEND_FILE_STEP_); - if (rlen <= 0) { - return writeLen; - } else if (rlen < _SEND_FILE_STEP_) { - taosWriteSocket(dfd, buffer, rlen); - return (int64_t)(writeLen + rlen); - } else { - taosWriteSocket(dfd, buffer, _SEND_FILE_STEP_); - writeLen += _SEND_FILE_STEP_; - } - } - - int64_t remain = count - writeLen; - if (remain > 0) { - int32_t rlen = read(sfd, buffer, (int32_t)remain); - if (rlen <= 0) { - return writeLen; - } else { - taosWriteSocket(sfd, buffer, (int32_t)remain); - writeLen += remain; - } - } - - return writeLen; -} #elif defined(_TD_DARWIN_64) -int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) { int r = 0; if (offset) { r = fseek(in_file, *offset, SEEK_SET); if (r == -1) return -1; } - off_t len = count; + off_t len = size; while (len > 0) { char buf[1024 * 16]; off_t n = sizeof(buf); @@ -651,64 +630,10 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co } len -= m; } - return count - len; -} - -int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) { - int r = 0; - if (offset) { - r = lseek(sfd, *offset, SEEK_SET); - if (r == -1) return -1; - } - off_t len = count; - while (len > 0) { - char buf[1024 * 16]; - off_t n = sizeof(buf); - if (len < n) n = len; - size_t m = read(sfd, buf, n); - if (m == -1) return -1; - if (m == 0) break; - size_t l = write(dfd, buf, m); - if (l == -1) return -1; - len -= l; - } - return count - len; -} + return size - len; #else -// int64_t taosSendFile(int fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size) { -// if (pFileSrc == NULL) { -// return 0; -// } -// assert(pFileSrc->fd >= 0); - -// int64_t leftbytes = size; -// int64_t sentbytes; - -// while (leftbytes > 0) { -// sentbytes = sendfile(fdDst, pFileSrc->fd, offset, leftbytes); -// if (sentbytes == -1) { -// if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { -// continue; -// } else { -// return -1; -// } -// } else if (sentbytes == 0) { -// return (int64_t)(size - leftbytes); -// } - -// leftbytes -= sentbytes; -// } - -// return size; -// } - -int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) { - if (pFileOut == NULL || pFileIn == NULL) { - return 0; - } - assert(pFileIn->fd >= 0 && pFileOut->fd >= 0); int64_t leftbytes = size; int64_t sentbytes; @@ -728,9 +653,8 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in } return size; -} - #endif +} void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { if (pFile == NULL) { @@ -745,22 +669,10 @@ void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { fflush(pFile->fp); } -#if !defined(WINDOWS) -void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length) { - if (pFile == NULL) { - return NULL; - } - assert(pFile->fd >= 0); // Please check if you have closed the file. - - void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, pFile->fd, 0); - return ptr; -} -#endif - bool taosValidFile(TdFilePtr pFile) { return pFile != NULL; } int32_t taosUmaskFile(int32_t maskVal) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else return umask(maskVal); @@ -769,16 +681,26 @@ int32_t taosUmaskFile(int32_t maskVal) { int32_t taosGetErrorFile(TdFilePtr pFile) { return errno; } int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { - if (pFile == NULL) { + if (pFile == NULL || ptrBuf == NULL ) { return -1; } if (*ptrBuf != NULL) { taosMemoryFreeClear(*ptrBuf); } assert(pFile->fp != NULL); - +#ifdef WINDOWS + *ptrBuf = taosMemoryMalloc(1024); + if (*ptrBuf == NULL) return -1; + if (fgets(*ptrBuf, 1023, pFile->fp) == NULL) { + taosMemoryFreeClear(*ptrBuf); + return -1; + } + (*ptrBuf)[1023] = 0; + return strlen(*ptrBuf); +#else size_t len = 0; return getline(ptrBuf, &len, pFile->fp); +#endif } int32_t taosEOFFile(TdFilePtr pFile) { if (pFile == NULL) { @@ -789,8 +711,6 @@ int32_t taosEOFFile(TdFilePtr pFile) { return feof(pFile->fp); } -#if !defined(WINDOWS) - bool taosCheckAccessFile(const char *pathname, int32_t tdFileAccessOptions) { int flags = 0; @@ -805,10 +725,11 @@ bool taosCheckAccessFile(const char *pathname, int32_t tdFileAccessOptions) { if (tdFileAccessOptions & TD_FILE_ACCESS_WRITE_OK) { flags |= W_OK; } - +#ifdef WINDOWS + return _access(pathname, flags) == 0; +#else return access(pathname, flags) == 0; +#endif } bool taosCheckExistFile(const char *pathname) { return taosCheckAccessFile(pathname, TD_FILE_ACCESS_EXIST_OK); }; - -#endif // WINDOWS diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c index 2df61c08d4..d2369ea2a2 100644 --- a/source/os/src/osLocale.c +++ b/source/os/src/osLocale.c @@ -17,7 +17,7 @@ #define _DEFAULT_SOURCE #include "osLocale.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #if (_WIN64) #include #include @@ -87,7 +87,7 @@ void taosSetSystemLocale(const char *inLocale, const char *inCharSet) { } void taosGetSystemLocale(char *outLocale, char *outCharset) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS char *locale = setlocale(LC_CTYPE, "chs"); if (locale != NULL) { tstrncpy(outLocale, locale, TD_LOCALE_LEN); diff --git a/source/os/src/osLz4.c b/source/os/src/osLz4.c index 29d219f37b..f4e4951857 100644 --- a/source/os/src/osLz4.c +++ b/source/os/src/osLz4.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS /* * windows implementation diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index cd756b45e2..5b733daec2 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -35,7 +35,7 @@ typedef struct TdMemoryInfo { // static TdMemoryInfoPtr GlobalMemoryPtr = NULL; -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #define tstrdup(str) _strdup(str) #else #define tstrdup(str) strdup(str) @@ -181,7 +181,11 @@ int32_t taosMemorySize(void *ptr) { assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); return pTdMemoryInfo->memorySize; +#else +#ifdef WINDOWS + return _msize(ptr); #else return malloc_usable_size(ptr); #endif +#endif } diff --git a/source/os/src/osProc.c b/source/os/src/osProc.c index d569582256..ae1b6c9e1f 100644 --- a/source/os/src/osProc.c +++ b/source/os/src/osProc.c @@ -20,6 +20,9 @@ char *tsProcPath = NULL; int32_t taosNewProc(char **args) { +#ifdef WINDOWS + return 0; +#else int32_t pid = fork(); if (pid == 0) { args[0] = tsProcPath; @@ -30,23 +33,36 @@ int32_t taosNewProc(char **args) { } else { return pid; } +#endif } void taosWaitProc(int32_t pid) { +#ifdef WINDOWS +#else int32_t status = -1; waitpid(pid, &status, 0); +#endif } -void taosKillProc(int32_t pid) { kill(pid, SIGINT); } +void taosKillProc(int32_t pid) { +#ifdef WINDOWS +#else + kill(pid, SIGINT); +#endif +} bool taosProcExist(int32_t pid) { +#ifdef WINDOWS + return false; +#else int32_t p = getpgid(pid); return p >= 0; +#endif } // the length of the new name must be less than the original name to take effect void taosSetProcName(int32_t argc, char **argv, const char *name) { - prctl(PR_SET_NAME, name); + setThreadName(name); for (int32_t i = 0; i < argc; ++i) { int32_t len = strlen(argv[i]); diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c index 847f3a52a6..461a72e962 100644 --- a/source/os/src/osRand.c +++ b/source/os/src/osRand.c @@ -15,7 +15,9 @@ #define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS +#include "windows.h" +#include "wincrypt.h" #else #include #include @@ -25,9 +27,25 @@ void taosSeedRand(uint32_t seed) { return srand(seed); } uint32_t taosRand(void) { return rand(); } -uint32_t taosRandR(uint32_t *pSeed) { return rand_r(pSeed); } +uint32_t taosRandR(uint32_t *pSeed) { +#ifdef WINDOWS + return rand_s(pSeed); +#else + return rand_r(pSeed); +#endif +} uint32_t taosSafeRand(void) { +#ifdef WINDOWS + uint32_t seed; + HCRYPTPROV hCryptProv; + if (!CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_FULL, 0)) return seed; + if (hCryptProv != NULL) { + if (!CryptGenRandom(hCryptProv, 4, &seed)) return seed; + } + if (hCryptProv != NULL) CryptReleaseContext(hCryptProv, 0); + return seed; +#else TdFilePtr pFile; int seed; @@ -43,6 +61,7 @@ uint32_t taosSafeRand(void) { } return (uint32_t)seed; +#endif } void taosRandStr(char* str, int32_t size) { diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 1297fdbc27..b2c7d185e6 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS /* * windows implementation diff --git a/source/os/src/osShm.c b/source/os/src/osShm.c index b276b48d0e..1cd51f94a0 100644 --- a/source/os/src/osShm.c +++ b/source/os/src/osShm.c @@ -22,15 +22,20 @@ static int32_t shmids[MAX_SHMIDS] = {0}; static void taosDeleteCreatedShms() { +#if defined(WINDOWS) +#else for (int32_t i = 0; i < MAX_SHMIDS; ++i) { int32_t shmid = shmids[i] - 1; if (shmid >= 0) { shmctl(shmid, IPC_RMID, NULL); } } +#endif } int32_t taosCreateShm(SShm* pShm, int32_t key, int32_t shmsize) { +#if defined(WINDOWS) +#else pShm->id = -1; #if 1 @@ -64,10 +69,13 @@ int32_t taosCreateShm(SShm* pShm, int32_t key, int32_t shmsize) { shmctl(pShm->id, IPC_RMID, NULL); #endif +#endif return 0; } void taosDropShm(SShm* pShm) { +#if defined(WINDOWS) +#else if (pShm->id >= 0) { if (pShm->ptr != NULL) { shmdt(pShm->ptr); @@ -77,14 +85,18 @@ void taosDropShm(SShm* pShm) { pShm->id = -1; pShm->size = 0; pShm->ptr = NULL; +#endif } int32_t taosAttachShm(SShm* pShm) { +#if defined(WINDOWS) +#else errno = 0; void* ptr = shmat(pShm->id, NULL, 0); if (errno == 0) { pShm->ptr = ptr; } +#endif return errno; } diff --git a/source/os/src/osSignal.c b/source/os/src/osSignal.c index 1d7fa517e5..d9b225868a 100644 --- a/source/os/src/osSignal.c +++ b/source/os/src/osSignal.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS /* * windows implementation @@ -47,6 +47,8 @@ void taosDflSignal(int32_t signum) { signal(signum, SIG_DFL); } +void taosKillChildOnParentStopped() { } + #else /* diff --git a/source/os/src/osSleep.c b/source/os/src/osSleep.c index 724347b0bc..870467ceef 100644 --- a/source/os/src/osSleep.c +++ b/source/os/src/osSleep.c @@ -23,7 +23,7 @@ #endif void taosSsleep(int32_t s) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS Sleep(1000 * s); #else sleep(s); @@ -31,7 +31,7 @@ void taosSsleep(int32_t s) { } void taosMsleep(int32_t ms) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS Sleep(ms); #else usleep(ms * 1000); @@ -39,8 +39,15 @@ void taosMsleep(int32_t ms) { } void taosUsleep(int32_t us) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - nanosleep(1000 * us); +#ifdef WINDOWS + HANDLE timer; + LARGE_INTEGER interval; + interval.QuadPart = (10 * us); + + timer = CreateWaitableTimer(NULL, TRUE, NULL); + SetWaitableTimer(timer, &interval, 0, NULL, NULL, 0); + WaitForSingleObject(timer, INFINITE); + CloseHandle(timer); #else usleep(us); #endif diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index d79d7c3d58..0430f68a26 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -25,9 +25,6 @@ #include #include #include -#include -#include -#include "winsock2.h" #else #include #include @@ -68,7 +65,7 @@ int32_t taosSendto(TdSocketPtr pSocket, void *buf, int len, unsigned int flags, if (pSocket == NULL || pSocket->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return sendto(pSocket->fd, buf, len, flags, dest_addr, addrlen); #else return sendto(pSocket->fd, buf, len, flags, dest_addr, addrlen); @@ -78,7 +75,7 @@ int32_t taosWriteSocket(TdSocketPtr pSocket, void *buf, int len) { if (pSocket == NULL || pSocket->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return send(pSocket->fd, buf, len, 0); ; #else @@ -89,7 +86,7 @@ int32_t taosReadSocket(TdSocketPtr pSocket, void *buf, int len) { if (pSocket == NULL || pSocket->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return recv(pSocket->fd, buf, len, 0); ; #else @@ -97,14 +94,14 @@ int32_t taosReadSocket(TdSocketPtr pSocket, void *buf, int len) { #endif } -int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen) { +int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, int *addrLen) { if (pSocket == NULL || pSocket->fd < 0) { return -1; } return recvfrom(pSocket->fd, buf, len, flags, destAddr, addrLen); } int32_t taosCloseSocketNoCheck1(SocketFd fd) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return closesocket(fd); #else return close(fd); @@ -205,8 +202,8 @@ int32_t taosShutDownSocketServerRDWR(TdSocketServerPtr pSocketServer) { #endif } -void taosWinSocketInit1() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +void taosWinSocketInit() { +#ifdef WINDOWS static char flag = 0; if (flag == 0) { WORD wVersionRequested; @@ -223,7 +220,7 @@ int32_t taosSetNonblocking(TdSocketPtr pSocket, int32_t on) { if (pSocket == NULL || pSocket->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS u_long mode; if (on) { mode = 1; @@ -255,7 +252,7 @@ int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void if (pSocket == NULL || pSocket->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { return 0; } @@ -274,21 +271,21 @@ int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void return setsockopt(pSocket->fd, level, optname, optval, optlen); #else - return setsockopt(pSocket->fd, level, optname, optval, (socklen_t)optlen); + return setsockopt(pSocket->fd, level, optname, optval, (int)optlen); #endif } int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t *optlen) { if (pSocket == NULL || pSocket->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else - return getsockopt(pSocket->fd, level, optname, optval, (socklen_t *)optlen); + return getsockopt(pSocket->fd, level, optname, optval, (int *)optlen); #endif } uint32_t taosInetAddr(const char *ipAddr) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS uint32_t value; int32_t ret = inet_pton(AF_INET, ipAddr, &value); if (ret <= 0) { @@ -301,7 +298,7 @@ uint32_t taosInetAddr(const char *ipAddr) { #endif } const char *taosInetNtoa(struct in_addr ipInt) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS // not thread safe, only for debug usage while print log static char tmpDstStr[16]; return inet_ntop(AF_INET, &ipInt, tmpDstStr, INET6_ADDRSTRLEN); @@ -693,7 +690,7 @@ TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { } TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, - socklen_t *addrLen) { + int *addrLen) { if (pServerSocket == NULL || pServerSocket->fd < 0) { return NULL; } @@ -753,12 +750,12 @@ int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len } void taosBlockSIGPIPE() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #else sigset_t signal_mask; sigemptyset(&signal_mask); sigaddset(&signal_mask, SIGPIPE); - int32_t rc = taosThreadSigMask(SIG_BLOCK, &signal_mask, NULL); + int32_t rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL); if (rc != 0) { // printf("failed to block SIGPIPE"); } @@ -864,26 +861,26 @@ void tinet_ntoa(char *ipstr, uint32_t ip) { } void taosIgnSIGPIPE() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #else signal(SIGPIPE, SIG_IGN); #endif } void taosSetMaskSIGPIPE() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #else sigset_t signal_mask; sigemptyset(&signal_mask); sigaddset(&signal_mask, SIGPIPE); - int32_t rc = taosThreadSigMask(SIG_SETMASK, &signal_mask, NULL); + int32_t rc = pthread_sigmask(SIG_SETMASK, &signal_mask, NULL); if (rc != 0) { // printf("failed to setmask SIGPIPE"); } #endif } -int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, socklen_t *addrLen) { +int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, int *addrLen) { if (pSocket == NULL || pSocket->fd < 0) { return -1; } @@ -893,7 +890,7 @@ int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, sockle TdEpollPtr taosCreateEpoll(int32_t size) { EpollFd fd = -1; -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #else fd = epoll_create(size); #endif @@ -915,7 +912,7 @@ int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocke if (pEpoll == NULL || pEpoll->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #else code = epoll_ctl(pEpoll->fd, epollOperate, pSocket->fd, event); #endif @@ -926,7 +923,7 @@ int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxE if (pEpoll == NULL || pEpoll->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #else code = epoll_wait(pEpoll->fd, event, maxEvents, timeout); #endif diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 5adb564e52..ed596a051d 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -24,6 +24,55 @@ extern int wcwidth(wchar_t c); extern int wcswidth(const wchar_t *s, size_t n); +#ifdef WINDOWS +char *strsep(char **stringp, const char *delim) { + char * s; + const char *spanp; + int32_t c, sc; + char *tok; + if ((s = *stringp) == NULL) + return (NULL); + for (tok = s;;) { + c = *s++; + spanp = delim; + do { + if ((sc = *spanp++) == c) { + if (c == 0) + s = NULL; + else + s[-1] = 0; + *stringp = s; + return (tok); + } + } while (sc != 0); + } + /* NOTREACHED */ +} +/* Duplicate a string, up to at most size characters */ +char *strndup(const char *s, size_t size) { + size_t l; + char *s2; + l = strlen(s); + if (l > size) l=size; + s2 = malloc(l+1); + if (s2) { + strncpy(s2, s, l); + s2[l] = '\0'; + } + return s2; +} +/* Copy no more than N characters of SRC to DEST, returning the address of + the terminating '\0' in DEST, if any, or else DEST + N. */ +char *stpncpy (char *dest, const char *src, size_t n) { + size_t size = strnlen (src, n); + memcpy (dest, src, size); + dest += size; + if (size == n) + return dest; + return memset (dest, '\0', n - size); +} +#endif + int64_t taosStr2int64(const char *str) { char *endptr = NULL; return strtoll(str, &endptr, 10); diff --git a/source/os/src/osStrptime.c b/source/os/src/osStrptime.c index 99549f02cb..8d878577ea 100644 --- a/source/os/src/osStrptime.c +++ b/source/os/src/osStrptime.c @@ -38,7 +38,7 @@ // //#include "lukemftp.h" -// #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +// #ifdef WINDOWS // #include // #include @@ -100,7 +100,7 @@ // #endif // char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) { -// #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +// #ifdef WINDOWS // char c; // const char *bp; // size_t len = 0; diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 4ffbc13fb3..ccc302e646 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -17,7 +17,23 @@ #include "os.h" #include "taoserror.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#define PROCESS_ITEM 12 + +typedef struct { + uint64_t user; + uint64_t nice; + uint64_t system; + uint64_t idle; +} SysCpuInfo; + +typedef struct { + uint64_t utime; // user time + uint64_t stime; // kernel time + uint64_t cutime; // all user time + uint64_t cstime; // all dead time +} ProcCpuInfo; + +#ifdef WINDOWS /* * windows implementation @@ -92,22 +108,6 @@ LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) { #include #include -#define PROCESS_ITEM 12 - -typedef struct { - uint64_t user; - uint64_t nice; - uint64_t system; - uint64_t idle; -} SysCpuInfo; - -typedef struct { - uint64_t utime; // user time - uint64_t stime; // kernel time - uint64_t cutime; // all user time - uint64_t cstime; // all dead time -} ProcCpuInfo; - static pid_t tsProcId; static char tsSysNetFile[] = "/proc/net/dev"; static char tsSysCpuFile[] = "/proc/stat"; @@ -125,8 +125,12 @@ static void taosGetProcIOnfos() { snprintf(tsProcCpuFile, sizeof(tsProcCpuFile), "/proc/%d/stat", tsProcId); snprintf(tsProcIOFile, sizeof(tsProcIOFile), "/proc/%d/io", tsProcId); } +#endif static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { +#ifdef WINDOWS +#elif defined(_TD_DARWIN_64) +#else TdFilePtr pFile = taosOpenFile(tsSysCpuFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { return -1; @@ -145,10 +149,14 @@ static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { if (line != NULL) taosMemoryFreeClear(line); taosCloseFile(&pFile); +#endif return 0; } static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { +#ifdef WINDOWS +#elif defined(_TD_DARWIN_64) +#else TdFilePtr pFile = taosOpenFile(tsProcCpuFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { return -1; @@ -172,10 +180,10 @@ static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { if (line != NULL) taosMemoryFreeClear(line); taosCloseFile(&pFile); +#endif return 0; } -#endif bool taosCheckSystemIsSmallEnd() { union check { @@ -187,7 +195,7 @@ bool taosCheckSystemIsSmallEnd() { } void taosGetSystemInfo() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS taosGetCpuCores(&tsNumOfCores); taosGetTotalMemory(&tsTotalMemoryKB); @@ -210,7 +218,7 @@ void taosGetSystemInfo() { } int32_t taosGetEmail(char *email, int32_t maxLen) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #elif defined(_TD_DARWIN_64) const char *filepath = "/usr/local/taos/email"; @@ -241,7 +249,7 @@ int32_t taosGetEmail(char *email, int32_t maxLen) { } int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #elif defined(_TD_DARWIN_64) char *line = NULL; size_t size = 0; @@ -296,7 +304,7 @@ int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { } int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #elif defined(_TD_DARWIN_64) char *line = NULL; size_t size = 0; @@ -355,7 +363,7 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { } int32_t taosGetCpuCores(float *numOfCores) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS SYSTEM_INFO info; GetSystemInfo(&info); *numOfCores = info.dwNumberOfProcessors; @@ -399,7 +407,7 @@ void taosGetCpuUsage(double *cpu_system, double *cpu_engine) { } int32_t taosGetTotalMemory(int64_t *totalKB) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS MEMORYSTATUSEX memsStat; memsStat.dwLength = sizeof(memsStat); if (!GlobalMemoryStatusEx(&memsStat)) { @@ -417,7 +425,7 @@ int32_t taosGetTotalMemory(int64_t *totalKB) { } int32_t taosGetProcMemory(int64_t *usedKB) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS unsigned bytes_used = 0; #if defined(_WIN64) && defined(_MSC_VER) @@ -469,7 +477,7 @@ int32_t taosGetProcMemory(int64_t *usedKB) { } int32_t taosGetSysMemory(int64_t *usedKB) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS MEMORYSTATUSEX memsStat; memsStat.dwLength = sizeof(memsStat); if (!GlobalMemoryStatusEx(&memsStat)) { @@ -534,7 +542,7 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { } int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS IO_COUNTERS io_counter; if (GetProcessIoCounters(GetCurrentProcess(), &io_counter)) { if (rchars) *rchars = io_counter.ReadTransferCount; @@ -620,7 +628,7 @@ void taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, i } int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS *receive_bytes = 0; *transmit_bytes = 0; return 0; @@ -691,7 +699,7 @@ void taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes) { } void taosKillSystem() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS printf("function taosKillSystem, exit!"); exit(0); #elif defined(_TD_DARWIN_64) @@ -705,7 +713,7 @@ void taosKillSystem() { } int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS GUID guid; CoCreateGuid(&guid); @@ -741,7 +749,7 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { } char *taosGetCmdlineByPID(int pid) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return ""; #elif defined(_TD_DARWIN_64) static char cmdline[1024]; @@ -777,7 +785,7 @@ char *taosGetCmdlineByPID(int pid) { } void taosSetCoreDump(bool enable) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS SetUnhandledExceptionFilter(&FlCrashDump); #elif defined(_TD_DARWIN_64) #else @@ -857,7 +865,7 @@ void taosSetCoreDump(bool enable) { } SysNameInfo taosGetSysNameInfo() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #elif defined(_TD_DARWIN_64) SysNameInfo info = {0}; diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index 665f6370e1..cf3c95e658 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -183,16 +183,33 @@ void resetTerminalMode() { TdCmdPtr taosOpenCmd(const char *cmd) { if (cmd == NULL) return NULL; +#ifdef WINDOWS + return (TdCmdPtr)_popen(cmd, "r"); +#else return (TdCmdPtr)popen(cmd, "r"); +#endif } int64_t taosGetLineCmd(TdCmdPtr pCmd, char ** __restrict ptrBuf) { - if (pCmd == NULL) { + if (pCmd == NULL || ptrBuf == NULL ) { return -1; } - + if (*ptrBuf != NULL) { + taosMemoryFreeClear(*ptrBuf); + } +#ifdef WINDOWS + *ptrBuf = taosMemoryMalloc(1024); + if (*ptrBuf == NULL) return -1; + if (fgets(*ptrBuf, 1023, (FILE*)pCmd) == NULL) { + taosMemoryFreeClear(*ptrBuf); + return -1; + } + (*ptrBuf)[1023] = 0; + return strlen(*ptrBuf); +#else size_t len = 0; return getline(ptrBuf, &len, (FILE*)pCmd); +#endif } int32_t taosEOFCmd(TdCmdPtr pCmd) { @@ -206,7 +223,11 @@ int64_t taosCloseCmd(TdCmdPtr *ppCmd) { if (ppCmd == NULL || *ppCmd == NULL) { return 0; } +#ifdef WINDOWS + _pclose((FILE*)(*ppCmd)); +#else pclose((FILE*)(*ppCmd)); +#endif *ppCmd = NULL; return 0; } diff --git a/source/os/src/osThread.c b/source/os/src/osThread.c index 656871fd3c..a2778e44c5 100644 --- a/source/os/src/osThread.c +++ b/source/os/src/osThread.c @@ -294,7 +294,7 @@ int32_t taosThreadSetSpecific(TdThreadKey key, const void *value) { } int32_t taosThreadSpinDestroy(TdThreadSpinlock * lock) { -#ifndef __USE_XOPEN2K +#ifdef TD_USE_SPINLOCK_AS_MUTEX return pthread_mutex_destroy((pthread_mutex_t*)lock); #else return pthread_spin_destroy((pthread_spinlock_t*)lock); @@ -302,15 +302,16 @@ int32_t taosThreadSpinDestroy(TdThreadSpinlock * lock) { } int32_t taosThreadSpinInit(TdThreadSpinlock * lock, int32_t pshared) { -#ifndef __USE_XOPEN2K - return pthread_mutex_init((pthread_mutex_t*)lock, pshared); +#ifdef TD_USE_SPINLOCK_AS_MUTEX + assert(pshared == NULL); + return pthread_mutex_init((pthread_mutex_t*)lock, NULL); #else return pthread_spin_init((pthread_spinlock_t*)lock, pshared); #endif } int32_t taosThreadSpinLock(TdThreadSpinlock * lock) { -#ifndef __USE_XOPEN2K +#ifdef TD_USE_SPINLOCK_AS_MUTEX return pthread_mutex_lock((pthread_mutex_t*)lock); #else return pthread_spin_lock((pthread_spinlock_t*)lock); @@ -318,7 +319,7 @@ int32_t taosThreadSpinLock(TdThreadSpinlock * lock) { } int32_t taosThreadSpinTrylock(TdThreadSpinlock * lock) { -#ifndef __USE_XOPEN2K +#ifdef TD_USE_SPINLOCK_AS_MUTEX return pthread_mutex_trylock((pthread_mutex_t*)lock); #else return pthread_spin_trylock((pthread_spinlock_t*)lock); @@ -326,7 +327,7 @@ int32_t taosThreadSpinTrylock(TdThreadSpinlock * lock) { } int32_t taosThreadSpinUnlock(TdThreadSpinlock * lock) { -#ifndef __USE_XOPEN2K +#ifdef TD_USE_SPINLOCK_AS_MUTEX return pthread_mutex_unlock((pthread_mutex_t*)lock); #else return pthread_spin_unlock((pthread_spinlock_t*)lock); @@ -337,10 +338,6 @@ void taosThreadTestCancel(void) { return pthread_testcancel(); } -int32_t taosThreadSigMask(int32_t how, sigset_t const *set, sigset_t * oset) { - return pthread_sigmask(how, set, oset); -} - -int32_t taosThreadSigWait(const sigset_t * set, int32_t *sig) { - return sigwait(set, sig); +void taosThreadClear(TdThread *thread) { + memset(thread, 0, sizeof(TdThread)); } \ No newline at end of file diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index 9ea49b364e..5d6b4e7c2b 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -26,12 +26,11 @@ #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #include #include #include -#include //#define TM_YEAR_BASE 1970 //origin #define TM_YEAR_BASE 1900 //slguan /* @@ -85,13 +84,43 @@ static const char *am_pm[2] = { "AM", "PM" }; +#define BILLION (1E9) + +static BOOL g_first_time = 1; +static LARGE_INTEGER g_counts_per_sec; + +int clock_gettime(int dummy, struct timespec *ct) +{ + LARGE_INTEGER count; + + if (g_first_time) + { + g_first_time = 0; + + if (0 == QueryPerformanceFrequency(&g_counts_per_sec)) + { + g_counts_per_sec.QuadPart = 0; + } + } + + if ((NULL == ct) || (g_counts_per_sec.QuadPart <= 0) || + (0 == QueryPerformanceCounter(&count))) + { + return -1; + } + + ct->tv_sec = count.QuadPart / g_counts_per_sec.QuadPart; + ct->tv_nsec = ((count.QuadPart % g_counts_per_sec.QuadPart) * BILLION) / g_counts_per_sec.QuadPart; + + return 0; +} #else #include #endif char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS char c; const char *bp; size_t len = 0; @@ -391,7 +420,7 @@ char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) { } FORCE_INLINE int32_t taosGetTimeOfDay(struct timeval *tv) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS time_t t; t = taosGetTimestampSec(); SYSTEMTIME st; @@ -418,7 +447,7 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result) { if (result == NULL) { return localtime(timep); } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS localtime_s(result, timep); #else localtime_r(timep, result); @@ -427,3 +456,5 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result) { } int32_t taosGetTimestampSec() { return (int32_t)time(NULL); } + +int32_t taosClockGetTime(int clock_id, struct timespec *pTS) { return clock_gettime(clock_id, pTS); } \ No newline at end of file diff --git a/source/os/src/osTimer.c b/source/os/src/osTimer.c index e6e3ec7962..2992b29098 100644 --- a/source/os/src/osTimer.c +++ b/source/os/src/osTimer.c @@ -17,7 +17,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #include #include #include @@ -143,7 +143,7 @@ static void * taosProcessAlarmSignal(void *tharg) { #endif int taosInitTimer(void (*callback)(int), int ms) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS DWORD_PTR param = *((int64_t *)&callback); timerId = timeSetEvent(ms, 1, (LPTIMECALLBACK)taosWinOnTimer, param, TIME_PERIODIC); @@ -190,7 +190,7 @@ int taosInitTimer(void (*callback)(int), int ms) { } void taosUninitTimer() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS timeKillEvent(timerId); #elif defined(_TD_DARWIN_64) int r = 0; diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index dc23eaae1a..cbf20f02cd 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -17,7 +17,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #if (_WIN64) #include #include @@ -59,11 +59,12 @@ void taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8 buf[i] = inTimezoneStr[i]; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS char winStr[TD_LOCALE_LEN * 2]; sprintf(winStr, "TZ=%s", buf); putenv(winStr); tzset(); + /* * get CURRENT time zone. * system current time zone is affected by daylight saving time(DST) * @@ -116,7 +117,7 @@ void taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8 } void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS char *tz = getenv("TZ"); if (tz == NULL || strlen(tz) == 0) { strcpy(outTimezoneStr, "not configured"); diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index 8609c6eb21..8553d714fc 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( ) target_link_libraries( util - PRIVATE os + PUBLIC os PUBLIC lz4_static PUBLIC api cjson ) diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index a69f11f285..5f838b9af6 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "tcache.h" #include "taoserror.h" +#include "osThread.h" #include "tlog.h" #include "tutil.h" @@ -24,7 +25,7 @@ static TdThread cacheRefreshWorker = {0}; static TdThreadOnce cacheThreadInit = PTHREAD_ONCE_INIT; -static TdThreadMutex guard = PTHREAD_MUTEX_INITIALIZER; +static TdThreadMutex guard = TD_PTHREAD_MUTEX_INITIALIZER; static SArray *pCacheArrayList = NULL; static bool stopRefreshWorker = false; static bool refreshWorkerNormalStopped = false; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 9a0af24319..eea5c9fecc 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -139,7 +139,7 @@ static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { return -1; } - if (taosRealPath(fullDir, PATH_MAX) != 0) { + if (taosRealPath(fullDir, NULL, PATH_MAX) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); uError("failed to get realpath of dir:%s since %s", inputDir, terrstr()); return -1; diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 6d3a259079..d2b07875a3 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -135,7 +135,7 @@ static FORCE_INLINE void taosHashEntryRUnlock(const SHashObj *pHashObj, SHashEnt } static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { - int32_t len = MIN(length, HASH_MAX_CAPACITY); + int32_t len = (length < HASH_MAX_CAPACITY ? length : HASH_MAX_CAPACITY); int32_t i = 4; while (i < len) i = (i << 1u); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index c57883cb73..23fe2e8301 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#include "os.h" #include "tlog.h" #include "tutil.h" diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index ae7b3d6f1f..03c9c255b0 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -286,13 +286,13 @@ static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int16_t *pHea pQueue->head = headLen + bodyLen; } else if (remain < 8 + headLen) { memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, remain - 8); - memcpy(pHead + remain - 8, pQueue->pBuffer, headLen - (remain - 8)); + memcpy((char*)pHead + remain - 8, pQueue->pBuffer, headLen - (remain - 8)); memcpy(pBody, pQueue->pBuffer + headLen - (remain - 8), bodyLen); pQueue->head = headLen - (remain - 8) + bodyLen; } else if (remain < 8 + headLen + bodyLen) { memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, headLen); memcpy(pBody, pQueue->pBuffer + pQueue->head + 8 + headLen, remain - 8 - headLen); - memcpy(pBody + remain - 8 - headLen, pQueue->pBuffer, bodyLen - (remain - 8 - headLen)); + memcpy((char*)pBody + remain - 8 - headLen, pQueue->pBuffer, bodyLen - (remain - 8 - headLen)); pQueue->head = bodyLen - (remain - 8 - headLen); } else { memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, headLen); @@ -434,7 +434,7 @@ void taosProcStop(SProcObj *pProc) { } tsem_post(&pQueue->sem); taosThreadJoin(pProc->thread, NULL); - pProc->thread = 0; + taosThreadClear(&pProc->thread); } void taosProcCleanup(SProcObj *pProc) { diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 118fe58d2e..13637b8fe4 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -559,7 +559,7 @@ static FORCE_INLINE int32_t getSkipListNodeRandomHeight(SSkipList *pSkipList) { const uint32_t factor = 4; int32_t n = 1; -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS while ((taosRand() % factor) == 0 && n <= pSkipList->maxLevel) { #else while ((taosRandR(&(pSkipList->seed)) % factor) == 0 && n <= pSkipList->maxLevel) { diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index fecc58c236..34a59a7d48 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -132,7 +132,7 @@ static timer_map_t timerMap; static uintptr_t getNextTimerId() { uintptr_t id; do { - id = (uintptr_t)atomic_add_fetch_ptr((void **)&nextTimerId, 1); + id = (uintptr_t)atomic_add_fetch_ptr((void **)&nextTimerId, (void*)1); } while (id == 0); return id; } diff --git a/source/util/test/freelistTest.cpp b/source/util/test/freelistTest.cpp index 440e997042..a445a16ad3 100644 --- a/source/util/test/freelistTest.cpp +++ b/source/util/test/freelistTest.cpp @@ -14,4 +14,4 @@ TEST(TD_UTIL_FREELIST_TEST, simple_test) { } tFreeListClear(&fl); -} \ No newline at end of file +} diff --git a/source/util/test/procTest.cpp b/source/util/test/procTest.cpp index e05008e8e1..3c014369fb 100644 --- a/source/util/test/procTest.cpp +++ b/source/util/test/procTest.cpp @@ -63,19 +63,19 @@ TEST_F(UtilTesProc, 00_Init_Cleanup) { ASSERT_EQ(taosCreateShm(&shm, 1234, 1024 * 1024 * 2), 0); shm.size = 1023; - SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)NULL, - .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, - .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, - .childMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .childFreeBodyFp = (ProcFreeFp)taosMemoryMalloc, - .parentConsumeFp = (ProcConsumeFp)NULL, - .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, - .parentMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeBodyFp = (ProcFreeFp)taosMemoryMalloc, - .shm = shm, - .parent = &shm, - .name = "1234"}; + SProcCfg cfg = {(ProcConsumeFp)NULL, + (ProcMallocFp)taosAllocateQitem, + (ProcFreeFp)taosFreeQitem, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryMalloc, + (ProcConsumeFp)NULL, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryMalloc, + shm, + &shm, + "1234"}; SProcObj *proc = taosProcInit(&cfg); ASSERT_EQ(proc, nullptr); @@ -104,19 +104,19 @@ void ConsumeChild1(void *parent, void *pHead, int16_t headLen, void *pBody, int3 TEST_F(UtilTesProc, 01_Push_Pop_Child) { shm.size = 3000; ASSERT_EQ(taosCreateShm(&shm, 1235, shm.size), 0); - SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)ConsumeChild1, - .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, - .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, - .childMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .childFreeBodyFp = (ProcFreeFp)taosMemoryFree, - .parentConsumeFp = (ProcConsumeFp)NULL, - .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, - .parentMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeBodyFp = (ProcFreeFp)taosMemoryFree, - .shm = shm, - .parent = (void *)((int64_t)1235), - .name = "1235_c"}; + SProcCfg cfg = {(ProcConsumeFp)ConsumeChild1, + (ProcMallocFp)taosAllocateQitem, + (ProcFreeFp)taosFreeQitem, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcConsumeFp)NULL, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + shm, + (void *)((int64_t)1235), + "1235_c"}; SProcObj *cproc = taosProcInit(&cfg); ASSERT_NE(cproc, nullptr); @@ -162,19 +162,19 @@ void ConsumeParent1(void *parent, void *pHead, int16_t headLen, void *pBody, int TEST_F(UtilTesProc, 02_Push_Pop_Parent) { shm.size = 3000; ASSERT_EQ(taosCreateShm(&shm, 1236, shm.size), 0); - SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)NULL, - .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, - .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, - .childMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .childFreeBodyFp = (ProcFreeFp)taosMemoryFree, - .parentConsumeFp = (ProcConsumeFp)ConsumeParent1, - .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, - .parentMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeBodyFp = (ProcFreeFp)taosMemoryFree, - .shm = shm, - .parent = (void *)((int64_t)1236), - .name = "1236_c"}; + SProcCfg cfg = {(ProcConsumeFp)NULL, + (ProcMallocFp)taosAllocateQitem, + (ProcFreeFp)taosFreeQitem, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcConsumeFp)ConsumeParent1, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + shm, + (void *)((int64_t)1236), + "1236_c"}; SProcObj *cproc = taosProcInit(&cfg); ASSERT_NE(cproc, nullptr); @@ -216,19 +216,19 @@ TEST_F(UtilTesProc, 03_Handle) { // uDebugFlag = 207; shm.size = 3000; ASSERT_EQ(taosCreateShm(&shm, 1237, shm.size), 0); - SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)ConsumeChild3, - .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, - .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, - .childMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .childFreeBodyFp = (ProcFreeFp)taosMemoryFree, - .parentConsumeFp = (ProcConsumeFp)NULL, - .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, - .parentMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeBodyFp = (ProcFreeFp)taosMemoryFree, - .shm = shm, - .parent = (void *)((int64_t)1235), - .name = "1237_p"}; + SProcCfg cfg = {(ProcConsumeFp)ConsumeChild3, + (ProcMallocFp)taosAllocateQitem, + (ProcFreeFp)taosFreeQitem, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcConsumeFp)NULL, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + shm, + (void *)((int64_t)1235), + "1237_p"}; SProcObj *cproc = taosProcInit(&cfg); ASSERT_NE(cproc, nullptr); diff --git a/source/util/test/trefTest.c b/source/util/test/trefTest.c index 3174d57aef..89561e61fe 100644 --- a/source/util/test/trefTest.c +++ b/source/util/test/trefTest.c @@ -36,7 +36,7 @@ void *addRef(void *param) { for (int i=0; i < pSpace->steps; ++i) { printf("a"); - id = random() % pSpace->refNum; + id = taosRand() % pSpace->refNum; if (pSpace->rid[id] <= 0) { pSpace->p[id] = taosMemoryMalloc(128); pSpace->rid[id] = taosAddRef(pSpace->rsetId, pSpace->p[id]); @@ -53,7 +53,7 @@ void *removeRef(void *param) { for (int i=0; i < pSpace->steps; ++i) { printf("d"); - id = random() % pSpace->refNum; + id = taosRand() % pSpace->refNum; if (pSpace->rid[id] > 0) { code = taosRemoveRef(pSpace->rsetId, pSpace->rid[id]); if (code == 0) pSpace->rid[id] = 0; @@ -72,7 +72,7 @@ void *acquireRelease(void *param) { for (int i=0; i < pSpace->steps; ++i) { printf("a"); - id = random() % pSpace->refNum; + id = taosRand() % pSpace->refNum; void *p = taosAcquireRef(pSpace->rsetId, (int64_t) pSpace->p[id]); if (p) { taosUsleep(id % 5 + 1); diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 50b4a0800f..dc1a77c231 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -14,14 +14,13 @@ */ #include -#include #include #include #include #include #include #include -#include +// #include #include "taos.h" #include "taoserror.h" @@ -227,7 +226,7 @@ int64_t getDirectorySize(char* dir) { } } - taosCloseDir(pDir); + taosCloseDir(&pDir); return totalSize; } @@ -595,8 +594,8 @@ void printParaIntoFile() { taosOpenFile(g_stConfInfo.resultFileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_STREAM); if (NULL == pFile) { fprintf(stderr, "Failed to open %s for save result\n", g_stConfInfo.resultFileName); - exit - 1; - }; + exit(-1); + } g_fp = pFile; time_t tTime = taosGetTimestampSec(); diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index af7944eb27..f9ba0e418c 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -14,14 +14,13 @@ */ #include -#include #include #include #include #include #include #include -#include +// #include #include "taos.h" #include "taoserror.h" @@ -103,8 +102,8 @@ void initLogFile() { TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_STREAM); if (NULL == pFile) { fprintf(stderr, "Failed to open %s for save result\n", "./tmqlog.txt"); - exit -1; - }; + exit(-1); + } g_fp = pFile; } diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 36d2866fb5..12206ce290 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -29,7 +29,6 @@ #include "tutil.h" #include -#include /**************** Global variables ****************/ #ifdef _TD_POWER_ diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 1ee42358f6..db73f2fc5c 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -18,9 +18,11 @@ #include "tglobal.h" #include "tlog.h" +#ifndef WINDOWS #include #include #include +#endif #define OPT_ABORT 1 /* abort */ @@ -37,6 +39,7 @@ TdThread pid; static tsem_t cancelSem; extern void taos_init(); +#ifndef WINDOWS static struct argp_option options[] = { {"host", 'h', "HOST", 0, "TDengine server FQDN to connect. The default host is localhost."}, {"password", 'p', NULL, 0, "The password to use when connecting to the server."}, @@ -213,7 +216,10 @@ static void parse_args(int argc, char *argv[], SShellArguments *arguments) { } } +#endif void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { +#ifdef WINDOWS +#else static char verType[32] = {0}; sprintf(verType, "version: %s\n", version); @@ -233,9 +239,12 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { abort(); #endif } +#endif } int32_t shellReadCommand(TAOS *con, char *command) { +#ifdef WINDOWS +#else unsigned hist_counter = history.hend; char utf8_array[10] = "\0"; Command cmd; @@ -390,6 +399,7 @@ int32_t shellReadCommand(TAOS *con, char *command) { } } +#endif return 0; } @@ -435,6 +445,8 @@ void *shellLoopQuery(void *arg) { void get_history_path(char *_history) { snprintf(_history, TSDB_FILENAME_LEN, "%s/%s", getenv("HOME"), HISTORY_FILE); } void clearScreen(int ecmd_pos, int cursor_pos) { +#ifdef WINDOWS +#else struct winsize w; if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { // fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n"); @@ -453,9 +465,12 @@ void clearScreen(int ecmd_pos, int cursor_pos) { fprintf(stdout, "\033[2K"); } fflush(stdout); +#endif } void showOnScreen(Command *cmd) { +#ifdef WINDOWS +#else struct winsize w; if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { // fprintf(stderr, "No stream device\n"); @@ -526,6 +541,7 @@ void showOnScreen(Command *cmd) { positionCursor(cursor_x, DOWN); positionCursor(cursor_y, RIGHT); fflush(stdout); +#endif } void cleanup_handler(void *arg) { resetTerminalMode(); } From d537322e419868c33bb9b60d8156132bb1ce1dc0 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 22 Apr 2022 10:10:14 +0800 Subject: [PATCH 50/83] udf scalar api change --- source/libs/function/inc/tudf.h | 4 +- source/libs/function/src/tudf.c | 39 +++++++++-- source/libs/function/src/udfd.c | 108 ++++++++++++++--------------- source/libs/function/test/runUdf.c | 13 ++-- 4 files changed, 99 insertions(+), 65 deletions(-) diff --git a/source/libs/function/inc/tudf.h b/source/libs/function/inc/tudf.h index b5c839c811..5f4f96c4cc 100644 --- a/source/libs/function/inc/tudf.h +++ b/source/libs/function/inc/tudf.h @@ -21,6 +21,7 @@ #include #include "tmsg.h" #include "tcommon.h" +#include "function.h" #ifdef __cplusplus extern "C" { @@ -118,8 +119,7 @@ int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfIn int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf); // input: block // output: resultData -int32_t callUdfScalaProcess(UdfcFuncHandle handle, SSDataBlock *block, SSDataBlock *resultData); - +int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t numOfCols, SScalarParam *output); /** * tearn down udf * @param handle diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index e31a860e85..317339af04 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -14,6 +14,7 @@ */ #include "uv.h" #include "os.h" +#include "fnLog.h" #include "tudf.h" #include "tudfInt.h" #include "tarray.h" @@ -557,6 +558,34 @@ int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) { return 0; } +int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SSDataBlock *output) { + output->info.rows = input->numOfRows; + output->info.numOfCols = numOfCols; + bool hasVarCol = false; + for (int32_t i = 0; i < numOfCols; ++i) { + if (IS_VAR_DATA_TYPE((input+i)->columnData->info.type)) { + hasVarCol = true; + break; + } + } + output->info.hasVarCol = hasVarCol; + + //TODO: free the array output->pDataBlock + output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + taosArrayPush(output->pDataBlock, input->columnData); + return 0; +} + +int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output) { + if (input->info.numOfCols != 1) { + fnError("scalar function only support one column"); + return -1; + } + output->numOfRows = input->info.rows; + //TODO: memory + output->columnData = taosArrayGet(input->pDataBlock, 0); + return 0; +} void onUdfcPipeClose(uv_handle_t *handle) { SClientUvConn *conn = handle->data; @@ -1108,11 +1137,13 @@ int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfIn return err; } -// input: block -// output: resultData -int32_t callUdfScalaProcess(UdfcFuncHandle handle, SSDataBlock *block, SSDataBlock *resultData) { +int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t numOfCols, SScalarParam* output) { int8_t callType = TSDB_UDF_CALL_SCALA_PROC; - int32_t err = callUdf(handle, callType, block, NULL, NULL, resultData, NULL); + SSDataBlock inputBlock = {0}; + convertScalarParamToDataBlock(input, numOfCols, &inputBlock); + SSDataBlock resultBlock = {0}; + int32_t err = callUdf(handle, callType, &inputBlock, NULL, NULL, &resultBlock, NULL); + convertDataBlockToScalarParm(&resultBlock, output); return err; } diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index d6e7a43666..5f7532da87 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -26,15 +26,15 @@ #include "trpc.h" typedef struct SUdfdContext { - uv_loop_t *loop; - uv_pipe_t ctrlPipe; + uv_loop_t *loop; + uv_pipe_t ctrlPipe; uv_signal_t intrSignal; - char listenPipeName[UDF_LISTEN_PIPE_NAME_LEN]; - uv_pipe_t listeningPipe; - void *clientRpc; + char listenPipeName[UDF_LISTEN_PIPE_NAME_LEN]; + uv_pipe_t listeningPipe; + void *clientRpc; uv_mutex_t udfsMutex; - SHashObj* udfsHash; + SHashObj *udfsHash; bool printVersion; } SUdfdContext; @@ -55,22 +55,17 @@ typedef struct SUvUdfWork { uv_buf_t output; } SUvUdfWork; -typedef enum { - UDF_STATE_INIT = 0, - UDF_STATE_LOADING, - UDF_STATE_READY, - UDF_STATE_UNLOADING -} EUdfState; +typedef enum { UDF_STATE_INIT = 0, UDF_STATE_LOADING, UDF_STATE_READY, UDF_STATE_UNLOADING } EUdfState; typedef struct SUdf { - int32_t refCount; - EUdfState state; + int32_t refCount; + EUdfState state; uv_mutex_t lock; - uv_cond_t condReady; + uv_cond_t condReady; char name[16]; int8_t type; - char path[PATH_MAX]; + char path[PATH_MAX]; uv_lib_t lib; TUdfScalarProcFunc scalarProcFunc; @@ -83,24 +78,28 @@ typedef struct SUdfcFuncHandle { SUdf *udf; } SUdfcFuncHandle; -int32_t udfdLoadUdf(char* udfName, SUdf* udf) { - strcpy(udf->name, udfName); +int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName, SUdf *udf); - int err = uv_dlopen(udf->path, &udf->lib); - if (err != 0) { - fnError("can not load library %s. error: %s", udf->path, uv_strerror(err)); - // TODO set error - } - //TODO: find all the functions - char normalFuncName[TSDB_FUNC_NAME_LEN] = {0}; - strcpy(normalFuncName, udfName); - uv_dlsym(&udf->lib, normalFuncName, (void **)(&udf->scalarProcFunc)); - char freeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; - char *freeSuffix = "_free"; - strncpy(freeFuncName, normalFuncName, strlen(normalFuncName)); - strncat(freeFuncName, freeSuffix, strlen(freeSuffix)); - uv_dlsym(&udf->lib, freeFuncName, (void **)(&udf->freeUdfColumn)); - return 0; +int32_t udfdLoadUdf(char *udfName, SEpSet *pEpSet, SUdf *udf) { + strcpy(udf->name, udfName); + + udfdFillUdfInfoFromMNode(global.clientRpc, pEpSet, udf->name, udf); + + int err = uv_dlopen(udf->path, &udf->lib); + if (err != 0) { + fnError("can not load library %s. error: %s", udf->path, uv_strerror(err)); + // TODO set error + } + // TODO: find all the functions + char normalFuncName[TSDB_FUNC_NAME_LEN] = {0}; + strcpy(normalFuncName, udfName); + uv_dlsym(&udf->lib, normalFuncName, (void **)(&udf->scalarProcFunc)); + char freeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; + char *freeSuffix = "_free"; + strncpy(freeFuncName, normalFuncName, strlen(normalFuncName)); + strncat(freeFuncName, freeSuffix, strlen(freeSuffix)); + uv_dlsym(&udf->lib, freeFuncName, (void **)(&udf->freeUdfColumn)); + return 0; } void udfdProcessRequest(uv_work_t *req) { @@ -110,13 +109,13 @@ void udfdProcessRequest(uv_work_t *req) { switch (request.type) { case UDF_TASK_SETUP: { - //TODO: tracable id from client. connect, setup, call, teardown - fnInfo("%"PRId64" setup request. udf name: %s", request.seqNum, request.setup.udfName); + // TODO: tracable id from client. connect, setup, call, teardown + fnInfo("%" PRId64 " setup request. udf name: %s", request.seqNum, request.setup.udfName); SUdfSetupRequest *setup = &request.setup; - SUdf* udf = NULL; + SUdf *udf = NULL; uv_mutex_lock(&global.udfsMutex); - SUdf** udfInHash = taosHashGet(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN); + SUdf **udfInHash = taosHashGet(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN); if (*udfInHash) { ++(*udfInHash)->refCount; udf = *udfInHash; @@ -136,7 +135,7 @@ void udfdProcessRequest(uv_work_t *req) { uv_mutex_lock(&udf->lock); if (udf->state == UDF_STATE_INIT) { udf->state = UDF_STATE_LOADING; - udfdLoadUdf(setup->udfName, udf); + udfdLoadUdf(setup->udfName, &setup->epSet, udf); udf->state = UDF_STATE_READY; uv_cond_broadcast(&udf->condReady); uv_mutex_unlock(&udf->lock); @@ -168,8 +167,9 @@ void udfdProcessRequest(uv_work_t *req) { case UDF_TASK_CALL: { SUdfCallRequest *call = &request.call; - fnDebug("%"PRId64 "call request. call type %d, handle: %"PRIx64, request.seqNum, call->callType, call->udfHandle); - SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(call->udfHandle); + fnDebug("%" PRId64 "call request. call type %d, handle: %" PRIx64, request.seqNum, call->callType, + call->udfHandle); + SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(call->udfHandle); SUdf *udf = handle->udf; SUdfDataBlock input = {0}; @@ -206,10 +206,10 @@ void udfdProcessRequest(uv_work_t *req) { } case UDF_TASK_TEARDOWN: { SUdfTeardownRequest *teardown = &request.teardown; - fnInfo("teardown. %"PRId64"handle:%"PRIx64, request.seqNum, teardown->udfHandle) - SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(teardown->udfHandle); - SUdf *udf = handle->udf; - bool unloadUdf = false; + fnInfo("teardown. %" PRId64 "handle:%" PRIx64, request.seqNum, teardown->udfHandle) SUdfcFuncHandle *handle = + (SUdfcFuncHandle *)(teardown->udfHandle); + SUdf *udf = handle->udf; + bool unloadUdf = false; uv_mutex_lock(&global.udfsMutex); udf->refCount--; if (udf->refCount == 0) { @@ -250,7 +250,7 @@ void udfdProcessRequest(uv_work_t *req) { void udfdOnWrite(uv_write_t *req, int status) { SUvUdfWork *work = (SUvUdfWork *)req->data; if (status < 0) { - //TODO:log error and process it. + // TODO:log error and process it. } fnDebug("send response. length:%zu, status: %s", work->output.len, uv_err_name(status)); taosMemoryFree(work->output.base); @@ -393,7 +393,7 @@ void udfdIntrSignalHandler(uv_signal_t *handle, int signum) { void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { return; } -int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char* udfName, SUdf* udf) { +int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName, SUdf *udf) { SRetrieveFuncReq retrieveReq = {0}; retrieveReq.numOfFuncs = 1; retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); @@ -505,7 +505,7 @@ void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *bu void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf) { if (nread < 0) { fnError("udfd ctrl pipe read error. %s", uv_err_name(nread)); - uv_close((uv_handle_t*)q, NULL); + uv_close((uv_handle_t *)q, NULL); uv_stop(global.loop); return; } @@ -515,13 +515,13 @@ void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf) { static int32_t removeListeningPipe() { uv_fs_t req; - int err = uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL); + int err = uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL); uv_fs_req_cleanup(&req); return err; } static int32_t udfdUvInit() { - uv_loop_t* loop = taosMemoryMalloc(sizeof(uv_loop_t)); + uv_loop_t *loop = taosMemoryMalloc(sizeof(uv_loop_t)); if (loop) { uv_loop_init(loop); } @@ -529,10 +529,10 @@ static int32_t udfdUvInit() { uv_pipe_init(global.loop, &global.ctrlPipe, 1); uv_pipe_open(&global.ctrlPipe, 0); - uv_read_start((uv_stream_t*)&global.ctrlPipe, udfdCtrlAllocBufCb, udfdCtrlReadCb); + uv_read_start((uv_stream_t *)&global.ctrlPipe, udfdCtrlAllocBufCb, udfdCtrlReadCb); - char dnodeId[8] = {0}; - size_t dnodeIdSize; + char dnodeId[8] = {0}; + size_t dnodeIdSize; int32_t err = uv_os_getenv("DNODE_ID", dnodeId, &dnodeIdSize); if (err != 0) { dnodeId[0] = '1'; @@ -567,7 +567,7 @@ static int32_t udfdRun() { global.udfsHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); uv_mutex_init(&global.udfsMutex); - //TOOD: client rpc to fetch udf function info from mnode + // TOOD: client rpc to fetch udf function info from mnode if (udfdOpenClientRpc() != 0) { fnError("open rpc connection to mnode failure"); return -1; @@ -589,7 +589,7 @@ static int32_t udfdRun() { return code; } -int main(int argc, char* argv[]) { +int main(int argc, char *argv[]) { if (!taosCheckSystemIsSmallEnd()) { printf("failed to start since on non-small-end machines\n"); return -1; diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c index fb9c3c678a..0727a4a1d2 100644 --- a/source/libs/function/test/runUdf.c +++ b/source/libs/function/test/runUdf.c @@ -44,12 +44,15 @@ int main(int argc, char *argv[]) { } taosArrayPush(pBlock->pDataBlock, &colInfo); } - - SSDataBlock output = {0}; - callUdfScalaProcess(handle, pBlock, &output); - SColumnInfoData *col = taosArrayGet(output.pDataBlock, 0); - for (int32_t i = 0; i < output.info.rows; ++i) { + SScalarParam input = {0}; + input.numOfRows = pBlock->info.rows; + input.columnData = taosArrayGet(pBlock->pDataBlock, 0); + SScalarParam output = {0}; + callUdfScalarFunc(handle, &input, 1 , &output); + + SColumnInfoData *col = output.columnData; + for (int32_t i = 0; i < output.numOfRows; ++i) { fprintf(stderr, "%d\t%d\n" , i, *(int32_t*)(col->pData + i *sizeof(int32_t))); } teardownUdf(handle); From c80da6133ad68633b355574e6119696f0466da0a Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Fri, 22 Apr 2022 10:21:39 +0800 Subject: [PATCH 51/83] fix(os): windows compile 3.0. --- source/os/src/osProc.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/os/src/osProc.c b/source/os/src/osProc.c index ae1b6c9e1f..f92a3b3783 100644 --- a/source/os/src/osProc.c +++ b/source/os/src/osProc.c @@ -17,8 +17,6 @@ #define _DEFAULT_SOURCE #include "os.h" -char *tsProcPath = NULL; - int32_t taosNewProc(char **args) { #ifdef WINDOWS return 0; From d51336384fbda14e2ec0acd3363185bcafa1ffa7 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 22 Apr 2022 10:55:17 +0800 Subject: [PATCH 52/83] feat(tmq): add db subscribe --- include/client/taos.h | 11 - source/client/src/tmq.c | 270 +----------------------- source/dnode/vnode/inc/vnode.h | 3 +- source/dnode/vnode/src/inc/tq.h | 40 ++-- source/dnode/vnode/src/tq/tq.c | 146 ++++++++----- source/dnode/vnode/src/tq/tqRead.c | 5 +- source/dnode/vnode/src/vnd/vnodeOpen.c | 2 +- source/libs/executor/src/scanoperator.c | 3 +- 8 files changed, 123 insertions(+), 357 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index fc71a2dad2..3d139ce6d2 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -239,7 +239,6 @@ typedef struct tmq_topic_vgroup_list_t tmq_topic_vgroup_list_t; typedef struct tmq_conf_t tmq_conf_t; typedef struct tmq_list_t tmq_list_t; -// typedef struct tmq_message_t tmq_message_t; typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *)); @@ -285,12 +284,6 @@ DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf); DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb); -#if 0 -// temporary used function for demo only -void tmqShowMsg(tmq_message_t *tmq_message); -int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message); -#endif - /* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */ DLL_EXPORT char *tmq_get_topic_name(TAOS_RES *res); @@ -301,12 +294,8 @@ DLL_EXPORT char *tmq_get_block_table_name(TAOS_RES *res); #endif #if 0 -DLL_EXPORT TAOS_ROW tmq_get_row(tmq_message_t *message); DLL_EXPORT int64_t tmq_get_request_offset(tmq_message_t *message); DLL_EXPORT int64_t tmq_get_response_offset(tmq_message_t *message); -DLL_EXPORT TAOS_FIELD *tmq_get_fields(tmq_t *tmq, const char *topic); -DLL_EXPORT int32_t tmq_field_count(tmq_t *tmq, const char *topic); -DLL_EXPORT void tmq_message_destroy(TAOS_RES *res); #endif /* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */ #if 0 diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 8a3990e2c3..51e422f5c2 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -24,16 +24,6 @@ #include "tqueue.h" #include "tref.h" -#if 0 -struct tmq_message_t { - SMqPollRsp msg; - char* topic; - SArray* res; // SArray - int32_t vgId; - int32_t resIter; -}; -#endif - typedef struct { int8_t tmqRspType; int32_t epoch; @@ -770,105 +760,12 @@ _return: } #endif -static char* formatTimestamp(char* buf, int64_t val, int precision) { - time_t tt; - int32_t ms = 0; - if (precision == TSDB_TIME_PRECISION_NANO) { - tt = (time_t)(val / 1000000000); - ms = val % 1000000000; - } else if (precision == TSDB_TIME_PRECISION_MICRO) { - tt = (time_t)(val / 1000000); - ms = val % 1000000; - } else { - tt = (time_t)(val / 1000); - ms = val % 1000; - } - - /* comment out as it make testcases like select_with_tags.sim fail. - but in windows, this may cause the call to localtime crash if tt < 0, - need to find a better solution. - if (tt < 0) { - tt = 0; - } - */ - -#ifdef WINDOWS - if (tt < 0) tt = 0; -#endif - if (tt <= 0 && ms < 0) { - tt--; - if (precision == TSDB_TIME_PRECISION_NANO) { - ms += 1000000000; - } else if (precision == TSDB_TIME_PRECISION_MICRO) { - ms += 1000000; - } else { - ms += 1000; - } - } - - struct tm* ptm = taosLocalTime(&tt, NULL); - size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); - - if (precision == TSDB_TIME_PRECISION_NANO) { - sprintf(buf + pos, ".%09d", ms); - } else if (precision == TSDB_TIME_PRECISION_MICRO) { - sprintf(buf + pos, ".%06d", ms); - } else { - sprintf(buf + pos, ".%03d", ms); - } - - return buf; -} #if 0 int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) { if (tmq_message == NULL) return 0; SMqPollRsp* pRsp = &tmq_message->msg; return pRsp->skipLogNum; } - -void tmqShowMsg(tmq_message_t* tmq_message) { - if (tmq_message == NULL) return; - - static bool noPrintSchema; - char pBuf[128]; - SMqPollRsp* pRsp = &tmq_message->msg; - int32_t colNum = 2; - if (!noPrintSchema) { - printf("|"); - for (int32_t i = 0; i < colNum; i++) { - if (i == 0) - printf(" %25s |", pRsp->schema->pSchema[i].name); - else - printf(" %15s |", pRsp->schema->pSchema[i].name); - } - printf("\n"); - printf("===============================================\n"); - noPrintSchema = true; - } - int32_t sz = taosArrayGetSize(pRsp->pBlockData); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(pRsp->pBlockData, i); - int32_t rows = pDataBlock->info.rows; - for (int32_t j = 0; j < rows; j++) { - printf("|"); - for (int32_t k = 0; k < colNum; k++) { - SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); - switch (pColInfoData->info.type) { - case TSDB_DATA_TYPE_TIMESTAMP: - formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); - printf(" %25s |", pBuf); - break; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - printf(" %15u |", *(uint32_t*)var); - break; - } - } - printf("\n"); - } - } -} #endif int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { @@ -1049,7 +946,6 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { } tDeleteSMqCMGetSubEpRsp(&rsp); } else { - /*SMqCMGetSubEpRsp* pRsp = taosAllocateQitem(sizeof(SMqCMGetSubEpRsp));*/ SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper)); if (pWrapper == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1208,7 +1104,6 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) { pRspObj->resIter = -1; memcpy(&pRspObj->rsp, &pWrapper->msg, sizeof(SMqDataBlkRsp)); - /*SRetrieveTableRsp* pRetrieve = taosArrayGetP(pWrapper->msg.blockData, 0);*/ pRspObj->resInfo.totalRows = 0; pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols); @@ -1355,31 +1250,6 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) { } } -#if 0 -tmq_message_t* tmq_consumer_poll_v1(tmq_t* tmq, int64_t blocking_time) { - tmq_message_t* rspMsg = NULL; - int64_t startTime = taosGetTimestampMs(); - - int64_t status = atomic_load_64(&tmq->status); - tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT); - - while (1) { - rspMsg = tmqSyncPollImpl(tmq, blocking_time); - if (rspMsg && rspMsg->consumeRsp.numOfTopics) { - return rspMsg; - } - - if (blocking_time != 0) { - int64_t endTime = taosGetTimestampMs(); - if (endTime - startTime > blocking_time) { - return NULL; - } - } else - return NULL; - } -} -#endif - TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { SMqRspObj* rspObj; int64_t startTime = taosGetTimestampMs(); @@ -1417,137 +1287,10 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { } } -#if 0 - - if (blocking_time <= 0) blocking_time = 1; - if (blocking_time > 1000) blocking_time = 1000; - /*blocking_time = 1;*/ - - if (taosArrayGetSize(tmq->clientTopics) == 0) { - tscDebug("consumer:%ld poll but not assigned", tmq->consumerId); - /*printf("over1\n");*/ - taosMsleep(blocking_time); - return NULL; - } - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx); - if (taosArrayGetSize(pTopic->vgs) == 0) { - /*printf("over2\n");*/ - taosMsleep(blocking_time); - return NULL; - } - - tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics); - int32_t beginVgIdx = pTopic->nextVgIdx; - while (1) { - pTopic->nextVgIdx = (pTopic->nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs); - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); - /*printf("consume vg %d, offset %ld\n", pVg->vgId, pVg->currentOffset);*/ - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, pTopic, pVg); - if (pReq == NULL) { - ASSERT(false); - taosMsleep(blocking_time); - return NULL; - } - - SMqPollCbParam* param = taosMemoryMalloc(sizeof(SMqPollCbParam)); - if (param == NULL) { - ASSERT(false); - taosMsleep(blocking_time); - return NULL; - } - param->tmq = tmq; - param->retMsg = &tmq_message; - param->pVg = pVg; - tsem_init(¶m->rspSem, 0, 0); - - SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); - pRequest->body.requestMsg = (SDataBuf){ - .pData = pReq, - .len = sizeof(SMqConsumeReq), - .handle = NULL, - }; - - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - sendInfo->requestObjRefId = 0; - sendInfo->param = param; - sendInfo->fp = tmqPollCb; - - /*printf("req offset: %ld\n", pReq->offset);*/ - - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); - tmq->pollCnt++; - - tsem_wait(¶m->rspSem); - tsem_destroy(¶m->rspSem); - taosMemoryFree(param); - - if (tmq_message == NULL) { - if (beginVgIdx == pTopic->nextVgIdx) { - taosMsleep(blocking_time); - } else { - continue; - } - } - - return tmq_message; - } - - /*tsem_wait(&pRequest->body.rspSem);*/ - - /*if (body != NULL) {*/ - /*destroySendMsgInfo(body);*/ - /*}*/ - - /*if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {*/ - /*pRequest->code = terrno;*/ - /*}*/ - - /*return pRequest;*/ +tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { + // TODO + return TMQ_RESP_ERR__SUCCESS; } -#endif - -#if 0 -tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) { - if (tmq_topic_vgroup_list != NULL) { - // TODO - } - - // TODO: change semaphore to gate - for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, pTopic, pVg); - - SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); - pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; - SMqCommitCbParam* pParam = taosMemoryMalloc(sizeof(SMqCommitCbParam)); - if (pParam == NULL) { - continue; - } - pParam->tmq = tmq; - pParam->pVg = pVg; - pParam->async = async; - if (!async) tsem_init(&pParam->rspSem, 0, 0); - - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - sendInfo->requestObjRefId = 0; - sendInfo->param = pParam; - sendInfo->fp = tmqCommitCb; - - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); - - if (!async) tsem_wait(&pParam->rspSem); - } - } - - return 0; -} -#endif - -tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { return TMQ_RESP_ERR__SUCCESS; } const char* tmq_err2str(tmq_resp_err_t err) { if (err == TMQ_RESP_ERR__SUCCESS) { @@ -1573,10 +1316,3 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) { return -1; } } - -void tmq_message_destroy(TAOS_RES* res) { - if (res == NULL) return; - if (TD_RES_TMQ(res)) { - SMqRspObj* pRspObj = (SMqRspObj*)res; - } -} diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 2b52d333da..51ee00d823 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -109,7 +109,8 @@ int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList int tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList); int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); bool tqNextDataBlock(STqReadHandle *pHandle); -int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, int32_t *pNumOfRows); +int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, int32_t *pNumOfRows, + int32_t *pNumOfCols); // need to reposition diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 5e074eeb12..6f929ce829 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -88,22 +88,12 @@ struct STqReadHandle { SSubmitMsgIter msgIter; SSubmitBlkIter blkIter; SMeta* pVnodeMeta; - SArray* pColIdList; // SArray + SArray* pColIdList; // SArray int32_t sver; SSchemaWrapper* pSchemaWrapper; STSchema* pSchema; }; -typedef struct { - int8_t type; - int8_t reserved[7]; - union { - void* data; - int64_t wmTs; - int64_t checkpointId; - }; -} STqStreamToken; - typedef struct { int16_t ver; int16_t action; @@ -155,24 +145,26 @@ typedef struct { char subKey[TSDB_SUBSCRIBE_KEY_LEN]; int64_t consumerId; int32_t epoch; + int8_t subType; + int8_t withTbName; + int8_t withSchema; + int8_t withTag; + int8_t withTagSchema; char* qmsg; // SRWLatch lock; - SWalReadHandle* pReadHandle; + SWalReadHandle* pWalReader; // number should be identical to fetch thread num - qTaskInfo_t task[4]; + STqReadHandle* pStreamReader[4]; + qTaskInfo_t task[4]; } STqExec; struct STQ { - // the collection of groups - // the handle of meta kvstore - bool writeTrigger; - char* path; - STqMetaStore* tqMeta; - SHashObj* tqMetaNew; // subKey -> tqExec - SHashObj* pStreamTasks; - SVnode* pVnode; - SWal* pWal; - SMeta* pVnodeMeta; + char* path; + // STqMetaStore* tqMeta; + SHashObj* execs; // subKey -> tqExec + SHashObj* pStreamTasks; + SVnode* pVnode; + SWal* pWal; }; typedef struct { @@ -252,7 +244,7 @@ int tqInit(); void tqCleanUp(); // open in each vnode -STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pMeta, SMemAllocatorFactory* allocFac); +STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal); void tqClose(STQ*); // required by vnode int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 510dd32459..30f75218b4 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -19,7 +19,7 @@ int32_t tqInit() { return tqPushMgrInit(); } void tqCleanUp() { tqPushMgrCleanUp(); } -STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, SMemAllocatorFactory* allocFac) { +STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { STQ* pTq = taosMemoryMalloc(sizeof(STQ)); if (pTq == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; @@ -28,15 +28,16 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, SMe pTq->path = strdup(path); pTq->pVnode = pVnode; pTq->pWal = pWal; - pTq->pVnodeMeta = pVnodeMeta; +#if 0 pTq->tqMeta = tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, (FTqDelete)taosMemoryFree, 0); if (pTq->tqMeta == NULL) { taosMemoryFree(pTq); return NULL; } +#endif - pTq->tqMetaNew = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); + pTq->execs = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); pTq->pStreamTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); @@ -104,7 +105,11 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t versi return 0; } -int tqCommit(STQ* pTq) { return tqStorePersist(pTq->tqMeta); } +int tqCommit(STQ* pTq) { + // do nothing + /*return tqStorePersist(pTq->tqMeta);*/ + return 0; +} int32_t tqGetTopicHandleSize(const STqTopic* pTopic) { return strlen(pTopic->topicName) + strlen(pTopic->sql) + strlen(pTopic->physicalPlan) + strlen(pTopic->qmsg) + @@ -219,10 +224,10 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu } for (int j = 0; j < TQ_BUFFER_SIZE; j++) { pTopic->buffer.output[j].status = 0; - STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta); + STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); SReadHandle handle = { .reader = pReadHandle, - .meta = pTq->pVnodeMeta, + .meta = pTq->pVnode->pMeta, }; pTopic->buffer.output[j].pReadHandle = pReadHandle; pTopic->buffer.output[j].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle); @@ -238,6 +243,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { int32_t reqEpoch = pReq->epoch; int64_t fetchOffset; + // get offset to fetch message if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { fetchOffset = walGetFirstVer(pTq->pWal); } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) { @@ -249,7 +255,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch, TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset); - STqExec* pExec = taosHashGet(pTq->tqMetaNew, pReq->subKey, strlen(pReq->subKey)); + STqExec* pExec = taosHashGet(pTq->execs, pReq->subKey, strlen(pReq->subKey)); ASSERT(pExec); int32_t consumerEpoch = atomic_load_32(&pExec->epoch); @@ -271,7 +277,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } SWalReadHead* pHead; - if (walReadWithHandle_s(pExec->pReadHandle, fetchOffset, &pHead) < 0) { + if (walReadWithHandle_s(pExec->pWalReader, fetchOffset, &pHead) < 0) { // TODO: no more log, set timer to wait blocking time // if data inserted during waiting, launch query and // response to user @@ -285,41 +291,73 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { if (pHead->msgType == TDMT_VND_SUBMIT) { SSubmitReq* pCont = (SSubmitReq*)&pHead->body; - qTaskInfo_t task = pExec->task[workerId]; - ASSERT(task); - qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK); - while (1) { - SSDataBlock* pDataBlock = NULL; - uint64_t ts = 0; - if (qExecTask(task, &pDataBlock, &ts) < 0) { - ASSERT(0); + if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { + qTaskInfo_t task = pExec->task[workerId]; + ASSERT(task); + qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK); + while (1) { + SSDataBlock* pDataBlock = NULL; + uint64_t ts = 0; + if (qExecTask(task, &pDataBlock, &ts) < 0) { + ASSERT(0); + } + if (pDataBlock == NULL) break; + + ASSERT(pDataBlock->info.rows != 0); + ASSERT(pDataBlock->info.numOfCols != 0); + + int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pDataBlock); + void* buf = taosMemoryCalloc(1, dataStrLen); + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf; + pRetrieve->useconds = ts; + pRetrieve->precision = TSDB_DEFAULT_PRECISION; + pRetrieve->compressed = 0; + pRetrieve->completed = 1; + pRetrieve->numOfRows = htonl(pDataBlock->info.rows); + + // TODO enable compress + int32_t actualLen = 0; + blockCompressEncode(pDataBlock, pRetrieve->data, &actualLen, pDataBlock->info.numOfCols, false); + actualLen += sizeof(SRetrieveTableRsp); + ASSERT(actualLen <= dataStrLen); + taosArrayPush(rsp.blockDataLen, &actualLen); + taosArrayPush(rsp.blockData, &buf); + rsp.blockNum++; } - if (pDataBlock == NULL) break; + } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { + STqReadHandle* pReader = pExec->pStreamReader[workerId]; + tqReadHandleSetMsg(pReader, pCont, 0); + while (tqNextDataBlock(pReader)) { + SSDataBlock block = {0}; + if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.rows, + &block.info.numOfCols) < 0) { + ASSERT(0); + } + int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(&block); + void* buf = taosMemoryCalloc(1, dataStrLen); + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf; + /*pRetrieve->useconds = 0;*/ + pRetrieve->precision = TSDB_DEFAULT_PRECISION; + pRetrieve->compressed = 0; + pRetrieve->completed = 1; + pRetrieve->numOfRows = htonl(block.info.rows); - ASSERT(pDataBlock->info.rows != 0); - ASSERT(pDataBlock->info.numOfCols != 0); - - int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pDataBlock); - void* buf = taosMemoryCalloc(1, dataStrLen); - SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf; - pRetrieve->useconds = ts; - pRetrieve->precision = TSDB_DEFAULT_PRECISION; - pRetrieve->compressed = 0; - pRetrieve->completed = 1; - pRetrieve->numOfRows = htonl(pDataBlock->info.rows); - - // TODO enable compress - int32_t actualLen = 0; - blockCompressEncode(pDataBlock, pRetrieve->data, &actualLen, pDataBlock->info.numOfCols, false); - actualLen += sizeof(SRetrieveTableRsp); - ASSERT(actualLen <= dataStrLen); - taosArrayPush(rsp.blockDataLen, &actualLen); - taosArrayPush(rsp.blockData, &buf); - rsp.blockNum++; + // TODO enable compress + int32_t actualLen = 0; + blockCompressEncode(&block, pRetrieve->data, &actualLen, block.info.numOfCols, false); + actualLen += sizeof(SRetrieveTableRsp); + ASSERT(actualLen <= dataStrLen); + taosArrayPush(rsp.blockDataLen, &actualLen); + taosArrayPush(rsp.blockData, &buf); + rsp.blockNum++; + } + } else { + ASSERT(0); } } - // TODO batch optimization + // TODO batch optimization: + // TODO continue scan until meeting batch requirement if (rsp.blockNum != 0) break; rsp.skipLogNum++; fetchOffset++; @@ -572,10 +610,10 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { // TODO: persist meta into tdb int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { - SMqRebVgReq req; + SMqRebVgReq req = {0}; tDecodeSMqRebVgReq(msg, &req); // todo lock - STqExec* pExec = taosHashGet(pTq->tqMetaNew, req.subKey, strlen(req.subKey)); + STqExec* pExec = taosHashGet(pTq->execs, req.subKey, strlen(req.subKey)); if (pExec == NULL) { ASSERT(req.oldConsumerId == -1); ASSERT(req.newConsumerId != -1); @@ -586,19 +624,27 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { memcpy(pExec->subKey, req.subKey, TSDB_SUBSCRIBE_KEY_LEN); pExec->consumerId = req.newConsumerId; pExec->epoch = -1; + + pExec->subType = req.subType; + pExec->withTbName = req.withTbName; + pExec->withSchema = req.withSchema; + pExec->withTag = req.withTag; + pExec->withTagSchema = req.withTagSchema; + pExec->qmsg = req.qmsg; req.qmsg = NULL; - pExec->pReadHandle = walOpenReadHandle(pTq->pVnode->pWal); + + pExec->pWalReader = walOpenReadHandle(pTq->pVnode->pWal); for (int32_t i = 0; i < 4; i++) { - STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta); - SReadHandle handle = { - .reader = pReadHandle, - .meta = pTq->pVnodeMeta, + pExec->pStreamReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); + SReadHandle handle = { + .reader = pExec->pStreamReader[i], + .meta = pTq->pVnode->pMeta, }; pExec->task[i] = qCreateStreamExecTaskInfo(pExec->qmsg, &handle); ASSERT(pExec->task[i]); } - taosHashPut(pTq->tqMetaNew, req.subKey, strlen(req.subKey), pExec, sizeof(STqExec)); + taosHashPut(pTq->execs, req.subKey, strlen(req.subKey), pExec, sizeof(STqExec)); return 0; } else { /*if (req.newConsumerId != -1) {*/ @@ -627,12 +673,12 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) { return -1; } for (int32_t i = 0; i < parallel; i++) { - STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta); + STqReadHandle* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); SReadHandle handle = { - .reader = pReadHandle, - .meta = pTq->pVnodeMeta, + .reader = pStreamReader, + .meta = pTq->pVnode->pMeta, }; - pTask->exec.runners[i].inputHandle = pReadHandle; + pTask->exec.runners[i].inputHandle = pStreamReader; pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); ASSERT(pTask->exec.runners[i].executor); } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 02ce6c4aad..7b21d3342d 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -82,7 +82,8 @@ bool tqNextDataBlock(STqReadHandle* pHandle) { return false; } -int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* pGroupId, int32_t* pNumOfRows) { +int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* pGroupId, int32_t* pNumOfRows, + int32_t* pNumOfCols) { /*int32_t sversion = pHandle->pBlock->sversion;*/ // TODO set to real sversion int32_t sversion = 0; @@ -104,7 +105,6 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper; *pNumOfRows = pHandle->pBlock->numOfRows; - /*int32_t numOfCols = pHandle->pSchema->numOfCols;*/ int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList); if (colNumNeed > pSchemaWrapper->nCols) { @@ -142,6 +142,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p } int32_t colActual = taosArrayGetSize(*ppCols); + *pNumOfCols = colActual; // TODO in stream shuffle case, fetch groupId *pGroupId = 0; diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 9e4aa714e2..44af83791a 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -112,7 +112,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { // open tq sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_TQ_DIR); - pVnode->pTq = tqOpen(tdir, pVnode, pVnode->pWal, pVnode->pMeta, vBufPoolGetMAF(pVnode)); + pVnode->pTq = tqOpen(tdir, pVnode, pVnode->pWal); if (pVnode->pTq == NULL) { vError("vgId: %d failed to open vnode tq since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b054bbfcb6..228b42f7f6 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -561,7 +561,8 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) SArray* pCols = NULL; uint64_t groupId; int32_t numOfRows; - int32_t code = tqRetrieveDataBlock(&pCols, pInfo->readerHandle, &groupId, &numOfRows); + int32_t numOfCols; + int32_t code = tqRetrieveDataBlock(&pCols, pInfo->readerHandle, &groupId, &numOfRows, &numOfCols); if (code != TSDB_CODE_SUCCESS || numOfRows == 0) { pTaskInfo->code = code; From 9a3365e1f1e5add013aa825dcc02192884b1e82d Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 22 Apr 2022 10:59:09 +0800 Subject: [PATCH 53/83] fix pointer type --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/tq/tqRead.c | 2 +- source/libs/executor/src/scanoperator.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 51ee00d823..a6e7b37b73 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -110,7 +110,7 @@ int tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); bool tqNextDataBlock(STqReadHandle *pHandle); int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, int32_t *pNumOfRows, - int32_t *pNumOfCols); + int16_t *pNumOfCols); // need to reposition diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 7b21d3342d..eb45577e0a 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -83,7 +83,7 @@ bool tqNextDataBlock(STqReadHandle* pHandle) { } int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* pGroupId, int32_t* pNumOfRows, - int32_t* pNumOfCols) { + int16_t* pNumOfCols) { /*int32_t sversion = pHandle->pBlock->sversion;*/ // TODO set to real sversion int32_t sversion = 0; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 228b42f7f6..06a9096615 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -561,8 +561,8 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) SArray* pCols = NULL; uint64_t groupId; int32_t numOfRows; - int32_t numOfCols; - int32_t code = tqRetrieveDataBlock(&pCols, pInfo->readerHandle, &groupId, &numOfRows, &numOfCols); + int16_t outputCol; + int32_t code = tqRetrieveDataBlock(&pCols, pInfo->readerHandle, &groupId, &numOfRows, &outputCol); if (code != TSDB_CODE_SUCCESS || numOfRows == 0) { pTaskInfo->code = code; From d67643e631a261ace8b589ff7bec8d68a2a4ba87 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 22 Apr 2022 11:46:47 +0800 Subject: [PATCH 54/83] fix test failure --- source/dnode/mgmt/implement/src/dmHandle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/implement/src/dmHandle.c b/source/dnode/mgmt/implement/src/dmHandle.c index 376f589acd..122c121ed8 100644 --- a/source/dnode/mgmt/implement/src/dmHandle.c +++ b/source/dnode/mgmt/implement/src/dmHandle.c @@ -307,11 +307,11 @@ int32_t dmStartUdfd(SDnode *pDnode) { dInfo("dnode-mgmt start udfd already called"); return 0; } + pData->startCalled = true; uv_barrier_init(&pData->barrier, 2); pData->stopping = 0; uv_thread_create(&pData->thread, dmWatchUdfd, pDnode); uv_barrier_wait(&pData->barrier); - pData->startCalled = true; pData->needCleanUp = true; return pData->spawnErr; } From 0afdb145a44253a77f1a533cec73f1c2d82d3d2c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 22 Apr 2022 13:54:00 +0800 Subject: [PATCH 55/83] feature/qnode --- include/libs/catalog/catalog.h | 1 - source/libs/parser/src/parInsert.c | 1 - source/libs/qworker/inc/qworkerMsg.h | 3 +- source/libs/qworker/src/qworker.c | 45 ++++++++++++++++++++++++---- source/libs/qworker/src/qworkerMsg.c | 44 +++++++++++++++++++++++++-- 5 files changed, 84 insertions(+), 10 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index f61073c9a5..b88afcb39d 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -213,7 +213,6 @@ int32_t catalogGetTableHashVgroup(SCatalog* pCatalog, void * pTransporter, const */ int32_t catalogGetAllMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp); - int32_t catalogGetQnodeList(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, SArray* pQnodeList); int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index cdccf0b1a1..99c849634d 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1325,7 +1325,6 @@ int32_t qBindStmtColsValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32 STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header tdSRowResetBuf(pBuilder, row); - // 1. set the parsed value from sql string for (int c = 0; c < spd->numOfBound; ++c) { SSchema* pColSchema = &pSchema[spd->boundColumns[c] - 1]; diff --git a/source/libs/qworker/inc/qworkerMsg.h b/source/libs/qworker/inc/qworkerMsg.h index c0f4d6d157..fb1950d16b 100644 --- a/source/libs/qworker/inc/qworkerMsg.h +++ b/source/libs/qworker/inc/qworkerMsg.h @@ -43,7 +43,8 @@ void qwFreeFetchRsp(void *msg); int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp); int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRsp **rsp); int32_t qwBuildAndSendHbRsp(SQWConnInfo *pConn, SSchedulerHbRsp *rsp, int32_t code); -int32_t qwRegisterBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn); +int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn); +int32_t qwRegisterHbBrokenLinkArg(SQWorkerMgmt *mgmt, uint64_t sId, SQWConnInfo *pConn); #ifdef __cplusplus } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index f97d7ffae0..a74c28cf53 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -948,7 +948,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t ex DataSinkHandle sinkHandle = NULL; SQWTaskCtx *ctx = NULL; - QW_ERR_JRET(qwRegisterBrokenLinkArg(QW_FPARAMS(), &qwMsg->connInfo)); + QW_ERR_JRET(qwRegisterQueryBrokenLinkArg(QW_FPARAMS(), &qwMsg->connInfo)); QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_QUERY, &input, NULL)); @@ -1285,23 +1285,51 @@ _return: QW_RET(TSDB_CODE_SUCCESS); } +int32_t qwProcessHbLinkBroken(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { + int32_t code = 0; + SSchedulerHbRsp rsp = {0}; + SQWSchStatus *sch = NULL; + + QW_ERR_RET(qwAcquireAddScheduler(mgmt, req->sId, QW_READ, &sch)); + + QW_LOCK(QW_WRITE, &sch->hbConnLock); + + if (qwMsg->connInfo.handle == sch->hbConnInfo.handle) { + tmsgReleaseHandle(sch->hbConnInfo.handle, TAOS_CONN_SERVER); + sch->hbConnInfo.handle = NULL; + sch->hbConnInfo.ahandle = NULL; + + QW_DLOG("release hb handle due to connection broken, handle:%p", qwMsg->connInfo.handle); + } else { + QW_DLOG("ignore hb connection broken, handle:%p, currentHandle:%p", qwMsg->connInfo.handle, sch->hbConnInfo.handle); + } + + QW_UNLOCK(QW_WRITE, &sch->hbConnLock); + + qwReleaseScheduler(QW_READ, mgmt); + + QW_RET(TSDB_CODE_SUCCESS); +} + int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { int32_t code = 0; SSchedulerHbRsp rsp = {0}; SQWSchStatus *sch = NULL; - uint64_t seqId = 0; - void *origHandle = NULL; - memcpy(&rsp.epId, &req->epId, sizeof(req->epId)); + if (qwMsg->code) { + QW_RET(qwProcessHbLinkBroken(mgmt, qwMsg, req)); + } QW_ERR_JRET(qwAcquireAddScheduler(mgmt, req->sId, QW_READ, &sch)); + QW_ERR_JRET(qwRegisterHbBrokenLinkArg(mgmt, req->sId, &qwMsg->connInfo)); + QW_LOCK(QW_WRITE, &sch->hbConnLock); if (sch->hbConnInfo.handle) { tmsgReleaseHandle(sch->hbConnInfo.handle, TAOS_CONN_SERVER); } - + memcpy(&sch->hbConnInfo, &qwMsg->connInfo, sizeof(qwMsg->connInfo)); memcpy(&sch->hbEpId, &req->epId, sizeof(req->epId)); @@ -1314,7 +1342,14 @@ int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { _return: + memcpy(&rsp.epId, &req->epId, sizeof(req->epId)); + qwBuildAndSendHbRsp(&qwMsg->connInfo, &rsp, code); + + if (code) { + tmsgReleaseHandle(qwMsg->connInfo.handle, TAOS_CONN_SERVER); + } + QW_DLOG("hb rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); QW_RET(TSDB_CODE_SUCCESS); diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index 15a42d3a31..6723eb21c1 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -286,7 +286,7 @@ int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SQWConnInfo *pConn) { } -int32_t qwRegisterBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn) { +int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn) { STaskDropReq * req = (STaskDropReq *)rpcMallocCont(sizeof(STaskDropReq)); if (NULL == req) { QW_SCH_TASK_ELOG("rpcMallocCont %d failed", (int32_t)sizeof(STaskDropReq)); @@ -313,6 +313,42 @@ int32_t qwRegisterBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn) { return TSDB_CODE_SUCCESS; } +int32_t qwRegisterHbBrokenLinkArg(SQWorkerMgmt *mgmt, uint64_t sId, SQWConnInfo *pConn) { + SSchedulerHbReq req = {0}; + req.header.vgId = mgmt->nodeId; + req.sId = sId; + + int32_t msgSize = tSerializeSSchedulerHbReq(NULL, 0, &req); + if (msgSize < 0) { + QW_SCH_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + void *msg = rpcMallocCont(msgSize); + if (NULL == msg) { + QW_SCH_ELOG("calloc %d failed", msgSize); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + if (tSerializeSSchedulerHbReq(msg, msgSize, &req) < 0) { + QW_SCH_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); + taosMemoryFree(msg); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SRpcMsg pMsg = { + .handle = pConn->handle, + .ahandle = pConn->ahandle, + .msgType = TDMT_VND_QUERY_HEARTBEAT, + .pCont = msg, + .contLen = sizeof(SSchedulerHbReq), + .code = TSDB_CODE_RPC_NETWORK_UNAVAIL, + }; + + tmsgRegisterBrokenLinkArg(&mgmt->msgCb, &pMsg); + + return TSDB_CODE_SUCCESS; +} + + int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { @@ -587,10 +623,14 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { } uint64_t sId = req.sId; - SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0}; + SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .code = pMsg->code}; qwMsg.connInfo.handle = pMsg->handle; qwMsg.connInfo.ahandle = pMsg->ahandle; + if (TSDB_CODE_RPC_NETWORK_UNAVAIL == pMsg->code) { + QW_SCH_DLOG("receive Hb msg due to network broken, error:%s", tstrerror(pMsg->code)); + } + QW_SCH_DLOG("processHb start, node:%p, handle:%p", node, pMsg->handle); QW_ERR_RET(qwProcessHb(mgmt, &qwMsg, &req)); From 87bcbe8f52f578a36d00d04e5454ba148b44ceee Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 22 Apr 2022 14:25:15 +0800 Subject: [PATCH 56/83] fix(query): fix timediff bug TD-14802 --- source/libs/scalar/src/sclfunc.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 0559792740..938f9d1f2d 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1123,20 +1123,20 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); } - char *input[2]; - for (int32_t k = 0; k < 2; ++k) { - int32_t type = GET_PARAM_TYPE(&pInput[k]); - if (type != TSDB_DATA_TYPE_BIGINT && type != TSDB_DATA_TYPE_TIMESTAMP && - type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) { - return TSDB_CODE_FAILED; + int32_t numOfRows = 0; + for (int32_t i = 0; i < inputNum; ++i) { + if (pInput[i].numOfRows > numOfRows) { + numOfRows = pInput[i].numOfRows; } } - for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { + char *input[2]; + for (int32_t i = 0; i < numOfRows; ++i) { + bool hasNull = false; for (int32_t k = 0; k < 2; ++k) { - if (colDataIsNull_s(pInput[0].columnData, i)) { - colDataAppendNULL(pOutput->columnData, i); - continue; + if (colDataIsNull_s(pInput[k].columnData, i)) { + hasNull = true; + break; } int32_t rowIdx = (pInput[k].numOfRows == 1) ? 0 : i; @@ -1178,6 +1178,11 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p } } + if (hasNull) { + colDataAppendNULL(pOutput->columnData, i); + continue; + } + int64_t result = (timeVal[0] >= timeVal[1]) ? (timeVal[0] - timeVal[1]) : (timeVal[1] - timeVal[0]); @@ -1238,7 +1243,7 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p colDataAppend(pOutput->columnData, i, (char *)&result, false); } - pOutput->numOfRows = pInput->numOfRows; + pOutput->numOfRows = numOfRows; return TSDB_CODE_SUCCESS; } From 3035ecd58e3dac047dc52fb59ae4938c62fb288b Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 22 Apr 2022 14:26:36 +0800 Subject: [PATCH 57/83] add show topic --- example/src/tmq.c | 1 + source/dnode/mnode/impl/src/mndPerfSchema.c | 4 +- source/dnode/mnode/impl/src/mndStream.c | 5 +- source/dnode/mnode/impl/src/mndTopic.c | 52 +++++++-------------- 4 files changed, 24 insertions(+), 38 deletions(-) diff --git a/example/src/tmq.c b/example/src/tmq.c index 88fce7f4be..46f57799e7 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -103,6 +103,7 @@ int32_t create_topic() { /*const char* sql = "select * from tu1";*/ /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));*/ + /*pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");*/ pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from ct1"); if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); diff --git a/source/dnode/mnode/impl/src/mndPerfSchema.c b/source/dnode/mnode/impl/src/mndPerfSchema.c index 38e9e79228..fe96bcb709 100644 --- a/source/dnode/mnode/impl/src/mndPerfSchema.c +++ b/source/dnode/mnode/impl/src/mndPerfSchema.c @@ -41,10 +41,10 @@ static const SPerfsTableSchema queriesSchema[] = { static const SPerfsTableSchema topicSchema[] = { {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + /*{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},*/ {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "row_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + /*{.name = "row_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT},*/ }; static const SPerfsTableSchema consumerSchema[] = { diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 58e5b6c65b..1062154e02 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -59,7 +59,7 @@ int32_t mndInitStream(SMnode *pMnode) { /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM_RSP, mndProcessDropStreamInRsp);*/ // mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveStream); - mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextStream); + /*mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextStream);*/ return sdbSetTable(pMnode->pSdb, table); } @@ -247,7 +247,8 @@ static int32_t mndStreamGetPlanString(const char *ast, int8_t triggerType, int64 return code; } -int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, int8_t triggerType, int64_t watermark, STrans *pTrans) { +int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, int8_t triggerType, int64_t watermark, + STrans *pTrans) { SNode *pAst = NULL; if (nodesStringToNode(ast, &pAst) < 0) { diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 22b1b404bb..f17b8a4d88 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -35,7 +35,7 @@ static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pTopic, SMqTopicObj static int32_t mndProcessCreateTopicReq(SNodeMsg *pReq); static int32_t mndProcessDropTopicReq(SNodeMsg *pReq); static int32_t mndProcessDropTopicInRsp(SNodeMsg *pRsp); -static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter); int32_t mndInitTopic(SMnode *pMnode) { @@ -51,7 +51,7 @@ int32_t mndInitTopic(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_DROP_TOPIC, mndProcessDropTopicReq); mndSetMsgHandle(pMnode, TDMT_VND_DROP_TOPIC_RSP, mndProcessDropTopicInRsp); - // mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveTopic); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveTopic); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextTopic); return sdbSetTable(pMnode->pSdb, table); @@ -511,56 +511,40 @@ static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTo return 0; } -static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { +static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SMqTopicObj *pTopic = NULL; - int32_t cols = 0; - char *pWrite; - char prefix[TSDB_DB_FNAME_LEN] = {0}; - SDbObj *pDb = mndAcquireDb(pMnode, pShow->db); - if (pDb == NULL) return 0; - - tstrncpy(prefix, pShow->db, TSDB_DB_FNAME_LEN); - strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = (int32_t)strlen(prefix); - - while (numOfRows < rows) { + while (numOfRows < rowsCapacity) { pShow->pIter = sdbFetch(pSdb, SDB_TOPIC, pShow->pIter, (void **)&pTopic); if (pShow->pIter == NULL) break; - if (pTopic->dbUid != pDb->uid) { - if (strncmp(pTopic->name, prefix, prefixLen) != 0) { - mError("Inconsistent topic data, name:%s, db:%s, dbUid:%" PRIu64, pTopic->name, pDb->name, pDb->uid); - } + int32_t cols = 0; - sdbRelease(pSdb, pTopic); - continue; - } + char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tstrncpy(&topicName[VARSTR_HEADER_SIZE], pTopic->name, TSDB_TOPIC_NAME_LEN); + varDataSetLen(topicName, strlen(&topicName[VARSTR_HEADER_SIZE])); - cols = 0; + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)topicName, false); - char topicName[TSDB_TOPIC_NAME_LEN] = {0}; - tstrncpy(topicName, pTopic->name + prefixLen, TSDB_TOPIC_NAME_LEN); - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_TO_VARSTR(pWrite, topicName); - cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pTopic->createTime, false); - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = pTopic->createTime; - cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + char *sql = taosMemoryCalloc(1, strlen(pTopic->sql) + 1 + VARSTR_HEADER_SIZE); + strcpy(&sql[VARSTR_HEADER_SIZE], pTopic->sql); + varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE])); + colDataAppend(pColInfo, numOfRows, (const char *)sql, false); - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pTopic->sql, pShow->bytes[cols]); - cols++; + taosMemoryFree(sql); numOfRows++; sdbRelease(pSdb, pTopic); } - mndReleaseDb(pMnode, pDb); pShow->numOfRows += numOfRows; return numOfRows; } From 40d112e6d1a99fbbd259ee3ce1e40d71d18a95f3 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 22 Apr 2022 14:30:33 +0800 Subject: [PATCH 58/83] fix(query): fix convertStringToTimestamp considering daylight saving --- source/common/src/ttime.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 2f3677330b..43554aed8b 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -415,7 +415,7 @@ int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec if (type == TSDB_DATA_TYPE_BINARY) { newColData = taosMemoryCalloc(1, charLen + 1); memcpy(newColData, varDataVal(inputData), charLen); - bool ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, 0); + bool ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, tsDaylight); if (ret != TSDB_CODE_SUCCESS) { taosMemoryFree(newColData); return ret; @@ -429,7 +429,7 @@ int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec return TSDB_CODE_FAILED; } newColData[len] = 0; - bool ret = taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, 0); + bool ret = taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, tsDaylight); if (ret != TSDB_CODE_SUCCESS) { taosMemoryFree(newColData); return ret; From 1fe2abc8cdca620da390d453012e49e69d73b4e4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 22 Apr 2022 14:35:00 +0800 Subject: [PATCH 59/83] refactor:fix error in unit test for scalar test --- source/libs/scalar/test/scalar/scalarTests.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index de76e7e274..94ccdcdd0d 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -156,10 +156,9 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.colId = 3; int32_t size = idata.info.bytes * rowNum; idata.pData = (char *)taosMemoryCalloc(1, size); + colInfoDataEnsureCapacity(&idata, 0, rowNum); taosArrayPush(res->pDataBlock, &idata); - blockDataEnsureCapacity(res, rowNum); - SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); for (int32_t i = 0; i < rowNum; ++i) { colDataAppend(pColumn, i, (const char *)value, false); From e28b75b8d4754fb036c62bdf6b202f7b7d5aa315 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Apr 2022 15:07:20 +0800 Subject: [PATCH 60/83] refactor(query): do some internal refactor. --- include/libs/function/function.h | 4 +- source/libs/executor/inc/executil.h | 2 +- source/libs/executor/inc/executorimpl.h | 4 +- source/libs/executor/src/executorimpl.c | 77 ++++++++--------- source/libs/executor/src/groupoperator.c | 4 +- source/libs/function/inc/builtinsimpl.h | 7 +- source/libs/function/src/builtins.c | 2 +- source/libs/function/src/builtinsimpl.c | 100 +++++++++++++++++------ 8 files changed, 118 insertions(+), 82 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 68fc3be617..004d834287 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -37,7 +37,7 @@ typedef struct SFuncExecEnv { typedef bool (*FExecGetEnv)(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx); -typedef void (*FExecFinalize)(struct SqlFunctionCtx *pCtx); +typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock, int32_t slotId); typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); typedef struct SScalarFuncExecFuncs { @@ -113,7 +113,7 @@ typedef struct SResultRowEntryInfo { bool initialized:1; // output buffer has been initialized bool complete:1; // query has completed uint8_t isNullRes:6; // the result is null - uint8_t numOfRes; // num of output result in current buffer + uint8_t numOfRes; // num of output result in current buffer } SResultRowEntryInfo; // determine the real data need to calculated the result diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 7c931c860c..834d37927a 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -60,7 +60,7 @@ typedef struct SResultRow { uint32_t numOfRows; // number of rows of current time window struct SResultRowEntryInfo* pEntryInfo; // For each result column, there is a resultInfo STimeWindow win; - char *key; // start key of current result row + char *key; // start key of current result row } SResultRow; typedef struct SResultRowPosition { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 5adfb7caca..c6d209e706 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -608,8 +608,8 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, size_t keyBufSize, const char* pkey); void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows); -void doBuildResultDatablock(SSDataBlock* pBlock, int32_t rowCapacity, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, - SDiskbasedBuf* pBuf, int32_t* rowCellOffset); +void doBuildResultDatablock(SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, + SDiskbasedBuf* pBuf, int32_t* rowCellOffset, SqlFunctionCtx* pCtx); void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 7a7cb68424..4a0206f55e 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -230,10 +230,10 @@ int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) { void operatorDummyCloseFn(void* param, int32_t numOfCols) {} -static int32_t doCopyToSDataBlock(SSDataBlock* pBlock, int32_t rowCapacity, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, - int32_t orderType, int32_t* rowCellOffset); -static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); +static int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, + int32_t orderType, int32_t* rowCellOffset, SqlFunctionCtx* pCtx); +static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo); static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); static void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* pTaskInfo); @@ -2809,16 +2809,6 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { } } -void finalizeQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput) { - for (int32_t j = 0; j < numOfOutput; ++j) { - if (pCtx[j].functionId == -1) { - continue; - } - - pCtx[j].fpSet.finalize(&pCtx[j]); - } -} - void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { for (int32_t i = 0; i < pResultRowInfo->size; ++i) { @@ -2841,7 +2831,7 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD } if (pCtx[j].fpSet.process) { // TODO set the dummy function, to avoid the check for null ptr. - pCtx[j].fpSet.finalize(&pCtx[j]); +// pCtx[j].fpSet.finalize(&pCtx[j]); } if (pRow->numOfRows < pResInfo->numOfRes) { @@ -2872,7 +2862,7 @@ void finalizeUpdatedResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbased } if (pCtx[j].fpSet.process) { // TODO set the dummy function. - pCtx[j].fpSet.finalize(&pCtx[j]); +// pCtx[j].fpSet.finalize(&pCtx[j]); pResInfo->initialized = true; } @@ -3107,7 +3097,8 @@ void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWin * @param pQInfo * @param result */ -int32_t doCopyToSDataBlock(SSDataBlock* pBlock, int32_t rowCapacity, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, int32_t orderType, int32_t* rowCellOffset) { +int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, + int32_t orderType, int32_t* rowCellOffset, SqlFunctionCtx* pCtx) { int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); int32_t numOfResult = pBlock->info.rows; // there are already exists result rows @@ -3125,8 +3116,6 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, int32_t rowCapacity, SExprInfo* step = -1; } - int32_t nrows = pBlock->info.rows; - for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) { SResultRowPosition* pPos = taosArrayGet(pGroupResInfo->pRows, i); SFilePage* page = getBufPage(pBuf, pPos->pageId); @@ -3139,7 +3128,7 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, int32_t rowCapacity, SExprInfo* // TODO copy multiple rows? int32_t numOfRowsToCopy = pRow->numOfRows; - if (numOfResult + numOfRowsToCopy >= rowCapacity) { + if (numOfResult + numOfRowsToCopy >= pBlock->info.capacity) { break; } @@ -3148,31 +3137,32 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, int32_t rowCapacity, SExprInfo* for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) { int32_t slotId = pExprInfo[j].base.resSchema.slotId; - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); - SResultRowEntryInfo* pEntryInfo = getResultCell(pRow, j, rowCellOffset); + pCtx[j].resultInfo = getResultCell(pRow, j, rowCellOffset); + if (pCtx[j].fpSet.process) { + pCtx[j].fpSet.finalize(&pCtx[j], pBlock, slotId); + } else { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); - char* in = GET_ROWCELL_INTERBUF(pEntryInfo); - colDataAppend(pColInfoData, nrows, in, pEntryInfo->isNullRes); + char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); + colDataAppend(pColInfoData, pBlock->info.rows, in, pCtx[j].resultInfo->isNullRes); + } } releaseBufPage(pBuf, page); - nrows += 1; - numOfResult += numOfRowsToCopy; - if (numOfResult == rowCapacity) { // output buffer is full + pBlock->info.rows += pRow->numOfRows; + if (pBlock->info.rows >= pBlock->info.capacity) { // output buffer is full break; } } // qDebug("QInfo:0x%"PRIx64" copy data to query buf completed", GET_TASKID(pRuntimeEnv)); - pBlock->info.rows = numOfResult; blockDataUpdateTsWindow(pBlock); - return 0; } -void doBuildResultDatablock(SSDataBlock* pBlock, int32_t rowCapacity, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, - int32_t* rowCellOffset) { +void doBuildResultDatablock(SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, + int32_t* rowCellOffset, SqlFunctionCtx* pCtx) { assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); blockDataCleanup(pBlock); @@ -3181,7 +3171,7 @@ void doBuildResultDatablock(SSDataBlock* pBlock, int32_t rowCapacity, SGroupResI } int32_t orderType = TSDB_ORDER_ASC; - doCopyToSDataBlock(pBlock, rowCapacity, pExprInfo, pBuf, pGroupResInfo, orderType, rowCellOffset); + doCopyToSDataBlock(pBlock, pExprInfo, pBuf, pGroupResInfo, orderType, rowCellOffset, pCtx); // add condition (pBlock->info.rows >= 1) just to runtime happy blockDataUpdateTsWindow(pBlock); @@ -4405,7 +4395,7 @@ static void doFinalizeResultImpl(SqlFunctionCtx* pCtx, int32_t numOfExpr) { // SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1); // doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); // } else { - pCtx[j].fpSet.finalize(&pCtx[j]); +// pCtx[j].fpSet.finalize(&pCtx[j]); } } @@ -4805,7 +4795,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator, bool* newgroup) } blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pInfo->pRes, pOperator->resultInfo.capacity, &pAggInfo->groupResInfo, pOperator->pExpr, pAggInfo->aggSup.pResultBuf, pInfo->rowCellInfoOffset); + doBuildResultDatablock(pInfo->pRes, &pAggInfo->groupResInfo, pOperator->pExpr, pAggInfo->aggSup.pResultBuf, pInfo->rowCellInfoOffset, pInfo->pCtx); if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pAggInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -5129,8 +5119,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator, bool* newgro } blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); - doBuildResultDatablock(pBlock, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, - pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); + doBuildResultDatablock(pBlock, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); if (pBlock->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); @@ -5149,7 +5138,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo *pOperator, bool* newgroup } if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pInfo->binfo.pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); + doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } @@ -5184,7 +5173,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo *pOperator, bool* newgroup initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pInfo->binfo.pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); + doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); ASSERT(pInfo->binfo.pRes->info.rows > 0); pOperator->status = OP_RES_TO_RETURN; @@ -5229,7 +5218,7 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo *pOperator, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pSliceInfo->binfo.resultRowInfo); setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); - finalizeQueryResult(pSliceInfo->binfo.pCtx, pOperator->numOfOutput); +// finalizeQueryResult(pSliceInfo->binfo.pCtx, pOperator->numOfOutput); initGroupResInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo); // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pSliceInfo->pRes); @@ -5286,8 +5275,8 @@ static SSDataBlock* doSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgroup OPTR_SET_OPENED(pOperator); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pInfo->binfo.pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, - pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); + doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, + pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); @@ -5377,7 +5366,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pBInfo->pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset); + doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); return NULL; @@ -5409,7 +5398,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { initGroupResInfo(&pInfo->groupResInfo, &pBInfo->resultRowInfo); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pBInfo->pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset); + doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -5426,7 +5415,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pBInfo->pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset); + doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); return NULL; @@ -5458,7 +5447,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) initGroupResInfo(&pInfo->groupResInfo, &pBInfo->resultRowInfo); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pBInfo->pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset); + doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 9f75f97632..8a0da52a8e 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -265,7 +265,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou SSDataBlock* pRes = pInfo->binfo.pRes; if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); + doBuildResultDatablock(pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); if (pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } @@ -311,7 +311,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); while(1) { - doBuildResultDatablock(pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); + doBuildResultDatablock(pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); doFilter(pInfo->pCondition, pRes); bool hasRemain = hasRemainDataInCurrentGroup(&pInfo->groupResInfo); diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 8473138e4a..db92740fff 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -24,7 +24,7 @@ extern "C" { #include "functionMgt.h" bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); -void functionFinalize(SqlFunctionCtx *pCtx); +int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId); EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow); bool getCountFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); @@ -43,12 +43,12 @@ int32_t maxFunction(SqlFunctionCtx *pCtx); bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t stddevFunction(SqlFunctionCtx* pCtx); -void stddevFinalize(SqlFunctionCtx* pCtx); +int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId); bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t percentileFunction(SqlFunctionCtx *pCtx); -void percentileFinalize(SqlFunctionCtx* pCtx); +int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId); bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool diffFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo); @@ -60,6 +60,7 @@ int32_t lastFunction(SqlFunctionCtx *pCtx); bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); int32_t topFunction(SqlFunctionCtx *pCtx); +int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index efd8de84c4..4f5dc7d6a1 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -507,7 +507,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getTopBotFuncEnv, .initFunc = functionSetup, .processFunc = topFunction, - .finalizeFunc = functionFinalize + .finalizeFunc = topBotFinalize, }, { .name = "bottom", diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 0942726588..3dc2e62e92 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -49,11 +49,17 @@ bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { return true; } -void functionFinalize(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); +int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) { + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - cleanupResultRowEntry(pResInfo); + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); pResInfo->isNullRes = (pResInfo->numOfRes == 0)? 1:0; + cleanupResultRowEntry(pResInfo); + + char* in = GET_ROWCELL_INTERBUF(pResInfo); + colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes); + + return pResInfo->numOfRes; } EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) { @@ -612,12 +618,11 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } -void stddevFinalize(SqlFunctionCtx* pCtx) { - functionFinalize(pCtx); - +int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) { SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); double avg = pStddevRes->isum / ((double) pStddevRes->count); pStddevRes->result = sqrt(pStddevRes->quadraticISum/((double)pStddevRes->count) - avg*avg); + return functionFinalize(pCtx, pBlock, slotId); } typedef struct SPercentileInfo { @@ -739,7 +744,7 @@ int32_t percentileFunction(SqlFunctionCtx *pCtx) { return TSDB_CODE_SUCCESS; } -void percentileFinalize(SqlFunctionCtx* pCtx) { +int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) { SVariant* pVal = &pCtx->param[1].param; double v = pVal->nType == TSDB_DATA_TYPE_INT ? pVal->i : pVal->d; @@ -752,7 +757,7 @@ void percentileFinalize(SqlFunctionCtx* pCtx) { } tMemBucketDestroy(pMemBucket); - functionFinalize(pCtx); + return functionFinalize(pCtx, pBlock, slotId); } bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { @@ -1173,16 +1178,14 @@ typedef struct STopBotResItem { } STopBotResItem; typedef struct STopBotRes { - int32_t num; + int32_t pageId; +// int32_t num; STopBotResItem *pItems; } STopBotRes; bool getTopBotFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { - SColumnNode* pColNode = (SColumnNode*) nodesListGetNode(pFunc->pParameterList, 0); - int32_t bytes = pColNode->node.resType.bytes; SValueNode* pkNode = (SValueNode*) nodesListGetNode(pFunc->pParameterList, 1); - - pEnv->calcMemSize = sizeof(STopBotRes) + pkNode->datum.i * bytes; + pEnv->calcMemSize = sizeof(STopBotRes) + pkNode->datum.i * sizeof(STopBotResItem); return true; } @@ -1194,13 +1197,14 @@ static STopBotRes *getTopBotOutputInfo(SqlFunctionCtx *pCtx) { return pRes; } -static void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, uint16_t type, uint64_t uid); +static void doAddIntoResult(STopBotRes* pRes, int32_t maxSize, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, + uint16_t type, uint64_t uid, SResultRowEntryInfo* pEntryInfo); int32_t topFunction(SqlFunctionCtx *pCtx) { int32_t numOfElems = 0; + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STopBotRes *pRes = getTopBotOutputInfo(pCtx); - assert(pRes->num >= 0); // if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotRes) + POINTER_BYTES * pCtx->param[0].i)) { // buildTopBotStruct(pRes, pCtx); @@ -1221,11 +1225,9 @@ int32_t topFunction(SqlFunctionCtx *pCtx) { numOfElems++; char* data = colDataGetData(pCol, i); - doAddIntoResult(pRes, pCtx->param[1].param.i, data, type, pInput->uid); + doAddIntoResult(pRes, pCtx->param[1].param.i, data, i, NULL, type, pInput->uid, pResInfo); } - // treat the result as only one result - SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); return TSDB_CODE_SUCCESS; } @@ -1256,7 +1258,9 @@ static int32_t topBotResComparFn(const void *p1, const void *p2, const void *par return (val1->v.d > val2->v.d) ? 1 : -1; } -void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, uint16_t type, uint64_t uid) { + +void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, + uint64_t uid, SResultRowEntryInfo* pEntryInfo) { SVariant val = {0}; taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); @@ -1264,29 +1268,71 @@ void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, uint16_t ty assert(pItems != NULL); // not full yet - if (pRes->num < maxSize) { - STopBotResItem* pItem = &pItems[pRes->num]; + if (pEntryInfo->numOfRes < maxSize) { + STopBotResItem* pItem = &pItems[pEntryInfo->numOfRes]; pItem->v = val; pItem->uid = uid; pItem->tuplePos.pageId = -1; // todo set the corresponding tuple data in the disk-based buffer - pRes->num++; - taosheapsort((void *) pItem, sizeof(STopBotResItem), pRes->num, (const void *) &type, topBotResComparFn, false); + if (pRes->pageId == -1) { + SFilePage* pPage = getNewBufPage(NULL, 0, &pRes->pageId); + pPage->num = sizeof(SFilePage); + + // keep the current row data + for(int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { + SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); + bool isNull = colDataIsNull_s(pCol, rowIndex); + + + colDataGetData(pCol, rowIndex); + } + + } + + // allocate the buffer and keep the data of this row into the new allocated buffer + pEntryInfo->numOfRes++; + taosheapsort((void *) pItems, sizeof(STopBotResItem), pEntryInfo->numOfRes, (const void *) &type, topBotResComparFn, false); } else { // replace the minimum value in the result if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i > pItems[0].v.i) || (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u > pItems[0].v.u) || (IS_FLOAT_TYPE(type) && val.d > pItems[0].v.d)) { - STopBotResItem* pItem = &pItems[pRes->num]; + STopBotResItem* pItem = &pItems[0]; pItem->v = val; pItem->uid = uid; pItem->tuplePos.pageId = -1; // todo set the corresponding tuple data in the disk-based buffer - taosheapadjust((void *) pItem, sizeof(STopBotResItem), 0, pRes->num - 1, (const void *) &type, topBotResComparFn, NULL, false); + taosheapadjust((void *) pItems, sizeof(STopBotResItem), 0, pEntryInfo->numOfRes - 1, (const void *) &type, topBotResComparFn, NULL, false); } } } -void topBotFinalize(SqlFunctionCtx* pCtx) { - functionFinalize(pCtx); +int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) { + SResultRowEntryInfo *pEntryInfo = GET_RES_INFO(pCtx); + STopBotRes* pRes = GET_ROWCELL_INTERBUF(pEntryInfo); + pEntryInfo->complete = true; + int32_t type = pCtx->input.pData[0]->info.type; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + // todo assign the tag value and the corresponding row data + int32_t currentRow = pBlock->info.rows; + switch(type) { + case TSDB_DATA_TYPE_INT: { + for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) { + STopBotResItem* pItem = &pRes->pItems[i]; + colDataAppendInt32(pCol, currentRow++, (int32_t*)&pItem->v.i); + + int32_t pageId = pItem->tuplePos.pageId; + int32_t offset = pItem->tuplePos.offset; + if (pageId != -1) { + // todo + } + } + break; + } + } + + return pEntryInfo->numOfRes; + +// return functionFinalize(pCtx, pBlock, slotId); } \ No newline at end of file From a395b2157867ea677f419ca7a4d66f0baa51a4e8 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 22 Apr 2022 15:21:26 +0800 Subject: [PATCH 61/83] fix start/stop/restart udfd --- source/dnode/mgmt/implement/src/dmHandle.c | 41 ++++++++++++---------- source/dnode/mgmt/interface/inc/dmDef.h | 2 +- source/libs/function/src/udfd.c | 2 +- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/source/dnode/mgmt/implement/src/dmHandle.c b/source/dnode/mgmt/implement/src/dmHandle.c index 122c121ed8..c4eba53a34 100644 --- a/source/dnode/mgmt/implement/src/dmHandle.c +++ b/source/dnode/mgmt/implement/src/dmHandle.c @@ -220,13 +220,11 @@ static int32_t dmSpawnUdfd(SDnode *pDnode); void dmUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) { dInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal); - uv_close((uv_handle_t*)process, NULL); SDnode *pDnode = process->data; - SUdfdData *pData = &pDnode->udfdData; - if (atomic_load_8(&pData->stopping) != 0) { - dDebug("udfd process exit due to stopping"); + if (exitStatus == 0 && termSignal == 0) { + dInfo("udfd process exit due to SIGINT"); } else { - uv_close((uv_handle_t*)&pData->ctrlPipe, NULL); + dInfo("udfd process restart"); dmSpawnUdfd(pDnode); } } @@ -248,6 +246,7 @@ static int32_t dmSpawnUdfd(SDnode *pDnode) { options.file = path; options.exit_cb = dmUdfdExit; + SUdfdData *pData = &pDnode->udfdData; uv_pipe_init(&pData->loop, &pData->ctrlPipe, 1); @@ -260,6 +259,8 @@ static int32_t dmSpawnUdfd(SDnode *pDnode) { options.stdio_count = 3; options.stdio = child_stdio; + options.flags = UV_PROCESS_DETACHED; + char dnodeIdEnvItem[32] = {0}; char thrdPoolSizeEnvItem[32] = {0}; snprintf(dnodeIdEnvItem, 32, "%s=%d", "DNODE_ID", pDnode->data.dnodeId); @@ -284,24 +285,31 @@ static void dmUdfdCloseWalkCb(uv_handle_t* handle, void* arg) { } } -void dmWatchUdfd(void *args) { +static void dmUdfdStopAsyncCb(uv_async_t *async) { + SDnode *pDnode = async->data; + SUdfdData *pData = &pDnode->udfdData; + uv_stop(&pData->loop); +} + +static void dmWatchUdfd(void *args) { SDnode *pDnode = args; SUdfdData *pData = &pDnode->udfdData; uv_loop_init(&pData->loop); + uv_async_init(&pData->loop, &pData->stopAsync, dmUdfdStopAsyncCb); + pData->stopAsync.data = pDnode; int32_t err = dmSpawnUdfd(pDnode); atomic_store_32(&pData->spawnErr, err); uv_barrier_wait(&pData->barrier); uv_run(&pData->loop, UV_RUN_DEFAULT); - err = uv_loop_close(&pData->loop); - while (err == UV_EBUSY) { - uv_walk(&pData->loop, dmUdfdCloseWalkCb, NULL); - uv_run(&pData->loop, UV_RUN_DEFAULT); - err = uv_loop_close(&pData->loop); - } + uv_loop_close(&pData->loop); + + uv_walk(&pData->loop, dmUdfdCloseWalkCb, NULL); + uv_run(&pData->loop, UV_RUN_DEFAULT); + uv_loop_close(&pData->loop); return; } -int32_t dmStartUdfd(SDnode *pDnode) { +static int32_t dmStartUdfd(SDnode *pDnode) { SUdfdData *pData = &pDnode->udfdData; if (pData->startCalled) { dInfo("dnode-mgmt start udfd already called"); @@ -309,30 +317,27 @@ int32_t dmStartUdfd(SDnode *pDnode) { } pData->startCalled = true; uv_barrier_init(&pData->barrier, 2); - pData->stopping = 0; uv_thread_create(&pData->thread, dmWatchUdfd, pDnode); uv_barrier_wait(&pData->barrier); pData->needCleanUp = true; return pData->spawnErr; } -int32_t dmStopUdfd(SDnode *pDnode) { +static int32_t dmStopUdfd(SDnode *pDnode) { dInfo("dnode-mgmt to stop udfd. need cleanup: %d, spawn err: %d", pDnode->udfdData.needCleanUp, pDnode->udfdData.spawnErr); SUdfdData *pData = &pDnode->udfdData; if (!pData->needCleanUp) { return 0; } - atomic_store_8(&pData->stopping, 1); uv_barrier_destroy(&pData->barrier); if (pData->spawnErr == 0) { uv_process_kill(&pData->process, SIGINT); } - uv_stop(&pData->loop); + uv_async_send(&pData->stopAsync); uv_thread_join(&pData->thread); - atomic_store_8(&pData->stopping, 0); return 0; } diff --git a/source/dnode/mgmt/interface/inc/dmDef.h b/source/dnode/mgmt/interface/inc/dmDef.h index 4f4a2ed349..dd31faf1b7 100644 --- a/source/dnode/mgmt/interface/inc/dmDef.h +++ b/source/dnode/mgmt/interface/inc/dmDef.h @@ -151,8 +151,8 @@ typedef struct SUdfdData { uv_barrier_t barrier; uv_process_t process; int spawnErr; - int8_t stopping; uv_pipe_t ctrlPipe; + uv_async_t stopAsync; } SUdfdData; typedef struct SDnode { diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 5f7532da87..6540851758 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -586,7 +586,7 @@ static int32_t udfdRun() { udfdCloseClientRpc(); uv_mutex_destroy(&global.udfsMutex); taosHashCleanup(global.udfsHash); - return code; + return 0; } int main(int argc, char *argv[]) { From d28afa2bd9e657ef66ed2e25c37feb873e493ab8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Apr 2022 15:26:02 +0800 Subject: [PATCH 62/83] refactor: do some internal refactor. --- source/client/src/clientImpl.c | 7 ++++++- source/common/src/tdatablock.c | 1 - 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 10edb38bf1..890b7b87cb 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -436,7 +436,12 @@ int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSe return -1; } - taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]); + int32_t code = taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]); + if (code != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return terrno; + } + mgmtEpSet->numOfEps++; } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index d796df16d3..800bcd70ce 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -31,7 +31,6 @@ int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) { if (pEp->port == 0) { pEp->port = tsServerPort; - return -1; } return 0; From 7cd259592f2752b336a667e881f48dc753216a15 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Apr 2022 15:57:49 +0800 Subject: [PATCH 63/83] fix(driver): the user specified port that is in taos_connect API is active. --- source/client/src/clientImpl.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 890b7b87cb..aae9303bdc 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -95,16 +95,16 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, if (initEpSetFromCfg(ip, NULL, &epSet) < 0) { return NULL; } - - if (port) { - epSet.epSet.eps[0].port = port; - } } else { if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) { return NULL; } } + if (port) { + epSet.epSet.eps[0].port = port; + } + char* key = getClusterKey(user, secretEncrypt, ip, port); SAppInstInfo** pInst = NULL; From a601081e6d930e338b07ec98986847406bae6ddc Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 22 Apr 2022 15:56:52 +0800 Subject: [PATCH 64/83] refactor(query): limit cast var type length to 1000 --- source/libs/function/src/builtins.c | 2 +- source/libs/scalar/src/sclfunc.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index ce7384cbbe..909cf9d7ef 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -332,7 +332,7 @@ static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } int32_t para2Bytes = pFunc->node.resType.bytes; - if (para2Bytes <= 0) { //non-positive value or overflow + if (para2Bytes <= 0 || para2Bytes > 1000) { //cast dst var type length limits to 1000 return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 938f9d1f2d..c8e8780552 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -644,9 +644,9 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu } int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - int16_t inputType = pInput[0].columnData->info.type; - int16_t outputType = pOutput[0].columnData->info.type; - int64_t outputLen = pOutput[0].columnData->info.bytes; + int16_t inputType = GET_PARAM_TYPE(&pInput[0]); + int16_t outputType = GET_PARAM_TYPE(&pOutput[0]); + int64_t outputLen = GET_PARAM_BYTES(&pOutput[0]); if (IS_VAR_DATA_TYPE(outputType)) { int32_t factor = (TSDB_DATA_TYPE_NCHAR == outputType) ? TSDB_NCHAR_SIZE : 1; From 8ce6f55a9d4df6dcdd900e357b125f8e218fd670 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 22 Apr 2022 16:52:50 +0800 Subject: [PATCH 65/83] refine stop processing --- source/dnode/mgmt/implement/src/dmHandle.c | 12 +++++------- source/dnode/mgmt/interface/inc/dmDef.h | 1 + 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/source/dnode/mgmt/implement/src/dmHandle.c b/source/dnode/mgmt/implement/src/dmHandle.c index c4eba53a34..8ba12513db 100644 --- a/source/dnode/mgmt/implement/src/dmHandle.c +++ b/source/dnode/mgmt/implement/src/dmHandle.c @@ -221,8 +221,8 @@ static int32_t dmSpawnUdfd(SDnode *pDnode); void dmUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) { dInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal); SDnode *pDnode = process->data; - if (exitStatus == 0 && termSignal == 0) { - dInfo("udfd process exit due to SIGINT"); + if (exitStatus == 0 && termSignal == 0 || atomic_load_32(&pDnode->udfdData.stopCalled)) { + dInfo("udfd process exit due to SIGINT or dnode-mgmt called stop"); } else { dInfo("udfd process restart"); dmSpawnUdfd(pDnode); @@ -327,14 +327,12 @@ static int32_t dmStopUdfd(SDnode *pDnode) { dInfo("dnode-mgmt to stop udfd. need cleanup: %d, spawn err: %d", pDnode->udfdData.needCleanUp, pDnode->udfdData.spawnErr); SUdfdData *pData = &pDnode->udfdData; - if (!pData->needCleanUp) { + if (!pData->needCleanUp || atomic_load_32(&pData->stopCalled)) { return 0; } - + atomic_store_32(&pData->stopCalled, 1); + pData->needCleanUp = false; uv_barrier_destroy(&pData->barrier); - if (pData->spawnErr == 0) { - uv_process_kill(&pData->process, SIGINT); - } uv_async_send(&pData->stopAsync); uv_thread_join(&pData->thread); diff --git a/source/dnode/mgmt/interface/inc/dmDef.h b/source/dnode/mgmt/interface/inc/dmDef.h index dd31faf1b7..087892e741 100644 --- a/source/dnode/mgmt/interface/inc/dmDef.h +++ b/source/dnode/mgmt/interface/inc/dmDef.h @@ -153,6 +153,7 @@ typedef struct SUdfdData { int spawnErr; uv_pipe_t ctrlPipe; uv_async_t stopAsync; + int32_t stopCalled; } SUdfdData; typedef struct SDnode { From 4cc3b019b362c42c168eb3b72815ce031b9d4382 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 22 Apr 2022 17:06:08 +0800 Subject: [PATCH 66/83] [test: add test cases for tmq] --- tests/script/tsim/tmq/basic1.sim | 72 +++-- tests/script/tsim/tmq/basic2.sim | 219 ++++++++++++++ tests/script/tsim/tmq/basic3.sim | 278 ++++++++++++++++++ tests/script/tsim/tmq/basic4.sim | 216 ++++++++++++++ .../script/tsim/tmq/prepareBasicEnv-1vgrp.sim | 88 ++++++ .../script/tsim/tmq/prepareBasicEnv-4vgrp.sim | 88 ++++++ tests/script/tsim/tmq/prepareBasicEnv.sim | 2 +- tests/test/c/tmqSim.c | 13 +- 8 files changed, 947 insertions(+), 29 deletions(-) create mode 100644 tests/script/tsim/tmq/basic2.sim create mode 100644 tests/script/tsim/tmq/basic3.sim create mode 100644 tests/script/tsim/tmq/basic4.sim create mode 100644 tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim create mode 100644 tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim diff --git a/tests/script/tsim/tmq/basic1.sim b/tests/script/tsim/tmq/basic1.sim index df6a553d1a..f4e8f08c37 100644 --- a/tests/script/tsim/tmq/basic1.sim +++ b/tests/script/tsim/tmq/basic1.sim @@ -1,15 +1,16 @@ #### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -# vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics +#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics + # notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN # The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; # # notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). # -run tsim/tmq/prepareBasicEnv.sim +run tsim/tmq/prepareBasicEnv-1vgrp.sim #---- global parameters start ----# $dbName = db @@ -20,11 +21,11 @@ $ntbPrefix = ntb $stbNum = 1 $ctbNum = 10 $ntbNum = 10 -$rowsPerCtb = 100 +$rowsPerCtb = 10 $tstart = 1640966400000 # 2022-01-01 00:00:00.000 #---- global parameters end ----# -$pullDelay = 3 +$pullDelay = 5 $ifcheckdata = 1 $showMsg = 1 $showRow = 0 @@ -55,9 +56,33 @@ sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 $keyList = ' . group.id:cgrp1 $keyList = $keyList . ' +$cdb_index = 0 +#=============================== start consume =============================# + print ================ test consume from stb $loop_cnt = 0 loop_consume_diff_topic_from_stb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + if $loop_cnt == 0 then print == scenario 1: topic_stb_column $topicList = ' . topic_stb_column @@ -76,13 +101,12 @@ endi $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb -#$expectmsgcnt = $totalMsgOfStb + 1 -$expectmsgcnt = 110 +$expectmsgcnt = $totalMsgOfStb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) print == start consumer to pull msgs from stb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start print == check consume result wait_consumer_end_from_stb: @@ -106,11 +130,16 @@ $loop_cnt = $loop_cnt + 1 goto loop_consume_diff_topic_from_stb loop_consume_diff_topic_from_stb_end: +print ================ test consume from ctb +$loop_cnt = 0 +loop_consume_diff_topic_from_ctb: + ####################################################################################### # clear consume info and consume result #run tsim/tmq/clearConsume.sim # because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb1 +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName @@ -125,10 +154,6 @@ if $rows != 2 then endi ####################################################################################### - -print ================ test consume from ctb -$loop_cnt = 0 -loop_consume_diff_topic_from_ctb: if $loop_cnt == 0 then print == scenario 1: topic_ctb_column $topicList = ' . topic_ctb_column @@ -147,7 +172,7 @@ endi $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb + 1 +$expectmsgcnt = $totalMsgOfCtb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) print == start consumer to pull msgs from stb @@ -176,11 +201,16 @@ $loop_cnt = $loop_cnt + 1 goto loop_consume_diff_topic_from_ctb loop_consume_diff_topic_from_ctb_end: +print ================ test consume from ntb +$loop_cnt = 0 +loop_consume_diff_topic_from_ntb: + ####################################################################################### # clear consume info and consume result #run tsim/tmq/clearConsume.sim # because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb2 +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName @@ -195,10 +225,6 @@ if $rows != 2 then endi ####################################################################################### - -print ================ test consume from ntb -$loop_cnt = 0 -loop_consume_diff_topic_from_ntb: if $loop_cnt == 0 then print == scenario 1: topic_ntb_column $topicList = ' . topic_ntb_column @@ -217,7 +243,7 @@ endi $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb + 1 +$expectmsgcnt = $totalMsgOfNtb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) print == start consumer to pull msgs from stb diff --git a/tests/script/tsim/tmq/basic2.sim b/tests/script/tsim/tmq/basic2.sim new file mode 100644 index 0000000000..b3e9a67c62 --- /dev/null +++ b/tests/script/tsim/tmq/basic2.sim @@ -0,0 +1,219 @@ +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics +#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-1vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 5 +$ifcheckdata = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . ' + +$topicNum = 3 + +#=============================== start consume =============================# + + +print ================ test consume from stb +print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function +$topicList = ' . topic_stb_column +$topicList = $topicList . , +$topicList = $topicList . topic_stb_all +$topicList = $topicList . , +$topicList = $topicList . topic_stb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$totalMsgOfStb = $totalMsgOfStb * $topicNum +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $expectmsgcnt then + return -1 +endi +if $data[0][3] != $expectmsgcnt then + return -1 +endi + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb1 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ctb +print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function +$topicList = ' . topic_ctb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfCtb then + return -1 +endi +if $data[0][3] != $totalMsgOfCtb then + return -1 +endi + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb2 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ntb +print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function +$topicList = ' . topic_ntb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfNtb then + return -1 +endi +if $data[0][3] != $totalMsgOfNtb then + return -1 +endi + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic3.sim b/tests/script/tsim/tmq/basic3.sim new file mode 100644 index 0000000000..7a0ee6f917 --- /dev/null +++ b/tests/script/tsim/tmq/basic3.sim @@ -0,0 +1,278 @@ +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics +#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-4vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 4 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 5 +$ifcheckdata = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . ' + +$cdb_index = 0 +#=============================== start consume =============================# + +print ================ test consume from stb +$loop_cnt = 0 +loop_consume_diff_topic_from_stb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_stb_column + $topicList = ' . topic_stb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_stb_all + $topicList = ' . topic_stb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_stb_function + $topicList = ' . topic_stb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_stb_end +endi + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $expectmsgcnt then + return -1 +endi +if $data[0][3] != $expectmsgcnt then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_stb +loop_consume_diff_topic_from_stb_end: + +print ================ test consume from ctb +$loop_cnt = 0 +loop_consume_diff_topic_from_ctb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ctb_column + $topicList = ' . topic_ctb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ctb_all + $topicList = ' . topic_ctb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ctb_function + $topicList = ' . topic_ctb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ctb_end +endi + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfCtb then + return -1 +endi +if $data[0][3] != $totalMsgOfCtb then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ctb +loop_consume_diff_topic_from_ctb_end: + +print ================ test consume from ntb +$loop_cnt = 0 +loop_consume_diff_topic_from_ntb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ntb_column + $topicList = ' . topic_ntb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ntb_all + $topicList = ' . topic_ntb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ntb_function + $topicList = ' . topic_ntb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ntb_end +endi + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfNtb then + return -1 +endi +if $data[0][3] != $totalMsgOfNtb then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ntb +loop_consume_diff_topic_from_ntb_end: + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic4.sim b/tests/script/tsim/tmq/basic4.sim new file mode 100644 index 0000000000..1af02751b1 --- /dev/null +++ b/tests/script/tsim/tmq/basic4.sim @@ -0,0 +1,216 @@ +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics +#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-4vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 4 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 5 +$ifcheckdata = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . ' + +$topicNum = 3 + +print ================ test consume from stb +print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function +$topicList = ' . topic_stb_column +$topicList = $topicList . , +$topicList = $topicList . topic_stb_all +$topicList = $topicList . , +$topicList = $topicList . topic_stb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$totalMsgOfStb = $totalMsgOfStb * $topicNum +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $expectmsgcnt then + return -1 +endi +if $data[0][3] != $expectmsgcnt then + return -1 +endi + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb1 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ctb +print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function +$topicList = ' . topic_ctb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfCtb then + return -1 +endi +if $data[0][3] != $totalMsgOfCtb then + return -1 +endi + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb2 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ntb +print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function +$topicList = ' . topic_ntb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfNtb then + return -1 +endi +if $data[0][3] != $totalMsgOfNtb then + return -1 +endi + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim b/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim new file mode 100644 index 0000000000..db56bcf743 --- /dev/null +++ b/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim @@ -0,0 +1,88 @@ +# stop all dnodes before start this case +system sh/stop_dnodes.sh + +# deploy dnode 1 +system sh/deploy.sh -n dnode1 -i 1 + +# add some config items for this case +#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +# start dnode 1 +system sh/exec.sh -n dnode1 -s start + +sql connect + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +print == create database $dbName vgroups $vgroups +sql create database $dbName vgroups $vgroups + +#wait database ready +$loop_cnt = 0 +check_db_ready: +if $loop_cnt == 10 then + print ====> database not ready! + return -1 +endi +sql show databases +print ==> rows: $rows +print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] +print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] +if $data(db)[20] != nostrict then + sleep 100 + $loop_cnt = $loop_cnt + 1 + goto check_db_ready +endi + +sql use $dbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi + +print == create super table +sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) +sql show stables +if $rows != 1 then + return -1 +endi + +print == create child table, normal table and insert data +$i = 0 +while $i < $ctbNum + $ctb = $ctbPrefix . $i + $ntb = $ntbPrefix . $i + sql create table $ctb using $stbPrefix tags( $i ) + sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16)) + + $x = 0 + while $x < $rowsPerCtb + $binary = ' . binary- + $binary = $binary . $i + $binary = $binary . ' + + sql insert into $ctb values ($tstart , $i , $x , $binary ) + sql insert into $ntb values ($tstart , $i , $x , $binary ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + $i = $i + 1 + $tstart = 1640966400000 +endw diff --git a/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim b/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim new file mode 100644 index 0000000000..d7fa58558f --- /dev/null +++ b/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim @@ -0,0 +1,88 @@ +# stop all dnodes before start this case +system sh/stop_dnodes.sh + +# deploy dnode 1 +system sh/deploy.sh -n dnode1 -i 1 + +# add some config items for this case +#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +# start dnode 1 +system sh/exec.sh -n dnode1 -s start + +sql connect + +#---- global parameters start ----# +$dbName = db +$vgroups = 4 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +print == create database $dbName vgroups $vgroups +sql create database $dbName vgroups $vgroups + +#wait database ready +$loop_cnt = 0 +check_db_ready: +if $loop_cnt == 10 then + print ====> database not ready! + return -1 +endi +sql show databases +print ==> rows: $rows +print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] +print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] +if $data(db)[20] != nostrict then + sleep 100 + $loop_cnt = $loop_cnt + 1 + goto check_db_ready +endi + +sql use $dbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi + +print == create super table +sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) +sql show stables +if $rows != 1 then + return -1 +endi + +print == create child table, normal table and insert data +$i = 0 +while $i < $ctbNum + $ctb = $ctbPrefix . $i + $ntb = $ntbPrefix . $i + sql create table $ctb using $stbPrefix tags( $i ) + sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16)) + + $x = 0 + while $x < $rowsPerCtb + $binary = ' . binary- + $binary = $binary . $i + $binary = $binary . ' + + sql insert into $ctb values ($tstart , $i , $x , $binary ) + sql insert into $ntb values ($tstart , $i , $x , $binary ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + $i = $i + 1 + $tstart = 1640966400000 +endw diff --git a/tests/script/tsim/tmq/prepareBasicEnv.sim b/tests/script/tsim/tmq/prepareBasicEnv.sim index 066d7d4ab0..c4f706e262 100644 --- a/tests/script/tsim/tmq/prepareBasicEnv.sim +++ b/tests/script/tsim/tmq/prepareBasicEnv.sim @@ -14,7 +14,7 @@ sql connect #---- global parameters start ----# $dbName = db -$vgroups = 1 +$vgroups = 4 $stbPrefix = stb $ctbPrefix = ctb $ntbPrefix = ntb diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index f9ba0e418c..9f6c1a59eb 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -14,13 +14,14 @@ */ #include +#include #include #include #include #include #include #include -// #include +#include #include "taos.h" #include "taoserror.h" @@ -99,11 +100,11 @@ void initLogFile() { // FILE *fp = fopen(g_stConfInfo.resultFileName, "a"); char file[256]; sprintf(file, "%s/../log/tmqlog.txt", configDir); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_STREAM); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_TEXT | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); if (NULL == pFile) { fprintf(stderr, "Failed to open %s for save result\n", "./tmqlog.txt"); - exit(-1); - } + exit -1; + }; g_fp = pFile; } @@ -119,6 +120,7 @@ void saveConfigToLogFile() { taosFprintfFile(g_fp, "# showMsgFlag: %d\n", g_stConfInfo.showMsgFlag); taosFprintfFile(g_fp, "# showRowFlag: %d\n", g_stConfInfo.showRowFlag); taosFprintfFile(g_fp, "# consumeDelay: %d\n", g_stConfInfo.consumeDelay); + taosFprintfFile(g_fp, "# numOfThread: %d\n", g_stConfInfo.numOfThread); for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId); @@ -483,6 +485,7 @@ int main(int32_t argc, char* argv[]) { taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); // pthread_create one thread to consume + taosFprintfFile(g_fp, "==== create %d consume thread ====\n", g_stConfInfo.numOfThread); for (int32_t i = 0; i < g_stConfInfo.numOfThread; ++i) { taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, consumeThreadFunc, (void *)(&(g_stConfInfo.stThreads[i]))); } @@ -493,7 +496,7 @@ int main(int32_t argc, char* argv[]) { //printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt); - taosFprintfFile(g_fp, "\n"); + taosFprintfFile(g_fp, "==== close tmqlog ====\n"); taosCloseFile(&g_fp); return 0; From 98cccc17a759ead063d3c117840bfc2e4ed2a94a Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 22 Apr 2022 17:09:26 +0800 Subject: [PATCH 67/83] [test: modify tmq test cases] --- tests/script/jenkins/basic.txt | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 506abf3fc5..3159569f32 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -54,19 +54,10 @@ ./test.sh -f tsim/table/basic1.sim # ---- tmq -./test.sh -f tsim/tmq/basic.sim -#./test.sh -f tsim/tmq/basic1.sim -#./test.sh -f tsim/tmq/oneTopic.sim -#./test.sh -f tsim/tmq/multiTopic.sim - -#./test.sh -f tsim/tmq/mainConsumerInMultiTopic.sim -#./test.sh -f tsim/tmq/mainConsumerInOneTopic.sim - -#fail ./test.sh -f tsim/tmq/main2Con1Cgrp1TopicFrCtb.sim -#fail ./test.sh -f tsim/tmq/main2Con1Cgrp1TopicFrStb.sim -#./test.sh -f tsim/tmq/main2Con1Cgrp2TopicFrCtb.sim -#./test.sh -f tsim/tmq/main2Con1Cgrp2TopicFrStb.sim - +./test.sh -f tsim/tmq/basic1.sim +./test.sh -f tsim/tmq/basic2.sim +./test.sh -f tsim/tmq/basic3.sim +./test.sh -f tsim/tmq/basic4.sim # --- stable ./test.sh -f tsim/stable/disk.sim From 3aa40bd504cb6fbfd8c18e72ee3634a27dbc67ee Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 22 Apr 2022 17:13:15 +0800 Subject: [PATCH 68/83] [test: rm some discard file] --- tests/script/tsim/tmq/basic.sim | 86 ------ tests/script/tsim/tmq/insertDataV1.sim | 44 --- tests/script/tsim/tmq/insertDataV4.sim | 44 --- tests/script/tsim/tmq/insertFixedDataV2.sim | 51 ---- tests/script/tsim/tmq/insertFixedDataV4.sim | 51 ---- .../tsim/tmq/main2Con1Cgrp1TopicFrCtb.sim | 269 ------------------ .../tsim/tmq/main2Con1Cgrp1TopicFrStb.sim | 244 ---------------- .../tsim/tmq/main2Con1Cgrp2TopicFrCtb.sim | 239 ---------------- .../tsim/tmq/main2Con1Cgrp2TopicFrStb.sim | 244 ---------------- .../tsim/tmq/mainConsumerInMultiTopic.sim | 217 -------------- .../tsim/tmq/mainConsumerInOneTopic.sim | 246 ---------------- tests/script/tsim/tmq/multiTopic.sim | 228 --------------- tests/script/tsim/tmq/oneTopic.sim | 268 ----------------- .../script/tsim/tmq/overlapTopic2Con1Cgrp.sim | 255 ----------------- tests/script/tsim/tmq/prepareBasicEnv.sim | 88 ------ 15 files changed, 2574 deletions(-) delete mode 100644 tests/script/tsim/tmq/basic.sim delete mode 100644 tests/script/tsim/tmq/insertDataV1.sim delete mode 100644 tests/script/tsim/tmq/insertDataV4.sim delete mode 100644 tests/script/tsim/tmq/insertFixedDataV2.sim delete mode 100644 tests/script/tsim/tmq/insertFixedDataV4.sim delete mode 100644 tests/script/tsim/tmq/main2Con1Cgrp1TopicFrCtb.sim delete mode 100644 tests/script/tsim/tmq/main2Con1Cgrp1TopicFrStb.sim delete mode 100644 tests/script/tsim/tmq/main2Con1Cgrp2TopicFrCtb.sim delete mode 100644 tests/script/tsim/tmq/main2Con1Cgrp2TopicFrStb.sim delete mode 100644 tests/script/tsim/tmq/mainConsumerInMultiTopic.sim delete mode 100644 tests/script/tsim/tmq/mainConsumerInOneTopic.sim delete mode 100644 tests/script/tsim/tmq/multiTopic.sim delete mode 100644 tests/script/tsim/tmq/oneTopic.sim delete mode 100644 tests/script/tsim/tmq/overlapTopic2Con1Cgrp.sim delete mode 100644 tests/script/tsim/tmq/prepareBasicEnv.sim diff --git a/tests/script/tsim/tmq/basic.sim b/tests/script/tsim/tmq/basic.sim deleted file mode 100644 index 2f16b7c737..0000000000 --- a/tests/script/tsim/tmq/basic.sim +++ /dev/null @@ -1,86 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c wal -v 1 -system sh/exec.sh -n dnode1 -s start -sleep 500 -sql connect - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 100 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - -#root@trd02 /data2/dnode $ tmq_demo --help -#Used to tmq_demo -# -c Configuration directory, default is -# -d The name of the database to be created, default is tmqdb -# -s The name of the super table to be created, default is stb -# -f The file of result, default is ./tmqResult.txt -# -w The path of vnode of wal, default is /data2/dnode/data/vnodes/vnode2/wal -# -t numOfThreads, default is 1 -# -n numOfTables, default is 1 -# -v numOfVgroups, default is 1 -# -a runMode, default is 0 -# -l numOfColumn, default is 1 -# -q ratio, default is 1.000000 -# -b batchNumOfRow, default is 1 -# -r totalRowsOfPerTbl, default is 10000 -# -m startTimestamp, default is 1640966400000 [2022-01-01 00:00:00] -# -g showMsgFlag, default is 0 -# - -system_content echo -n \$BUILD_DIR -$tmq_demo = $system_content . /build/bin/tmq_demo -system_content echo -n \$SIM_DIR -$tsim_cfg = $system_content . /tsim/cfg -$sim_wal = $system_content . /dnode1/data/vnode/vnode4/wal - -print cmd===> system_content $tmq_demo -sim 1 -b 100 -c $tsim_cfg -w $sim_wal -system_content $tmq_demo -sim 1 -b 100 -c $tsim_cfg -w $sim_wal -print cmd result----> $system_content -if $system_content != @{consume success: 100}@ then - return -1 -endi - -sql show databases -print ===> $rows $data00 $data01 $data02 $data03 -if $rows != 3 then - return -1 -endi -if $data20 != tmqdb then - return -1 -endi - -sql use tmqdb -sql show tables -print ===> $rows $data00 $data01 $data02 $data03 -if $rows != 1 then - return -1 -endi -if $data00 != stb0 then - return -1 -endi - -sql select count(*) from stb0 -print ===> $rows $data00 $data01 $data02 $data03 -if $rows != 1 then - return -1 -endi -if $data00 != 10000 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/insertDataV1.sim b/tests/script/tsim/tmq/insertDataV1.sim deleted file mode 100644 index 0df74d53f8..0000000000 --- a/tests/script/tsim/tmq/insertDataV1.sim +++ /dev/null @@ -1,44 +0,0 @@ - -sql connect - -print ================ insert data -$dbNamme = d0 -$tbPrefix = ct -$tbNum = 10 -$rowNum = 100 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 - -sql use $dbNamme - -$loop_cnt = 0 - -loop_insert: -print ====> loop $loop_cnt insert -$loop_cnt = $loop_cnt + 1 - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - - $x = 0 - while $x < $rowNum - $binary = ' . binary - $binary = $binary . $i - $binary = $binary . ' - - #print ====> insert into $tb values ($tstart , $i , $x , $binary ) - #print ====> insert into ntb values ($tstart , $i , $x , $binary ) - sql insert into $tb values ($tstart , $i , $x , $binary ) - sql insert into ntb values ($tstart , 999 , 999 , 'binary-ntb' ) - $tstart = $tstart + 1 - $x = $x + 1 - endw - - print ====> insert rows: $rowNum into $tb and ntb - - $i = $i + 1 -# $tstart = 1640966400000 -endw -goto loop_insert - - diff --git a/tests/script/tsim/tmq/insertDataV4.sim b/tests/script/tsim/tmq/insertDataV4.sim deleted file mode 100644 index dbd52f56b8..0000000000 --- a/tests/script/tsim/tmq/insertDataV4.sim +++ /dev/null @@ -1,44 +0,0 @@ - -sql connect - -print ================ insert data -$dbNamme = d1 -$tbPrefix = ct -$tbNum = 10 -$rowNum = 100 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 - -sql use $dbNamme - -$loop_cnt = 0 - -loop_insert: -print ====> loop $loop_cnt insert -$loop_cnt = $loop_cnt + 1 - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - - $x = 0 - while $x < $rowNum - $binary = ' . binary - $binary = $binary . $i - $binary = $binary . ' - - #print ====> insert into $tb values ($tstart , $i , $x , $binary ) - #print ====> insert into ntb values ($tstart , $i , $x , $binary ) - sql insert into $tb values ($tstart , $i , $x , $binary ) - sql insert into ntb values ($tstart , 999 , 999 , 'binary-ntb' ) - $tstart = $tstart + 1 - $x = $x + 1 - endw - - #print ====> insert rows: $rowNum into $tb and ntb - - $i = $i + 1 -# $tstart = 1640966400000 -endw -goto loop_insert - - diff --git a/tests/script/tsim/tmq/insertFixedDataV2.sim b/tests/script/tsim/tmq/insertFixedDataV2.sim deleted file mode 100644 index a93be3e5a6..0000000000 --- a/tests/script/tsim/tmq/insertFixedDataV2.sim +++ /dev/null @@ -1,51 +0,0 @@ - -sql connect - -print ================ insert data -$dbNamme = d0 -$tbPrefix = ct -$tbNum = 10 -$rowNum = 100 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 - -$loopInsertNum = 10 - -sql use $dbNamme - -$loopIndex = 0 - -loop_insert: -print ====> loop $loopIndex insert -$loopIndex = $loopIndex + 1 - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - - $x = 0 - while $x < $rowNum - $binary = ' . binary - $binary = $binary . $i - $binary = $binary . ' - - #print ====> insert into $tb values ($tstart , $i , $x , $binary ) - #print ====> insert into ntb values ($tstart , $i , $x , $binary ) - sql insert into $tb values ($tstart , $i , $x , $binary ) - sql insert into ntb values ($tstart , 999 , 999 , 'binary-ntb' ) - $tstart = $tstart + 1 - $x = $x + 1 - endw - - #print ====> insert rows: $rowNum into $tb and ntb - - $i = $i + 1 -# $tstart = 1640966400000 -endw - - -if $loopIndex < $loopInsertNum then - goto loop_insert -endi - -print ====> insert data end =========== - diff --git a/tests/script/tsim/tmq/insertFixedDataV4.sim b/tests/script/tsim/tmq/insertFixedDataV4.sim deleted file mode 100644 index 9f7f86747f..0000000000 --- a/tests/script/tsim/tmq/insertFixedDataV4.sim +++ /dev/null @@ -1,51 +0,0 @@ - -sql connect - -print ================ insert data -$dbNamme = d1 -$tbPrefix = ct -$tbNum = 10 -$rowNum = 100 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 - -$loopInsertNum = 10 - -sql use $dbNamme - -$loopIndex = 0 - -loop_insert: -print ====> loop $loopIndex insert -$loopIndex = $loopIndex + 1 - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - - $x = 0 - while $x < $rowNum - $binary = ' . binary - $binary = $binary . $i - $binary = $binary . ' - - #print ====> insert into $tb values ($tstart , $i , $x , $binary ) - #print ====> insert into ntb values ($tstart , $i , $x , $binary ) - sql insert into $tb values ($tstart , $i , $x , $binary ) - sql insert into ntb values ($tstart , 999 , 999 , 'binary-ntb' ) - $tstart = $tstart + 1 - $x = $x + 1 - endw - - #print ====> insert rows: $rowNum into $tb and ntb - - $i = $i + 1 -# $tstart = 1640966400000 -endw - - -if $loopIndex < $loopInsertNum then - goto loop_insert -endi - -print ====> insert data end =========== - diff --git a/tests/script/tsim/tmq/main2Con1Cgrp1TopicFrCtb.sim b/tests/script/tsim/tmq/main2Con1Cgrp1TopicFrCtb.sim deleted file mode 100644 index cf53cc7bcd..0000000000 --- a/tests/script/tsim/tmq/main2Con1Cgrp1TopicFrCtb.sim +++ /dev/null @@ -1,269 +0,0 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -# scene1: vgroups=2, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene2: vgroups=2, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene3: vgroups=4, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene4: vgroups=4, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## -######## This test case include scene1 and scene3 -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## - -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - -sql connect - -$loop_cnt = 0 -$vgroups = 2 -$dbNamme = d0 -loop_vgroups: -print =============== create database $dbNamme vgroups $vgroups -sql create database $dbNamme vgroups $vgroups -sql show databases -print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 -print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 - -if $loop_cnt == 0 then - if $rows != 2 then - return -1 - endi - if $data02 != 2 then # vgroups - print vgroups: $data02 - return -1 - endi -else - if $rows != 3 then - return -1 - endi - if $data00 == d1 then - if $data02 != 4 then # vgroups - print vgroups: $data02 - return -1 - endi - else - if $data12 != 4 then # vgroups - print vgroups: $data12 - return -1 - endi - endi -endi - -sql use $dbNamme - -print =============== create super table -sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) - -sql show stables -if $rows != 1 then - return -1 -endi - -print =============== create child table -$tbPrefix = ct -$tbNum = 100 - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using stb tags( $i ) - $i = $i + 1 -endw - -print =============== create normal table -sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) - -print =============== create topics from child table - -sql create topic topic_ctb_column as select ts, c1, c3 from ct0 -sql create topic topic_ctb_all as select * from ct0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 - -#sql create topic topic_ntb_column as select ts, c1, c3 from ntb -#sql create topic topic_ntb_all as select * from ntb -#sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb - -sql show tables -if $rows != 101 then - return -1 -endi - -print =============== run_back insert data - -if $loop_cnt == 0 then - run_back tsim/tmq/insertFixedDataV2.sim -else - run_back tsim/tmq/insertFixedDataV4.sim -endi - -#sleep 1000 - -#$rowNum = 1000 -#$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -# -#$i = 0 -#while $i < $tbNum -# $tb = $tbPrefix . $i -# -# $x = 0 -# while $x < $rowNum -# $c = $x / 10 -# $c = $c * 10 -# $c = $x - $c -# -# $binary = ' . binary -# $binary = $binary . $c -# $binary = $binary . ' -# -# sql insert into $tb values ($tstart , $c , $x , $binary ) -# sql insert into ntb values ($tstart , $c , $x , $binary ) -# $tstart = $tstart + 1 -# $x = $x + 1 -# endw -# -# $i = $i + 1 -# $tstart = 1640966400000 -#endw - -#root@trd02 /home $ tmq_sim --help -# -c Configuration directory, default is -# -d The name of the database for cosumer, no default -# -t The topic string for cosumer, no default -# -k The key-value string for cosumer, no default -# -g showMsgFlag, default is 0 -# - -$tbNum = 10 -$consumeDelay = 10 -$expectMsgCntFromCtb = 300 -$expectMsgCntFromStb = $expectMsgCntFromCtb * $tbNum -print consumeDelay: $consumeDelay -print insert data child num: $tbNum -print expectMsgCntFromCtb: $expectMsgCntFromCtb -print expectMsgCntFromStb: $expectMsgCntFromStb - -# supported key: -# group.id: -# enable.auto.commit: -# auto.offset.reset: -# td.connect.ip: -# td.connect.user:root -# td.connect.pass:taosdata -# td.connect.port:6030 -# td.connect.db:db - -$expect_result = @{consume success: @ -$expect_result = $expect_result . $rowNum -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result - -system_content echo -n \$BUILD_DIR -$tmq_sim = $system_content . /build/bin/tmq_sim -$tsim_cfg = $system_content . /tsim/cfg - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 0 -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 0 -print cmd result----> $system_content -if $system_content != success then - return -1 -endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 0 -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 0 -print cmd result----> $system_content -if $system_content != success then - return -1 -endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 0 -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 0 -print cmd result----> $system_content -if $system_content != success then - return -1 -endi - -#$expect_result = @{consume success: @ -#$expect_result = $expect_result . $totalMsgCnt -#$expect_result = $expect_result . @, @ -#$expect_result = $expect_result . 0} -#print expect_result----> $expect_result -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#print cmd result----> $system_content -#if $system_content != success then -# return -1 -#endi -# -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#print cmd result----> $system_content -#if $system_content != success then -# return -1 -#endi -# -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#print cmd result----> $system_content -#if $system_content != success then -# return -1 -#endi -# -#$expect_result = @{consume success: @ -#$expect_result = $expect_result . $expectConsumeMsgCnt -#$expect_result = $expect_result . @, @ -#$expect_result = $expect_result . 0} -#print expect_result----> $expect_result -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -#print cmd result----> $system_content -#if $system_content != success then -# return -1 -#endi -# -##print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -##system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -##print cmd result----> $system_content -###if $system_content != @{consume success: 10000, 0}@ then -##if $system_content != success then -## return -1 -##endi -# -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -#print cmd result----> $system_content -##if $system_content != @{consume success: 10000, 0}@ then -#if $system_content != success then -# return -1 -#endi - -if $loop_cnt == 0 then - $loop_cnt = 1 - $vgroups = 4 - $dbNamme = d1 - goto loop_vgroups -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/main2Con1Cgrp1TopicFrStb.sim b/tests/script/tsim/tmq/main2Con1Cgrp1TopicFrStb.sim deleted file mode 100644 index fe37ffffcb..0000000000 --- a/tests/script/tsim/tmq/main2Con1Cgrp1TopicFrStb.sim +++ /dev/null @@ -1,244 +0,0 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -# scene1: vgroups=2, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene2: vgroups=2, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene3: vgroups=4, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene4: vgroups=4, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## -######## This test case include scene1 and scene3 -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## - -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - -sql connect - -$loop_cnt = 0 -$vgroups = 2 -$dbNamme = d0 -loop_vgroups: -print =============== create database $dbNamme vgroups $vgroups -sql create database $dbNamme vgroups $vgroups -sql use $dbNamme - -print =============== create super table -sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) - -sql show stables -if $rows != 1 then - return -1 -endi - -print =============== create child table -$tbPrefix = ct -$tbNum = 100 - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using stb tags( $i ) - $i = $i + 1 -endw - -print =============== create normal table -sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) - -print =============== create multi topics. notes: now only support: -print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb -print =============== will support: * from stb - -sql create topic topic_stb_column as select ts, c1, c3 from stb -#sql create topic topic_stb_all as select * from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -sql create topic topic_ctb_column as select ts, c1, c3 from ct0 -sql create topic topic_ctb_all as select * from ct0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 - -sql create topic topic_ntb_column as select ts, c1, c3 from ntb -sql create topic topic_ntb_all as select * from ntb -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb - -sql show tables -if $rows != 101 then - return -1 -endi - -print =============== run_back insert data - -if $loop_cnt == 0 then - run_back tsim/tmq/insertFixedDataV2.sim -else - run_back tsim/tmq/insertFixedDataV4.sim -endi - -#sleep 1000 - -#$rowNum = 1000 -#$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -# -#$i = 0 -#while $i < $tbNum -# $tb = $tbPrefix . $i -# -# $x = 0 -# while $x < $rowNum -# $c = $x / 10 -# $c = $c * 10 -# $c = $x - $c -# -# $binary = ' . binary -# $binary = $binary . $c -# $binary = $binary . ' -# -# sql insert into $tb values ($tstart , $c , $x , $binary ) -# sql insert into ntb values ($tstart , $c , $x , $binary ) -# $tstart = $tstart + 1 -# $x = $x + 1 -# endw -# -# $i = $i + 1 -# $tstart = 1640966400000 -#endw - -#root@trd02 /home $ tmq_sim --help -# -c Configuration directory, default is -# -d The name of the database for cosumer, no default -# -t The topic string for cosumer, no default -# -k The key-value string for cosumer, no default -# -g showMsgFlag, default is 0 -# - -$tbNum = 10 -$consumeDelay = 10 -$expectMsgCntFromCtb = 300 -$expectMsgCntFromStb = $expectMsgCntFromCtb * $tbNum -print consumeDelay: $consumeDelay -print insert data child num: $tbNum -print expectMsgCntFromCtb: $expectMsgCntFromCtb -print expectMsgCntFromStb: $expectMsgCntFromStb - -# supported key: -# group.id: -# enable.auto.commit: -# auto.offset.reset: -# td.connect.ip: -# td.connect.user:root -# td.connect.pass:taosdata -# td.connect.port:6030 -# td.connect.db:db - -#$expect_result = @{consume success: @ -#$expect_result = $expect_result . $rowNum -#$expect_result = $expect_result . @, @ -#$expect_result = $expect_result . 0} -#print expect_result----> $expect_result - -system_content echo -n \$BUILD_DIR -$tmq_sim = $system_content . /build/bin/tmq_sim -$tsim_cfg = $system_content . /tsim/cfg - -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#print cmd result----> $system_content -#if $system_content != success then -# return -1 -#endi -# -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#print cmd result----> $system_content -#if $system_content != success then -# return -1 -#endi -# -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#print cmd result----> $system_content -#if $system_content != success then -# return -1 -#endi -# -#$expect_result = @{consume success: @ -#$expect_result = $expect_result . $totalMsgCnt -#$expect_result = $expect_result . @, @ -#$expect_result = $expect_result . 0} -#print expect_result----> $expect_result -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#print cmd result----> $system_content -#if $system_content != success then -# return -1 -#endi -# -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#print cmd result----> $system_content -#if $system_content != success then -# return -1 -#endi -# -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#print cmd result----> $system_content -#if $system_content != success then -# return -1 -#endi - -$expect_result = @{consume success: @ -$expect_result = $expect_result . $expectMsgCntFromStb -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 0 -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 0 -print cmd result----> $system_content -if $system_content != success then - return -1 -endi - -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 0 -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 0 -#print cmd result----> $system_content -##if $system_content != @{consume success: 10000, 0}@ then -#if $system_content != success then -# return -1 -#endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 0 -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 0 -print cmd result----> $system_content -#if $system_content != @{consume success: 10000, 0}@ then -if $system_content != success then - return -1 -endi -if $loop_cnt == 0 then - $loop_cnt = 1 - $vgroups = 4 - $dbNamme = d1 - goto loop_vgroups -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/main2Con1Cgrp2TopicFrCtb.sim b/tests/script/tsim/tmq/main2Con1Cgrp2TopicFrCtb.sim deleted file mode 100644 index 76f3c93e27..0000000000 --- a/tests/script/tsim/tmq/main2Con1Cgrp2TopicFrCtb.sim +++ /dev/null @@ -1,239 +0,0 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -# scene1: vgroups=2, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene2: vgroups=2, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene3: vgroups=4, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene4: vgroups=4, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## -######## This test case include scene1 and scene3 -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## - -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - -sql connect - -$loop_cnt = 0 -$vgroups = 2 -$dbNamme = d0 -loop_vgroups: -print =============== create database $dbNamme vgroups $vgroups -sql create database $dbNamme vgroups $vgroups -sql use $dbNamme - -print =============== create super table -sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) - -sql show stables -if $rows != 1 then - return -1 -endi - -print =============== create child table -$tbPrefix = ct -$tbNum = 100 - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using stb tags( $i ) - $i = $i + 1 -endw - -print =============== create normal table -sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) - -print =============== create topics from child table - -sql create topic topic_ctb_column as select ts, c1, c3 from ct0 -sql create topic topic_ctb_all as select * from ct0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 - -#sql create topic topic_ntb_column as select ts, c1, c3 from ntb -#sql create topic topic_ntb_all as select * from ntb -#sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb - -sql show tables -if $rows != 101 then - return -1 -endi - -print =============== run_back insert data - -if $loop_cnt == 0 then - run_back tsim/tmq/insertFixedDataV2.sim -else - run_back tsim/tmq/insertFixedDataV4.sim -endi - -#sleep 1000 - -#$rowNum = 1000 -#$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -# -#$i = 0 -#while $i < $tbNum -# $tb = $tbPrefix . $i -# -# $x = 0 -# while $x < $rowNum -# $c = $x / 10 -# $c = $c * 10 -# $c = $x - $c -# -# $binary = ' . binary -# $binary = $binary . $c -# $binary = $binary . ' -# -# sql insert into $tb values ($tstart , $c , $x , $binary ) -# sql insert into ntb values ($tstart , $c , $x , $binary ) -# $tstart = $tstart + 1 -# $x = $x + 1 -# endw -# -# $i = $i + 1 -# $tstart = 1640966400000 -#endw - -#root@trd02 /home $ tmq_sim --help -# -c Configuration directory, default is -# -d The name of the database for cosumer, no default -# -t The topic string for cosumer, no default -# -k The key-value string for cosumer, no default -# -g showMsgFlag, default is 0 -# - -$tbNum = 10 -$consumeDelay = 10 -$expectMsgCntFromCtb = 300 -$expectMsgCntFromStb = $expectMsgCntFromCtb * $tbNum -print consumeDelay: $consumeDelay -print insert data child num: $tbNum -print expectMsgCntFromCtb: $expectMsgCntFromCtb -print expectMsgCntFromStb: $expectMsgCntFromStb - -# supported key: -# group.id: -# enable.auto.commit: -# auto.offset.reset: -# td.connect.ip: -# td.connect.user:root -# td.connect.pass:taosdata -# td.connect.port:6030 -# td.connect.db:db - -$expect_result = @{consume success: @ -$expect_result = $expect_result . $rowNum -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result - -system_content echo -n \$BUILD_DIR -$tmq_sim = $system_content . /build/bin/tmq_sim -$tsim_cfg = $system_content . /tsim/cfg - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1 -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1 -print cmd result----> $system_content -if $system_content != success then - return -1 -endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1 -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1 -print cmd result----> $system_content -if $system_content != success then - return -1 -endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1 -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1 -print cmd result----> $system_content -if $system_content != success then - return -1 -endi - -#$expect_result = @{consume success: @ -#$expect_result = $expect_result . $totalMsgCnt -#$expect_result = $expect_result . @, @ -#$expect_result = $expect_result . 0} -#print expect_result----> $expect_result -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1 -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1 -#print cmd result----> $system_content -#if $system_content != success then -# return -1 -#endi -# -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#print cmd result----> $system_content -#if $system_content != success then -# return -1 -#endi -# -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#print cmd result----> $system_content -#if $system_content != success then -# return -1 -#endi -# -#$expect_result = @{consume success: @ -#$expect_result = $expect_result . $expectConsumeMsgCnt -#$expect_result = $expect_result . @, @ -#$expect_result = $expect_result . 0} -#print expect_result----> $expect_result -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -#print cmd result----> $system_content -#if $system_content != success then -# return -1 -#endi -# -##print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -##system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -##print cmd result----> $system_content -###if $system_content != @{consume success: 10000, 0}@ then -##if $system_content != success then -## return -1 -##endi -# -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -#print cmd result----> $system_content -##if $system_content != @{consume success: 10000, 0}@ then -#if $system_content != success then -# return -1 -#endi - -if $loop_cnt == 0 then - $loop_cnt = 1 - $vgroups = 4 - $dbNamme = d1 - goto loop_vgroups -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/main2Con1Cgrp2TopicFrStb.sim b/tests/script/tsim/tmq/main2Con1Cgrp2TopicFrStb.sim deleted file mode 100644 index 86e0b32980..0000000000 --- a/tests/script/tsim/tmq/main2Con1Cgrp2TopicFrStb.sim +++ /dev/null @@ -1,244 +0,0 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -# scene1: vgroups=2, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene2: vgroups=2, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene3: vgroups=4, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene4: vgroups=4, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## -######## This test case include scene1 and scene3 -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## - -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - -sql connect - -$loop_cnt = 0 -$vgroups = 2 -$dbNamme = d0 -loop_vgroups: -print =============== create database $dbNamme vgroups $vgroups -sql create database $dbNamme vgroups $vgroups -sql use $dbNamme - -print =============== create super table -sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) - -sql show stables -if $rows != 1 then - return -1 -endi - -print =============== create child table -$tbPrefix = ct -$tbNum = 100 - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using stb tags( $i ) - $i = $i + 1 -endw - -print =============== create normal table -sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) - -print =============== create multi topics. notes: now only support: -print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb -print =============== will support: * from stb - -sql create topic topic_stb_column as select ts, c1, c3 from stb -#sql create topic topic_stb_all as select * from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -sql create topic topic_ctb_column as select ts, c1, c3 from ct0 -sql create topic topic_ctb_all as select * from ct0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 - -sql create topic topic_ntb_column as select ts, c1, c3 from ntb -sql create topic topic_ntb_all as select * from ntb -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb - -sql show tables -if $rows != 101 then - return -1 -endi - -print =============== run_back insert data - -if $loop_cnt == 0 then - run_back tsim/tmq/insertFixedDataV2.sim -else - run_back tsim/tmq/insertFixedDataV4.sim -endi - -#sleep 1000 - -#$rowNum = 1000 -#$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -# -#$i = 0 -#while $i < $tbNum -# $tb = $tbPrefix . $i -# -# $x = 0 -# while $x < $rowNum -# $c = $x / 10 -# $c = $c * 10 -# $c = $x - $c -# -# $binary = ' . binary -# $binary = $binary . $c -# $binary = $binary . ' -# -# sql insert into $tb values ($tstart , $c , $x , $binary ) -# sql insert into ntb values ($tstart , $c , $x , $binary ) -# $tstart = $tstart + 1 -# $x = $x + 1 -# endw -# -# $i = $i + 1 -# $tstart = 1640966400000 -#endw - -#root@trd02 /home $ tmq_sim --help -# -c Configuration directory, default is -# -d The name of the database for cosumer, no default -# -t The topic string for cosumer, no default -# -k The key-value string for cosumer, no default -# -g showMsgFlag, default is 0 -# - -$tbNum = 10 -$consumeDelay = 10 -$expectMsgCntFromCtb = 300 -$expectMsgCntFromStb = $expectMsgCntFromCtb * $tbNum -print consumeDelay: $consumeDelay -print insert data child num: $tbNum -print expectMsgCntFromCtb: $expectMsgCntFromCtb -print expectMsgCntFromStb: $expectMsgCntFromStb - -# supported key: -# group.id: -# enable.auto.commit: -# auto.offset.reset: -# td.connect.ip: -# td.connect.user:root -# td.connect.pass:taosdata -# td.connect.port:6030 -# td.connect.db:db - -#$expect_result = @{consume success: @ -#$expect_result = $expect_result . $rowNum -#$expect_result = $expect_result . @, @ -#$expect_result = $expect_result . 0} -#print expect_result----> $expect_result - -system_content echo -n \$BUILD_DIR -$tmq_sim = $system_content . /build/bin/tmq_sim -$tsim_cfg = $system_content . /tsim/cfg - -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#print cmd result----> $system_content -#if $system_content != success then -# return -1 -#endi -# -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#print cmd result----> $system_content -#if $system_content != success then -# return -1 -#endi -# -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#print cmd result----> $system_content -#if $system_content != success then -# return -1 -#endi -# -#$expect_result = @{consume success: @ -#$expect_result = $expect_result . $totalMsgCnt -#$expect_result = $expect_result . @, @ -#$expect_result = $expect_result . 0} -#print expect_result----> $expect_result -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#print cmd result----> $system_content -#if $system_content != success then -# return -1 -#endi -# -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#print cmd result----> $system_content -#if $system_content != success then -# return -1 -#endi -# -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -#print cmd result----> $system_content -#if $system_content != success then -# return -1 -#endi - -$expect_result = @{consume success: @ -$expect_result = $expect_result . $expectMsgCntFromStb -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 1 -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 1 -print cmd result----> $system_content -if $system_content != success then - return -1 -endi - -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 1 -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 1 -#print cmd result----> $system_content -##if $system_content != @{consume success: 10000, 0}@ then -#if $system_content != success then -# return -1 -#endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 1 -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 1 -print cmd result----> $system_content -#if $system_content != @{consume success: 10000, 0}@ then -if $system_content != success then - return -1 -endi -if $loop_cnt == 0 then - $loop_cnt = 1 - $vgroups = 4 - $dbNamme = d1 - goto loop_vgroups -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/mainConsumerInMultiTopic.sim b/tests/script/tsim/tmq/mainConsumerInMultiTopic.sim deleted file mode 100644 index d846632da4..0000000000 --- a/tests/script/tsim/tmq/mainConsumerInMultiTopic.sim +++ /dev/null @@ -1,217 +0,0 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -# scene1: vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene2: vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene3: vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene4: vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## -######## This test case include scene2 and scene4 -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## - -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - -sql connect - -$loop_cnt = 0 -$vgroups = 1 -$dbNamme = d0 -loop_vgroups: -print =============== create database $dbNamme vgroups $vgroups -sql create database $dbNamme vgroups $vgroups -sql use $dbNamme - -print =============== create super table -sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) - -sql show stables -if $rows != 1 then - return -1 -endi - -print =============== create child table -$tbPrefix = ct -$tbNum = 100 - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using stb tags( $i ) - $i = $i + 1 -endw - -print =============== create normal table -sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) - -print =============== create multi topics. notes: now only support: -print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb -print =============== will support: * from stb - -sql create topic topic_stb_column as select ts, c1, c3 from stb -#sql create topic topic_stb_all as select * from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -sql create topic topic_ctb_column as select ts, c1, c3 from ct0 -sql create topic topic_ctb_all as select * from ct0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 - -sql create topic topic_ntb_column as select ts, c1, c3 from ntb -sql create topic topic_ntb_all as select * from ntb -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb - -sql show tables -if $rows != 101 then - return -1 -endi - -print =============== run_back insert data - -if $loop_cnt == 0 then - run_back tsim/tmq/insertFixedDataV2.sim -else - run_back tsim/tmq/insertFixedDataV4.sim -endi - -#sleep 1000 - -#$rowNum = 1000 -#$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -# -#$i = 0 -#while $i < $tbNum -# $tb = $tbPrefix . $i -# -# $x = 0 -# while $x < $rowNum -# $c = $x / 10 -# $c = $c * 10 -# $c = $x - $c -# -# $binary = ' . binary -# $binary = $binary . $c -# $binary = $binary . ' -# -# sql insert into $tb values ($tstart , $c , $x , $binary ) -# sql insert into ntb values ($tstart , $c , $x , $binary ) -# $tstart = $tstart + 1 -# $x = $x + 1 -# endw -# -# $i = $i + 1 -# $tstart = 1640966400000 -#endw - -#root@trd02 /home $ tmq_sim --help -# -c Configuration directory, default is -# -d The name of the database for cosumer, no default -# -t The topic string for cosumer, no default -# -k The key-value string for cosumer, no default -# -g showMsgFlag, default is 0 -# - -$tbNum = 10 -$consumeDelay = 5 -$expectMsgCntFromCtb = 1000 -$expectMsgCntFromNtb = 1000 -$expectMsgCntFromStb = $expectMsgCntFromCtb * $tbNum -print consumeDelay: $consumeDelay -print insert data child num: $tbNum -print expectMsgCntFromCtb: $expectMsgCntFromCtb -print expectMsgCntFromStb: $expectMsgCntFromStb - - -# supported key: -# group.id: -# enable.auto.commit: -# auto.offset.reset: -# td.connect.ip: -# td.connect.user:root -# td.connect.pass:taosdata -# td.connect.port:6030 -# td.connect.db:db - -$numOfTopics = 2 -$expectMsgCntFromStb = $expectMsgCntFromStb * $numOfTopics -$expect_result = @{consume success: @ -$expect_result = $expect_result . $expectMsgCntFromStb -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result - -system_content echo -n \$BUILD_DIR -$tmq_sim = $system_content . /build/bin/tmq_sim -$tsim_cfg = $system_content . /tsim/cfg - -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -print cmd result----> $system_content -#if $system_content != @{consume success: 20000, 0}@ then -if $system_content != $expect_result then - print expect @{consume success: 20000, 0}@ , actual @system_content - return -1 -endi - -$numOfTopics = 3 -$expectMsgCntFromCtb = $expectMsgCntFromCtb * $numOfTopics -$expect_result = @{consume success: @ -$expect_result = $expect_result . $expectMsgCntFromCtb -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -print cmd result----> $system_content -#if $system_content != @{consume success: 300, 0}@ then -if $system_content != $expect_result then - return -1 -endi - -$numOfTopics = 3 -$expectMsgCntFromNtb = $expectMsgCntFromNtb * $tbNum -$expectMsgCntFromNtb = $expectMsgCntFromNtb * $numOfTopics -$expect_result = @{consume success: @ -$expect_result = $expect_result . $expectMsgCntFromNtb -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromNtb -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromNtb -print cmd result----> $system_content -#if $system_content != @{consume success: 30000, 0}@ then -if $system_content != $expect_result then - return -1 -endi - -if $loop_cnt == 0 then - $loop_cnt = 1 - $vgroups = 4 - $dbNamme = d1 - goto loop_vgroups -endi - - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/mainConsumerInOneTopic.sim b/tests/script/tsim/tmq/mainConsumerInOneTopic.sim deleted file mode 100644 index f6fccee5e5..0000000000 --- a/tests/script/tsim/tmq/mainConsumerInOneTopic.sim +++ /dev/null @@ -1,246 +0,0 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -# scene1: vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene2: vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene3: vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene4: vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## -######## This test case include scene1 and scene3 -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## - -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - -sql connect - -$loop_cnt = 0 -$vgroups = 1 -$dbNamme = d0 -loop_vgroups: -print =============== create database $dbNamme vgroups $vgroups -sql create database $dbNamme vgroups $vgroups -sql use $dbNamme - -print =============== create super table -sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) - -sql show stables -if $rows != 1 then - return -1 -endi - -print =============== create child table -$tbPrefix = ct -$tbNum = 100 - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using stb tags( $i ) - $i = $i + 1 -endw - -print =============== create normal table -sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) - -print =============== create multi topics. notes: now only support: -print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb -print =============== will support: * from stb - -sql create topic topic_stb_column as select ts, c1, c3 from stb -#sql create topic topic_stb_all as select * from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -sql create topic topic_ctb_column as select ts, c1, c3 from ct0 -sql create topic topic_ctb_all as select * from ct0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 - -sql create topic topic_ntb_column as select ts, c1, c3 from ntb -sql create topic topic_ntb_all as select * from ntb -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb - -sql show tables -if $rows != 101 then - return -1 -endi - -print =============== run_back insert data - -if $loop_cnt == 0 then - run_back tsim/tmq/insertFixedDataV2.sim -else - run_back tsim/tmq/insertFixedDataV4.sim -endi - -#sleep 1000 - -#$rowNum = 1000 -#$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -# -#$i = 0 -#while $i < $tbNum -# $tb = $tbPrefix . $i -# -# $x = 0 -# while $x < $rowNum -# $c = $x / 10 -# $c = $c * 10 -# $c = $x - $c -# -# $binary = ' . binary -# $binary = $binary . $c -# $binary = $binary . ' -# -# sql insert into $tb values ($tstart , $c , $x , $binary ) -# sql insert into ntb values ($tstart , $c , $x , $binary ) -# $tstart = $tstart + 1 -# $x = $x + 1 -# endw -# -# $i = $i + 1 -# $tstart = 1640966400000 -#endw - -#root@trd02 /home $ tmq_sim --help -# -c Configuration directory, default is -# -d The name of the database for cosumer, no default -# -t The topic string for cosumer, no default -# -k The key-value string for cosumer, no default -# -g showMsgFlag, default is 0 -# - -$tbNum = 10 -$consumeDelay = 5 -$expectMsgCntFromCtb = 1000 -$expectMsgCntFromStb = $expectMsgCntFromCtb * $tbNum -print consumeDelay: $consumeDelay -print insert data child num: $tbNum -print expectMsgCntFromCtb: $expectMsgCntFromCtb -print expectMsgCntFromStb: $expectMsgCntFromStb - - -# supported key: -# group.id: -# enable.auto.commit: -# auto.offset.reset: -# td.connect.ip: -# td.connect.user:root -# td.connect.pass:taosdata -# td.connect.port:6030 -# td.connect.db:db - -$expect_result = @{consume success: @ -$expect_result = $expect_result . $expectMsgCntFromStb -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result - -system_content echo -n \$BUILD_DIR -$tmq_sim = $system_content . /build/bin/tmq_sim -$tsim_cfg = $system_content . /tsim/cfg - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -print cmd result----> $system_content -if $system_content != $expect_result then - return -1 -endi - -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -#print cmd result----> $system_content -##if $system_content != @{consume success: 10000, 0}@ then -#if $system_content != $expect_result then -# return -1 -#endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -print cmd result----> $system_content -#if $system_content != @{consume success: 10000, 0}@ then -if $system_content != $expect_result then - return -1 -endi - -$expect_result = @{consume success: @ -$expect_result = $expect_result . $expectMsgCntFromCtb -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -print cmd result----> $system_content -if $system_content != $expect_result then - return -1 -endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -print cmd result----> $system_content -if $system_content != $expect_result then - return -1 -endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -print cmd result----> $system_content -if $system_content != $expect_result then - return -1 -endi - -$expect_result = @{consume success: @ -$expect_result = $expect_result . $expectMsgCntFromStb -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -print cmd result----> $system_content -if $system_content != $expect_result then - return -1 -endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -print cmd result----> $system_content -if $system_content != $expect_result then - return -1 -endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -print cmd result----> $system_content -if $system_content != $expect_result then - return -1 -endi - -if $loop_cnt == 0 then - $loop_cnt = 1 - $vgroups = 4 - $dbNamme = d1 - goto loop_vgroups -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/multiTopic.sim b/tests/script/tsim/tmq/multiTopic.sim deleted file mode 100644 index 3795e82e86..0000000000 --- a/tests/script/tsim/tmq/multiTopic.sim +++ /dev/null @@ -1,228 +0,0 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -# scene1: vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene2: vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene3: vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene4: vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## -######## This test case include scene2 and scene4 -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## - -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - -sql connect - -$loop_cnt = 0 -$vgroups = 1 -$dbNamme = d0 -loop_vgroups: -print =============== create database $dbNamme vgroups $vgroups -sql create database $dbNamme vgroups $vgroups -sql show databases -print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 -print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 - -if $loop_cnt == 0 then - if $rows != 3 then - return -1 - endi - if $data22 != 1 then # vgroups - print vgroups: $data02 - return -1 - endi -else - if $rows != 4 then - return -1 - endi - if $data20 == d1 then - if $data22 != 4 then # vgroups - print vgroups: $data02 - return -1 - endi - else - if $data32 != 4 then # vgroups - print vgroups: $data12 - return -1 - endi - endi -endi - -sql use $dbNamme - -print =============== create super table -sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) - -sql show stables -if $rows != 1 then - return -1 -endi - -print =============== create child table -$tbPrefix = ct -$tbNum = 100 - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using stb tags( $i ) - $i = $i + 1 -endw - -print =============== create normal table -sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) - -print =============== create multi topics. notes: now only support: -print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb -print =============== will support: * from stb - -sql create topic topic_stb_column as select ts, c1, c3 from stb -#sql create topic topic_stb_all as select * from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -sql create topic topic_ctb_column as select ts, c1, c3 from ct0 -sql create topic topic_ctb_all as select * from ct0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 - -sql create topic topic_ntb_column as select ts, c1, c3 from ntb -sql create topic topic_ntb_all as select * from ntb -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb - -sql show tables -if $rows != 101 then - return -1 -endi - -print =============== insert data -$rowNum = 100 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - - $x = 0 - while $x < $rowNum - $c = $x / 10 - $c = $c * 10 - $c = $x - $c - - $binary = ' . binary - $binary = $binary . $c - $binary = $binary . ' - - sql insert into $tb values ($tstart , $c , $x , $binary ) - sql insert into ntb values ($tstart , $c , $x , $binary ) - $tstart = $tstart + 1 - $x = $x + 1 - endw - - $i = $i + 1 - $tstart = 1640966400000 -endw - -#root@trd02 /home $ tmq_sim --help -# -c Configuration directory, default is -# -d The name of the database for cosumer, no default -# -t The topic string for cosumer, no default -# -k The key-value string for cosumer, no default -# -g showMsgFlag, default is 0 -# - -$totalMsgCnt = $rowNum * $tbNum -print inserted totalMsgCnt: $totalMsgCnt - -# supported key: -# group.id: -# enable.auto.commit: -# auto.offset.reset: -# td.connect.ip: -# td.connect.user:root -# td.connect.pass:taosdata -# td.connect.port:6030 -# td.connect.db:db - -$numOfTopics = 2 -$totalMsgCntOfmultiTopics = $totalMsgCnt * $numOfTopics -$expect_result = @{consume success: @ -$expect_result = $expect_result . $totalMsgCntOfmultiTopics -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result - -system_content echo -n \$BUILD_DIR -$tmq_sim = $system_content . /build/bin/tmq_sim -$tsim_cfg = $system_content . /tsim/cfg - -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2" -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2" -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2" -print cmd result----> $system_content -#if $system_content != @{consume success: 20000, 0}@ then -if $system_content != $expect_result then - return -1 -endi - -$numOfTopics = 3 -$totalMsgCntOfmultiTopics = $rowNum * $numOfTopics -$expect_result = @{consume success: @ -$expect_result = $expect_result . $totalMsgCntOfmultiTopics -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" -print cmd result----> $system_content -#if $system_content != @{consume success: 300, 0}@ then -if $system_content != $expect_result then - return -1 -endi - -$numOfTopics = 3 -$totalMsgCntOfmultiTopics = $totalMsgCnt * $numOfTopics -$expect_result = @{consume success: @ -$expect_result = $expect_result . $totalMsgCntOfmultiTopics -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2" -print cmd result----> $system_content -#if $system_content != @{consume success: 30000, 0}@ then -if $system_content != $expect_result then - return -1 -endi - -if $loop_cnt == 0 then - $loop_cnt = 1 - $vgroups = 4 - $dbNamme = d1 - goto loop_vgroups -endi - - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/oneTopic.sim b/tests/script/tsim/tmq/oneTopic.sim deleted file mode 100644 index d4353f60c9..0000000000 --- a/tests/script/tsim/tmq/oneTopic.sim +++ /dev/null @@ -1,268 +0,0 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -# scene1: vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene2: vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene3: vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene4: vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## -######## This test case include scene1 and scene3 -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## - -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - -sql connect - -$loop_cnt = 0 -$vgroups = 1 -$dbNamme = d0 -loop_vgroups: -print =============== create database $dbNamme vgroups $vgroups -sql create database $dbNamme vgroups $vgroups -sql show databases -print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 -print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 - -if $loop_cnt == 0 then - if $rows != 3 then - return -1 - endi - if $data22 != 1 then # vgroups - print vgroups: $data02 - return -1 - endi -else - if $rows != 4 then - return -1 - endi - if $data20 == d1 then - if $data22 != 4 then # vgroups - print vgroups: $data02 - return -1 - endi - else - if $data32 != 4 then # vgroups - print vgroups: $data12 - return -1 - endi - endi -endi - -sql use $dbNamme - -print =============== create super table -sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) - -sql show stables -if $rows != 1 then - return -1 -endi - -print =============== create child table -$tbPrefix = ct -$tbNum = 100 - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using stb tags( $i ) - $i = $i + 1 -endw - -print =============== create normal table -sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) - -print =============== create multi topics. notes: now only support: -print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb -print =============== will support: * from stb - -sql create topic topic_stb_column as select ts, c1, c3 from stb -#sql create topic topic_stb_all as select * from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -sql create topic topic_ctb_column as select ts, c1, c3 from ct0 -sql create topic topic_ctb_all as select * from ct0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 - -sql create topic topic_ntb_column as select ts, c1, c3 from ntb -sql create topic topic_ntb_all as select * from ntb -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb - -sql show tables -if $rows != 101 then - return -1 -endi - -print =============== insert data -$rowNum = 100 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - - $x = 0 - while $x < $rowNum - $c = $x / 10 - $c = $c * 10 - $c = $x - $c - - $binary = ' . binary - $binary = $binary . $c - $binary = $binary . ' - - sql insert into $tb values ($tstart , $c , $x , $binary ) - sql insert into ntb values ($tstart , $c , $x , $binary ) - $tstart = $tstart + 1 - $x = $x + 1 - endw - - $i = $i + 1 - $tstart = 1640966400000 -endw - -#root@trd02 /home $ tmq_sim --help -# -c Configuration directory, default is -# -d The name of the database for cosumer, no default -# -t The topic string for cosumer, no default -# -k The key-value string for cosumer, no default -# -g showMsgFlag, default is 0 -# - -$totalMsgCnt = $rowNum * $tbNum -print inserted totalMsgCnt: $totalMsgCnt - -# supported key: -# group.id: -# enable.auto.commit: -# auto.offset.reset: -# td.connect.ip: -# td.connect.user:root -# td.connect.pass:taosdata -# td.connect.port:6030 -# td.connect.db:db - -$expect_result = @{consume success: @ -$expect_result = $expect_result . $totalMsgCnt -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result - -system_content echo -n \$BUILD_DIR -$tmq_sim = $system_content . /build/bin/tmq_sim -$tsim_cfg = $system_content . /tsim/cfg - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -print cmd result----> $system_content -#if $system_content != @{consume success: 10000, 0}@ then -if $system_content != $expect_result then - return -1 -endi - -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -#print cmd result----> $system_content -##if $system_content != @{consume success: 10000, 0}@ then -#if $system_content != $expect_result then -# return -1 -#endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -print cmd result----> $system_content -#if $system_content != @{consume success: 10000, 0}@ then -if $system_content != $expect_result then - return -1 -endi - -$expect_result = @{consume success: @ -$expect_result = $expect_result . $rowNum -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -print cmd result----> $system_content -#if $system_content != @{consume success: 100, 0}@ then -if $system_content != $expect_result then - return -1 -endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -print cmd result----> $system_content -#if $system_content != @{consume success: 100, 0}@ then -if $system_content != $expect_result then - return -1 -endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -print cmd result----> $system_content -#if $system_content != @{consume success: 100, 0}@ then -if $system_content != $expect_result then - return -1 -endi - -$expect_result = @{consume success: @ -$expect_result = $expect_result . $totalMsgCnt -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -print cmd result----> $system_content -#if $system_content != @{consume success: 10000, 0}@ then -if $system_content != $expect_result then - return -1 -endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -print cmd result----> $system_content -#if $system_content != @{consume success: 10000, 0}@ then -if $system_content != $expect_result then - return -1 -endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -print cmd result----> $system_content -#if $system_content != @{consume success: 10000, 0}@ then -if $system_content != $expect_result then - return -1 -endi - -if $loop_cnt == 0 then - $loop_cnt = 1 - $vgroups = 4 - $dbNamme = d1 - goto loop_vgroups -endi - - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/overlapTopic2Con1Cgrp.sim b/tests/script/tsim/tmq/overlapTopic2Con1Cgrp.sim deleted file mode 100644 index 01bf91fd9f..0000000000 --- a/tests/script/tsim/tmq/overlapTopic2Con1Cgrp.sim +++ /dev/null @@ -1,255 +0,0 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -# scene1: vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene2: vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene3: vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene4: vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## -######## This test case include scene2 and scene4 -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## - -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - -sql connect - -$loop_cnt = 0 -$vgroups = 1 -$dbNamme = d0 - -loop_vgroups: -print =============== create database $dbNamme vgroups $vgroups -sql create database $dbNamme vgroups $vgroups -sql show databases -print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 -print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 - -if $loop_cnt == 0 then - if $rows != 3 then - return -1 - endi - if $data22 != 1 then # vgroups - print vgroups: $data02 - return -1 - endi -else - if $rows != 4 then - return -1 - endi - if $data00 == d1 then - if $data02 != 4 then # vgroups - print vgroups: $data02 - return -1 - endi - else - if $data12 != 4 then # vgroups - print vgroups: $data12 - return -1 - endi - endi -endi - -sql use $dbNamme - -print =============== create super table -sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) - -sql show stables -if $rows != 1 then - return -1 -endi - -print =============== create child table -$tbPrefix = ct -$tbNum = 10 - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using stb tags( $i ) - $i = $i + 1 -endw - -print =============== create normal table -sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) - -print =============== create multi topics. notes: now only support: -print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb -print =============== will support: * from stb - -sql create topic topic_stb_column as select ts, c1, c3 from stb -sql create topic topic_stb_all as select sqrt(c1) from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -sql create topic topic_ctb_column as select ts, c1, c3 from ct0 -sql create topic topic_ctb_all as select * from ct0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 - -sql create topic topic_ntb_column as select ts, c1, c3 from ntb -sql create topic topic_ntb_all as select * from ntb -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb - -sql show tables -if $rows != 11 then - return -1 -endi - -print =============== insert data -$rowNum = 100 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - - $x = 0 - while $x < $rowNum - $c = $x / 10 - $c = $c * 10 - $c = $x - $c - - $binary = ' . binary - $binary = $binary . $c - $binary = $binary . ' - - sql insert into $tb values ($tstart , $c , $x , $binary ) - sql insert into ntb values ($tstart , $c , $x , $binary ) - $tstart = $tstart + 1 - $x = $x + 1 - endw - - $i = $i + 1 -# $tstart = 1640966400000 -endw - -#root@trd02 /home $ tmq_sim --help -# -c Configuration directory, default is -# -d The name of the database for cosumer, no default -# -t The topic string for cosumer, no default -# -k The key-value string for cosumer, no default -# -g showMsgFlag, default is 0 -# - -$consumeDelay = 3 - -$expectMsgCntFromCtb = $rowNum -$expectMsgCntFromStb = $rowNum * $tbNum -$expectMsgCntFromNtb = $rowNum * $tbNum -print expectMsgCntFromCtb: $expectMsgCntFromCtb -print expectMsgCntFromStb: $expectMsgCntFromStb -print expectMsgCntFromNtb: $expectMsgCntFromNtb - -# supported key: -# group.id: -# enable.auto.commit: -# auto.offset.reset: -# td.connect.ip: -# td.connect.user:root -# td.connect.pass:taosdata -# td.connect.port:6030 -# td.connect.db:db - -$numOfTopics = 3 -$totalMsgCntOfmultiTopics = $expectMsgCntFromStb * $numOfTopics -$expect_result = @{consume success: @ -$expect_result = $expect_result . $totalMsgCntOfmultiTopics -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result - -$check_mode = 0 -if $loop_cnt == 0 then - $check_mode = 0 -else - $check_mode = 2 -endi - -$expectMsgCntFromStb0 = 2001 -$expectMsgCntFromStb1 = 2001 - -system_content echo -n \$BUILD_DIR -$tmq_sim = $system_content . /build/bin/tmq_sim -$tsim_cfg = $system_content . /tsim/cfg - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_column, topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_function, topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb0 -m1 $expectMsgCntFromStb1 -j 2 -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_column, topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_function, topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb0 -m1 $expectMsgCntFromStb1 -j 2 -print cmd result----> $system_content -if $system_content != success then - return -1 -endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_column, topic_stb_function" -k1 "group.id:tg1" -t "topic_stb_function, topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 3 -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t1 "topic_stb_column, topic_stb_function" -k1 "group.id:tg1" -t "topic_stb_function, topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 3 -print cmd result----> $system_content -if $system_content != success then - return -1 -endi - -#$numOfTopics = 3 -#$totalMsgCntOfmultiTopics = $rowNum * $numOfTopics -#$expect_result = @{consume success: @ -#$expect_result = $expect_result . $totalMsgCntOfmultiTopics -#$expect_result = $expect_result . @, @ -#$expect_result = $expect_result . 0} -#print expect_result----> $expect_result -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 4 -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 4 -print cmd result----> $system_content -if $system_content != success then - return -1 -endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function" -k "group.id:tg1" -t "topic_ctb_function, topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 3 -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function" -k "group.id:tg1" -t "topic_ctb_function, topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 3 -print cmd result----> $system_content -if $system_content != success then - return -1 -endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_function, topic_ntb_all" -k "group.id:tg2" -t "topic_ntb_column, topic_ntb_function, topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromNtb -j 4 -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_function, topic_ntb_all" -k "group.id:tg2" -t "topic_ntb_column, topic_ntb_function, topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromNtb -j 4 -print cmd result----> $system_content -if $system_content != success then - return -1 -endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_function" -k "group.id:tg1" -t "topic_ntb_function, topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromNtb -j 3 -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_function" -k "group.id:tg1" -t "topic_ntb_function, topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromNtb -j 3 -print cmd result----> $system_content -if $system_content != success then - return -1 -endi - - -if $loop_cnt == 0 then - $loop_cnt = 1 - $vgroups = 4 - $dbNamme = d1 - goto loop_vgroups -endi - - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/prepareBasicEnv.sim b/tests/script/tsim/tmq/prepareBasicEnv.sim deleted file mode 100644 index c4f706e262..0000000000 --- a/tests/script/tsim/tmq/prepareBasicEnv.sim +++ /dev/null @@ -1,88 +0,0 @@ -# stop all dnodes before start this case -system sh/stop_dnodes.sh - -# deploy dnode 1 -system sh/deploy.sh -n dnode1 -i 1 - -# add some config items for this case -#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 - -# start dnode 1 -system sh/exec.sh -n dnode1 -s start - -sql connect - -#---- global parameters start ----# -$dbName = db -$vgroups = 4 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 100 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -print == create database $dbName vgroups $vgroups -sql create database $dbName vgroups $vgroups - -#wait database ready -$loop_cnt = 0 -check_db_ready: -if $loop_cnt == 10 then - print ====> database not ready! - return -1 -endi -sql show databases -print ==> rows: $rows -print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] -print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] -if $data(db)[20] != nostrict then - sleep 100 - $loop_cnt = $loop_cnt + 1 - goto check_db_ready -endi - -sql use $dbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi - -print == create super table -sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) -sql show stables -if $rows != 1 then - return -1 -endi - -print == create child table, normal table and insert data -$i = 0 -while $i < $ctbNum - $ctb = $ctbPrefix . $i - $ntb = $ntbPrefix . $i - sql create table $ctb using $stbPrefix tags( $i ) - sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16)) - - $x = 0 - while $x < $rowsPerCtb - $binary = ' . binary- - $binary = $binary . $i - $binary = $binary . ' - - sql insert into $ctb values ($tstart , $i , $x , $binary ) - sql insert into $ntb values ($tstart , $i , $x , $binary ) - $tstart = $tstart + 1 - $x = $x + 1 - endw - - $i = $i + 1 - $tstart = 1640966400000 -endw From aa433a13d810814a48440c1bbc2e1ffa9ce00cde Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 22 Apr 2022 17:33:40 +0800 Subject: [PATCH 69/83] [test: modify test case] --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 3159569f32..a226e90094 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -76,7 +76,7 @@ ./test.sh -f tsim/insert/backquote.sim -m ./test.sh -f tsim/parser/fourArithmetic-basic.sim -m ./test.sh -f tsim/query/interval-offset.sim -m -./test.sh -f tsim/tmq/basic.sim -m +#./test.sh -f tsim/tmq/basic.sim -m ./test.sh -f tsim/stable/vnode3.sim -m ./test.sh -f tsim/qnode/basic1.sim -m ./test.sh -f tsim/mnode/basic1.sim -m From 8cfcd6d4f4c9a5493183aaed00b13ade10b94921 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 22 Apr 2022 17:48:12 +0800 Subject: [PATCH 70/83] feature/qnode --- source/client/src/clientMsgHandler.c | 3 +- source/dnode/mgmt/implement/src/dmHandle.c | 6 +- source/dnode/mnode/impl/inc/mndVgroup.h | 1 + source/dnode/mnode/impl/src/mndDb.c | 8 +-- source/libs/qworker/src/qworker.c | 8 +++ source/libs/scalar/inc/filterInt.h | 4 ++ source/libs/scalar/src/filter.c | 66 ++++++++++++++++++---- source/libs/scalar/src/sclvector.c | 18 +++++- 8 files changed, 89 insertions(+), 25 deletions(-) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 06b4c7b8e0..a33351373e 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -151,13 +151,14 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { taosMemoryFreeClear(output.dbVgroup); tscError("failed to build use db output since %s", terrstr()); - } else { + } else if (output.dbVgroup) { struct SCatalog* pCatalog = NULL; int32_t code1 = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); if (code1 != TSDB_CODE_SUCCESS) { tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->pTscObj->pAppInfo->clusterId, tstrerror(code1)); + taosMemoryFreeClear(output.dbVgroup); } else { catalogUpdateDBVgInfo(pCatalog, output.db, output.dbId, output.dbVgroup); } diff --git a/source/dnode/mgmt/implement/src/dmHandle.c b/source/dnode/mgmt/implement/src/dmHandle.c index 376f589acd..10cc1f4873 100644 --- a/source/dnode/mgmt/implement/src/dmHandle.c +++ b/source/dnode/mgmt/implement/src/dmHandle.c @@ -367,9 +367,9 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) { } dmReportStartup(pDnode, "dnode-transport", "initialized"); - if (dmStartUdfd(pDnode) != 0) { - dError("failed to start udfd"); - } +// if (dmStartUdfd(pDnode) != 0) { +// dError("failed to start udfd"); +// } dInfo("dnode-mgmt is initialized"); return 0; diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index f42829eddf..fe8c35004d 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -30,6 +30,7 @@ SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup); int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup); int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); +int32_t mndGetGlobalVgroupVersion(int32_t *vgId); void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index dab587432e..9caadb7e03 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1191,6 +1191,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { char *p = strchr(usedbReq.db, '.'); if (p && 0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB)) { memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN); + //mndGetGlobalVgroupVersion(); TODO static int32_t vgVersion = 1; if (usedbReq.vgVersion < vgVersion) { usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo)); @@ -1202,16 +1203,11 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos); usedbRsp.vgVersion = vgVersion++; - if (taosArrayGetSize(usedbRsp.pVgroupInfos) <= 0) { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; - } else { - code = 0; - } } else { usedbRsp.vgVersion = usedbReq.vgVersion; - code = 0; } usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); + code = 0; // no jump, need to construct rsp } else { diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index a74c28cf53..7329dc76a3 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1388,6 +1388,14 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { void *pIter = taosHashIterate(mgmt->schHash, NULL); while (pIter) { + SQWSchStatus *sch = (SQWSchStatus *)pIter; + if (NULL == sch->hbConnInfo.handle) { + uint64_t *sId = taosHashGetKey(pIter, NULL); + QW_DLOG("cancel send hb to sch %" PRIx64 " cause of no connection handle", *sId); + pIter = taosHashIterate(mgmt->schHash, pIter); + continue; + } + code = qwGenerateSchHbRsp(mgmt, (SQWSchStatus *)pIter, &rspList[i]); if (code) { taosHashCancelIterate(mgmt->schHash, pIter); diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index 99c33d3165..f4cd9dd1c1 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -186,6 +186,7 @@ typedef struct SFilterColCtx { typedef struct SFilterCompare { uint8_t type; + int8_t precision; uint8_t optr; uint8_t optr2; } SFilterCompare; @@ -218,6 +219,7 @@ typedef struct SFltTreeStat { int32_t code; int8_t precision; bool scalarMode; + SArray* nodeList; SFilterInfo* info; } SFltTreeStat; @@ -303,6 +305,7 @@ struct SFilterInfo { #define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx])) #define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx])) #define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnNode *)((fi)->desc))->node.resType.type) +#define FILTER_GET_COL_FIELD_PRECISION(fi) (((SColumnNode *)((fi)->desc))->node.resType.precision) #define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnNode *)((fi)->desc))->node.resType.bytes) #define FILTER_GET_COL_FIELD_ID(fi) (((SColumnNode *)((fi)->desc))->colId) #define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnNode *)((fi)->desc))->slotId) @@ -317,6 +320,7 @@ struct SFilterInfo { #define FILTER_UNIT_RIGHT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right) #define FILTER_UNIT_RIGHT2_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right2) #define FILTER_UNIT_DATA_TYPE(u) ((u)->compare.type) +#define FILTER_UNIT_DATA_PRECISION(u) ((u)->compare.precision) #define FILTER_UNIT_COL_DESC(i, u) FILTER_GET_COL_FIELD_DESC(FILTER_UNIT_LEFT_FIELD(i, u)) #define FILTER_UNIT_COL_DATA(i, u, ri) FILTER_GET_COL_FIELD_DATA(FILTER_UNIT_LEFT_FIELD(i, u), ri) #define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u)) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index e24162bccb..c17e854aa9 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -21,6 +21,7 @@ #include "sclInt.h" #include "tcompare.h" #include "tdatablock.h" +#include "ttime.h" OptrStr gOptrStr[] = { {0, "invalid"}, @@ -986,6 +987,7 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi assert(FILTER_GET_FLAG(col->flag, FLD_TYPE_COLUMN)); info->units[info->unitNum].compare.type = FILTER_GET_COL_FIELD_TYPE(col); + info->units[info->unitNum].compare.precision = FILTER_GET_COL_FIELD_PRECISION(col); *uidx = info->unitNum; @@ -1748,6 +1750,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { assert(FILTER_GET_FLAG(right->flag, FLD_TYPE_VALUE)); uint32_t type = FILTER_UNIT_DATA_TYPE(unit); + int8_t precision = FILTER_UNIT_DATA_PRECISION(unit); SFilterField* fi = right; SValueNode* var = (SValueNode *)fi->desc; @@ -1801,6 +1804,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { } else { SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; out.columnData->info.type = type; + out.columnData->info.precision = precision; if (IS_VAR_DATA_TYPE(type)) { out.columnData->info.bytes = bytes; } else { @@ -3475,6 +3479,33 @@ int32_t filterFreeNcharColumns(SFilterInfo* info) { return TSDB_CODE_SUCCESS; } +int32_t fltAddValueNodeToConverList(SFltTreeStat *stat, SValueNode* pNode) { + if (NULL == stat->nodeList) { + stat->nodeList = taosArrayInit(10, POINTER_BYTES); + if (NULL == stat->nodeList) { + FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + if (NULL == taosArrayPush(stat->nodeList, &pNode)) { + FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + return TSDB_CODE_SUCCESS; +} + +void fltConvertToTsValueNode(SFltTreeStat *stat, SValueNode* valueNode) { + char *timeStr = valueNode->datum.p; + if (convertStringToTimestamp(valueNode->node.resType.type, valueNode->datum.p, stat->precision, &valueNode->datum.i) != + TSDB_CODE_SUCCESS) { + valueNode->datum.i = 0; + } + taosMemoryFree(timeStr); + + valueNode->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; + valueNode->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; +} + EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { SFltTreeStat *stat = (SFltTreeStat *)pContext; @@ -3504,25 +3535,23 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { } SValueNode *valueNode = (SValueNode *)*pNode; - if (TSDB_DATA_TYPE_BINARY != valueNode->node.resType.type) { + if (TSDB_DATA_TYPE_BINARY != valueNode->node.resType.type && TSDB_DATA_TYPE_NCHAR != valueNode->node.resType.type) { return DEAL_RES_CONTINUE; } -#if 0 if (stat->precision < 0) { - //TODO + int32_t code = fltAddValueNodeToConverList(stat, valueNode); + if (code) { + stat->code = code; + return DEAL_RES_ERROR; + } + return DEAL_RES_CONTINUE; } - char *timeStr = valueNode->datum.p; - if (taosParseTime(valueNode->datum.p, &valueNode->datum.i, valueNode->node.resType.bytes, stat->precision, tsDaylight) != - TSDB_CODE_SUCCESS) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); - } - TODO -#else + fltConvertToTsValueNode(stat, valueNode); + return DEAL_RES_CONTINUE; -#endif } if (QUERY_NODE_COLUMN == nodeType(*pNode)) { @@ -3619,9 +3648,22 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { } int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) { + int32_t code = 0; nodesRewriteExprPostOrder(pNode, fltReviseRewriter, (void *)pStat); - FLT_RET(pStat->code); + FLT_ERR_JRET(pStat->code); + + int32_t nodeNum = taosArrayGetSize(pStat->nodeList); + for (int32_t i = 0; i < nodeNum; ++i) { + SValueNode *valueNode = *(SValueNode **)taosArrayGet(pStat->nodeList, i); + + fltConvertToTsValueNode(pStat, valueNode); + } + +_return: + + taosArrayDestroy(pStat->nodeList); + FLT_RET(code); } int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index d484738328..f63e539a96 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -24,6 +24,7 @@ #include "tcompare.h" #include "tdatablock.h" #include "ttypes.h" +#include "ttime.h" #define LEFT_COL ((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData)) #define RIGHT_COL ((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData)) @@ -252,6 +253,15 @@ _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { return p; } +static FORCE_INLINE void varToTimestamp(char *buf, SScalarParam* pOut, int32_t rowIndex) { + int64_t value = 0; + if (taosParseTime(buf, &value, strlen(buf), pOut->columnData->info.precision, tsDaylight) != TSDB_CODE_SUCCESS) { + value = 0; + } + + colDataAppendInt64(pOut->columnData, rowIndex, &value); +} + static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t rowIndex) { int64_t value = strtoll(buf, NULL, 10); colDataAppendInt64(pOut->columnData, rowIndex, &value); @@ -295,7 +305,7 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in _bufConverteFunc func = NULL; if (TSDB_DATA_TYPE_BOOL == outType) { func = varToBool; - } else if (IS_SIGNED_NUMERIC_TYPE(outType) || TSDB_DATA_TYPE_TIMESTAMP == outType) { + } else if (IS_SIGNED_NUMERIC_TYPE(outType)) { func = varToSigned; } else if (IS_UNSIGNED_NUMERIC_TYPE(outType)) { func = varToUnsigned; @@ -305,6 +315,8 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in ASSERT(inType == TSDB_DATA_TYPE_VARCHAR); func = varToNchar; vton = true; + } else if (TSDB_DATA_TYPE_TIMESTAMP == outType) { + func = varToTimestamp; } else { sclError("invalid convert outType:%d", outType); return TSDB_CODE_QRY_APP_ERROR; @@ -594,8 +606,8 @@ int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = { /*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 7, 0, 7, 5, 5, 5, 7, 0, 7, 0, 0, /*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, 7, 0, 0, /*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, 7, 0, 0, -/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 7, 7, 7, 0, 0, 0, 0, -/*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 9, 9, 7, 0, 7, 0, 0, +/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 7, 7, 7, 7, 0, 0, 0, 0, +/*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, 7, 0, 0, /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0, /*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, 7, 0, 0, /*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, 7, 0, 0, From 20879d0ef75a7d07367e322084793ca85f2cf12a Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 22 Apr 2022 18:23:37 +0800 Subject: [PATCH 71/83] fix: some syntax parsing problems --- include/common/tglobal.h | 3 + include/common/tmsg.h | 4 +- include/common/ttokendef.h | 178 +- include/libs/nodes/cmdnodes.h | 8 + include/libs/nodes/nodes.h | 1 + include/libs/nodes/querynodes.h | 4 +- include/libs/planner/planner.h | 2 + include/util/taoserror.h | 4 + include/util/tdef.h | 1 - source/client/src/clientImpl.c | 27 +- source/common/src/tglobal.c | 5 + source/common/src/tmsg.c | 5 +- source/common/src/ttime.c | 2 +- source/dnode/mnode/impl/src/mndInfoSchema.c | 10 - source/dnode/mnode/impl/src/mndShow.c | 2 - source/libs/nodes/src/nodesCodeFuncs.c | 4 +- source/libs/nodes/src/nodesUtilFuncs.c | 2 + source/libs/parser/inc/parAst.h | 5 +- source/libs/parser/inc/sql.y | 16 +- source/libs/parser/src/parAstCreater.c | 32 +- source/libs/parser/src/parCalcConst.c | 93 +- source/libs/parser/src/parInsert.c | 2 +- source/libs/parser/src/parTokenizer.c | 2 + source/libs/parser/src/parTranslater.c | 143 +- source/libs/parser/src/parUtil.c | 22 +- source/libs/parser/src/sql.c | 4609 ++++++++++--------- source/libs/planner/inc/planInt.h | 4 + source/libs/planner/src/planLogicCreater.c | 4 +- source/libs/planner/src/planPhysiCreater.c | 10 +- 29 files changed, 2796 insertions(+), 2408 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 88ce0cd970..8a8e3cd223 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -94,6 +94,9 @@ extern bool tsRetrieveBlockingModel; // retrieve threads will be blocked extern bool tsKeepOriginalColumnName; extern bool tsDeadLockKillQuery; +// query client +extern int32_t tsQueryPolicy; + // client extern int32_t tsMaxWildCardsLen; extern int32_t tsMaxRegexStringLen; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 5ea53b5510..9a6a14ab70 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1283,15 +1283,13 @@ typedef struct { } SMVCreateStreamRsp, SMSCreateStreamRsp; typedef struct { - char name[TSDB_TOPIC_FNAME_LEN]; + char name[TSDB_TOPIC_FNAME_LEN]; // accout.topic int8_t igExists; int8_t withTbName; int8_t withSchema; int8_t withTag; - int8_t withTagSchema; char* sql; char* ast; - int64_t subDbUid; char subscribeDbName[TSDB_DB_NAME_LEN]; } SCMCreateTopicReq; diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 39858cbbde..50be63dd56 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -156,87 +156,96 @@ #define TK_INTERVAL 138 #define TK_TOPIC 139 #define TK_AS 140 -#define TK_DESC 141 -#define TK_DESCRIBE 142 -#define TK_RESET 143 -#define TK_QUERY 144 -#define TK_EXPLAIN 145 -#define TK_ANALYZE 146 -#define TK_VERBOSE 147 -#define TK_NK_BOOL 148 -#define TK_RATIO 149 -#define TK_COMPACT 150 -#define TK_VNODES 151 -#define TK_IN 152 -#define TK_OUTPUTTYPE 153 -#define TK_AGGREGATE 154 -#define TK_BUFSIZE 155 -#define TK_STREAM 156 -#define TK_INTO 157 -#define TK_TRIGGER 158 -#define TK_AT_ONCE 159 -#define TK_WINDOW_CLOSE 160 -#define TK_WATERMARK 161 -#define TK_KILL 162 -#define TK_CONNECTION 163 -#define TK_MERGE 164 -#define TK_VGROUP 165 -#define TK_REDISTRIBUTE 166 -#define TK_SPLIT 167 -#define TK_SYNCDB 168 -#define TK_NULL 169 -#define TK_NK_QUESTION 170 -#define TK_NK_ARROW 171 -#define TK_ROWTS 172 -#define TK_TBNAME 173 -#define TK_QSTARTTS 174 -#define TK_QENDTS 175 -#define TK_WSTARTTS 176 -#define TK_WENDTS 177 -#define TK_WDURATION 178 -#define TK_CAST 179 -#define TK_NOW 180 -#define TK_TODAY 181 -#define TK_TIMEZONE 182 -#define TK_COUNT 183 -#define TK_FIRST 184 -#define TK_LAST 185 -#define TK_LAST_ROW 186 -#define TK_BETWEEN 187 -#define TK_IS 188 -#define TK_NK_LT 189 -#define TK_NK_GT 190 -#define TK_NK_LE 191 -#define TK_NK_GE 192 -#define TK_NK_NE 193 -#define TK_MATCH 194 -#define TK_NMATCH 195 -#define TK_CONTAINS 196 -#define TK_JOIN 197 -#define TK_INNER 198 -#define TK_SELECT 199 -#define TK_DISTINCT 200 -#define TK_WHERE 201 -#define TK_PARTITION 202 -#define TK_BY 203 -#define TK_SESSION 204 -#define TK_STATE_WINDOW 205 -#define TK_SLIDING 206 -#define TK_FILL 207 -#define TK_VALUE 208 -#define TK_NONE 209 -#define TK_PREV 210 -#define TK_LINEAR 211 -#define TK_NEXT 212 -#define TK_GROUP 213 -#define TK_HAVING 214 -#define TK_ORDER 215 -#define TK_SLIMIT 216 -#define TK_SOFFSET 217 -#define TK_LIMIT 218 -#define TK_OFFSET 219 -#define TK_ASC 220 -#define TK_NULLS 221 +#define TK_WITH 141 +#define TK_SCHEMA 142 +#define TK_DESC 143 +#define TK_DESCRIBE 144 +#define TK_RESET 145 +#define TK_QUERY 146 +#define TK_EXPLAIN 147 +#define TK_ANALYZE 148 +#define TK_VERBOSE 149 +#define TK_NK_BOOL 150 +#define TK_RATIO 151 +#define TK_COMPACT 152 +#define TK_VNODES 153 +#define TK_IN 154 +#define TK_OUTPUTTYPE 155 +#define TK_AGGREGATE 156 +#define TK_BUFSIZE 157 +#define TK_STREAM 158 +#define TK_INTO 159 +#define TK_TRIGGER 160 +#define TK_AT_ONCE 161 +#define TK_WINDOW_CLOSE 162 +#define TK_WATERMARK 163 +#define TK_KILL 164 +#define TK_CONNECTION 165 +#define TK_MERGE 166 +#define TK_VGROUP 167 +#define TK_REDISTRIBUTE 168 +#define TK_SPLIT 169 +#define TK_SYNCDB 170 +#define TK_NULL 171 +#define TK_NK_QUESTION 172 +#define TK_NK_ARROW 173 +#define TK_ROWTS 174 +#define TK_TBNAME 175 +#define TK_QSTARTTS 176 +#define TK_QENDTS 177 +#define TK_WSTARTTS 178 +#define TK_WENDTS 179 +#define TK_WDURATION 180 +#define TK_CAST 181 +#define TK_NOW 182 +#define TK_TODAY 183 +#define TK_TIMEZONE 184 +#define TK_COUNT 185 +#define TK_FIRST 186 +#define TK_LAST 187 +#define TK_LAST_ROW 188 +#define TK_BETWEEN 189 +#define TK_IS 190 +#define TK_NK_LT 191 +#define TK_NK_GT 192 +#define TK_NK_LE 193 +#define TK_NK_GE 194 +#define TK_NK_NE 195 +#define TK_MATCH 196 +#define TK_NMATCH 197 +#define TK_CONTAINS 198 +#define TK_JOIN 199 +#define TK_INNER 200 +#define TK_SELECT 201 +#define TK_DISTINCT 202 +#define TK_WHERE 203 +#define TK_PARTITION 204 +#define TK_BY 205 +#define TK_SESSION 206 +#define TK_STATE_WINDOW 207 +#define TK_SLIDING 208 +#define TK_FILL 209 +#define TK_VALUE 210 +#define TK_NONE 211 +#define TK_PREV 212 +#define TK_LINEAR 213 +#define TK_NEXT 214 +#define TK_GROUP 215 +#define TK_HAVING 216 +#define TK_ORDER 217 +#define TK_SLIMIT 218 +#define TK_SOFFSET 219 +#define TK_LIMIT 220 +#define TK_OFFSET 221 +#define TK_ASC 222 +#define TK_NULLS 223 +#define TK_ID 224 +#define TK_NK_BITNOT 225 +#define TK_INSERT 226 +#define TK_VALUES 227 +#define TK_IMPORT 228 +#define TK_NK_SEMI 229 +#define TK_FILE 230 #define TK_NK_SPACE 300 #define TK_NK_COMMENT 301 @@ -244,13 +253,6 @@ #define TK_NK_HEX 303 // hex number 0x123 #define TK_NK_OCT 304 // oct number #define TK_NK_BIN 305 // bin format data 0b111 -#define TK_NK_FILE 306 - -#define TK_NK_BITNOT 501 -#define TK_INSERT 502 -#define TK_VALUES 507 -#define TK_IMPORT 509 -#define TK_NK_SEMI 508 #define TK_NK_NIL 65535 diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 0394dfd9bd..44ff73c8bb 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -241,12 +241,20 @@ typedef struct SDropComponentNodeStmt { int32_t dnodeId; } SDropComponentNodeStmt; +typedef struct STopicOptions { + ENodeType type; + bool withTable; + bool withSchema; + bool withTag; +} STopicOptions; + typedef struct SCreateTopicStmt { ENodeType type; char topicName[TSDB_TABLE_NAME_LEN]; char subscribeDbName[TSDB_DB_NAME_LEN]; bool ignoreExists; SNode* pQuery; + STopicOptions* pOptions; } SCreateTopicStmt; typedef struct SDropTopicStmt { diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index a47bf4caee..6f77e8edb6 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -84,6 +84,7 @@ typedef enum ENodeType { QUERY_NODE_INDEX_OPTIONS, QUERY_NODE_EXPLAIN_OPTIONS, QUERY_NODE_STREAM_OPTIONS, + QUERY_NODE_TOPIC_OPTIONS, // Statement nodes are used in parser and planner module. QUERY_NODE_SET_OPERATOR, diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 143224d8f5..d8e2354e8e 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -237,8 +237,8 @@ typedef struct SSelectStmt { SNodeList* pGroupByList; // SGroupingSetNode SNode* pHaving; SNodeList* pOrderByList; // SOrderByExprNode - SNode* pLimit; - SNode* pSlimit; + SLimitNode* pLimit; + SLimitNode* pSlimit; char stmtName[TSDB_TABLE_NAME_LEN]; uint8_t precision; bool isEmptyResult; diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 851d4d63c5..2f6b9e1866 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -35,6 +35,8 @@ typedef struct SPlanContext { int8_t triggerType; int64_t watermark; bool isStmtQuery; + void* pTransporter; + struct SCatalog* pCatalog; } SPlanContext; // Create the physical plan for the query, according to the AST. diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 215e83c7d4..da7c1d4408 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -615,6 +615,10 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL TAOS_DEF_ERROR_CODE(0, 0x2632) #define TSDB_CODE_PAR_ONLY_ONE_JSON_TAG TAOS_DEF_ERROR_CODE(0, 0x2633) #define TSDB_CODE_PAR_INCORRECT_NUM_OF_COL TAOS_DEF_ERROR_CODE(0, 0x2634) +#define TSDB_CODE_PAR_INCORRECT_TIMESTAMP_VAL TAOS_DEF_ERROR_CODE(0, 0x2635) +#define TSDB_CODE_PAR_INVALID_DAYS_VALUE TAOS_DEF_ERROR_CODE(0, 0x2636) +#define TSDB_CODE_PAR_OFFSET_LESS_ZERO TAOS_DEF_ERROR_CODE(0, 0x2637) +#define TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY TAOS_DEF_ERROR_CODE(0, 0x2638) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/include/util/tdef.h b/include/util/tdef.h index 2548df7186..31fbe42b6c 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -118,7 +118,6 @@ extern const int32_t TYPE_BYTES[15]; #define TSDB_INS_TABLE_USER_USERS "user_users" #define TSDB_INS_TABLE_LICENCES "grants" #define TSDB_INS_TABLE_VGROUPS "vgroups" -#define TSDB_INS_TABLE_TOPICS "topics" #define TSDB_INS_TABLE_CONSUMERS "consumers" #define TSDB_INS_TABLE_SUBSCRIBES "subscribes" #define TSDB_INS_TABLE_TRANS "trans" diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 10edb38bf1..43a1f828c3 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -226,14 +226,17 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList) { pRequest->type = pQuery->msgType; - SPlanContext cxt = {.queryId = pRequest->requestId, - .acctId = pRequest->pTscObj->acctId, - .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), - .pAstRoot = pQuery->pRoot, - .showRewrite = pQuery->showRewrite}; - int32_t code = qCreateQueryPlan(&cxt, pPlan, pNodeList); - if (code != 0) { - return code; + SPlanContext cxt = { + .queryId = pRequest->requestId, + .acctId = pRequest->pTscObj->acctId, + .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), + .pAstRoot = pQuery->pRoot, + .showRewrite = pQuery->showRewrite, + .pTransporter = pRequest->pTscObj->pAppInfo->pTransporter + }; + int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &cxt.pCatalog); + if (TSDB_CODE_SUCCESS == code) { + code = qCreateQueryPlan(&cxt, pPlan, pNodeList); } return code; } @@ -302,8 +305,6 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList } SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery) { - SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); - if (TSDB_CODE_SUCCESS == code) { switch (pQuery->execMode) { case QUERY_EXEC_MODE_LOCAL: @@ -312,12 +313,15 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code case QUERY_EXEC_MODE_RPC: code = execDdlQuery(pRequest, pQuery); break; - case QUERY_EXEC_MODE_SCHEDULE: + case QUERY_EXEC_MODE_SCHEDULE: { + SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); code = getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList); if (TSDB_CODE_SUCCESS == code) { code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList); } + taosArrayDestroy(pNodeList); break; + } case QUERY_EXEC_MODE_EMPTY_RESULT: pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT; break; @@ -326,7 +330,6 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code } } - taosArrayDestroy(pNodeList); if (!keepQuery) { qDestroyQuery(pQuery); } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 73031c410d..b41bcd7145 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -80,6 +80,9 @@ int32_t tsTelemInterval = 86400; char tsTelemServer[TSDB_FQDN_LEN] = "telemetry.taosdata.com"; uint16_t tsTelemPort = 80; +// query +int32_t tsQueryPolicy = 1; + /* * denote if the server needs to compress response message at the application layer to client, including query rsp, * metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server. @@ -322,6 +325,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { return -1; if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1; if (cfgAddInt32(pCfg, "maxBinaryDisplayWidth", tsMaxBinaryDisplayWidth, 1, 65536, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1; tsNumOfTaskQueueThreads = tsNumOfCores / 4; tsNumOfTaskQueueThreads = TRANGE(tsNumOfTaskQueueThreads, 1, 2); @@ -515,6 +519,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32; tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32; + tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; return 0; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 0ea196f1a1..06a96cdf9a 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2734,11 +2734,11 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo if (tEncodeI8(&encoder, pReq->withTbName) < 0) return -1; if (tEncodeI8(&encoder, pReq->withSchema) < 0) return -1; if (tEncodeI8(&encoder, pReq->withTag) < 0) return -1; - if (tEncodeI8(&encoder, pReq->withTagSchema) < 0) return -1; if (tEncodeI32(&encoder, sqlLen) < 0) return -1; if (tEncodeI32(&encoder, astLen) < 0) return -1; if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1; if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1; + if (0 == astLen && tEncodeCStr(&encoder, pReq->subscribeDbName) < 0) return -1; tEndEncode(&encoder); @@ -2760,7 +2760,6 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR if (tDecodeI8(&decoder, &pReq->withTbName) < 0) return -1; if (tDecodeI8(&decoder, &pReq->withSchema) < 0) return -1; if (tDecodeI8(&decoder, &pReq->withTag) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->withTagSchema) < 0) return -1; if (tDecodeI32(&decoder, &sqlLen) < 0) return -1; if (tDecodeI32(&decoder, &astLen) < 0) return -1; @@ -2774,6 +2773,8 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR pReq->ast = taosMemoryCalloc(1, astLen + 1); if (pReq->ast == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1; + } else { + if (tDecodeCStrTo(&decoder, pReq->subscribeDbName) < 0) return -1; } tEndDecode(&decoder); diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 2f3677330b..491601b12d 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -412,7 +412,7 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec, int64_t *timeVal) { int32_t charLen = varDataLen(inputData); char *newColData; - if (type == TSDB_DATA_TYPE_BINARY) { + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY) { newColData = taosMemoryCalloc(1, charLen + 1); memcpy(newColData, varDataVal(inputData), charLen); bool ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, 0); diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c index 32c667faf9..5dbbd27930 100644 --- a/source/dnode/mnode/impl/src/mndInfoSchema.c +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -199,15 +199,6 @@ static const SInfosTableSchema vgroupsSchema[] = { {.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, }; -// TODO put into perf schema -static const SInfosTableSchema topicSchema[] = { - {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "sql", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "row_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, -}; - static const SInfosTableSchema consumerSchema[] = { {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, @@ -291,7 +282,6 @@ static const SInfosTableMeta infosMeta[] = { {TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)}, {TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema)}, {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)}, - {TSDB_INS_TABLE_TOPICS, topicSchema, tListLen(topicSchema)}, {TSDB_INS_TABLE_CONSUMERS, consumerSchema, tListLen(consumerSchema)}, {TSDB_INS_TABLE_SUBSCRIBES, subscribeSchema, tListLen(subscribeSchema)}, {TSDB_INS_TABLE_TRANS, transSchema, tListLen(transSchema)}, diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index f81179201c..059da38ee5 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -84,8 +84,6 @@ static int32_t convertToRetrieveType(char* name, int32_t len) { type = TSDB_MGMT_TABLE_GRANTS; } else if (strncasecmp(name, TSDB_INS_TABLE_VGROUPS, len) == 0) { type = TSDB_MGMT_TABLE_VGROUP; - } else if (strncasecmp(name, TSDB_INS_TABLE_TOPICS, len) == 0) { - type = TSDB_MGMT_TABLE_TOPICS; } else if (strncasecmp(name, TSDB_INS_TABLE_CONSUMERS, len) == 0) { type = TSDB_MGMT_TABLE_CONSUMERS; } else if (strncasecmp(name, TSDB_INS_TABLE_SUBSCRIBES, len) == 0) { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 7eb472dad8..676304f9a8 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -2465,10 +2465,10 @@ static int32_t jsonToSelectStmt(const SJson* pJson, void* pObj) { code = jsonToNodeList(pJson, jkSelectStmtOrderBy, &pNode->pOrderByList); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkSelectStmtLimit, &pNode->pLimit); + code = jsonToNodeObject(pJson, jkSelectStmtLimit, (SNode**)&pNode->pLimit); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkSelectStmtSlimit, &pNode->pSlimit); + code = jsonToNodeObject(pJson, jkSelectStmtSlimit, (SNode**)&pNode->pSlimit); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetStringValue(pJson, jkSelectStmtStmtName, pNode->stmtName); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index bc9614b169..6415c16db5 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -86,6 +86,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SExplainOptions)); case QUERY_NODE_STREAM_OPTIONS: return makeNode(type, sizeof(SStreamOptions)); + case QUERY_NODE_TOPIC_OPTIONS: + return makeNode(type, sizeof(STopicOptions)); case QUERY_NODE_SET_OPERATOR: return makeNode(type, sizeof(SSetOperator)); case QUERY_NODE_SELECT_STMT: diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 7cb64b78e3..f4e206af9d 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -128,7 +128,7 @@ SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, STo SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pOptions); SNode* createTableOptions(SAstCreateContext* pCxt); SNode* setTableAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption); -SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment); +SNode* createColumnDefNode(SAstCreateContext* pCxt, SToken* pColName, SDataType dataType, const SToken* pComment); SDataType createDataType(uint8_t type); SDataType createVarLenDataType(uint8_t type, const SToken* pLen); SNode* createCreateTableStmt(SAstCreateContext* pCxt, bool ignoreExists, SNode* pRealTable, SNodeList* pCols, SNodeList* pTags, SNode* pOptions); @@ -157,7 +157,8 @@ SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInt SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pIndexName, SToken* pTableName); SNode* createCreateComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId); SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId); -SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery, const SToken* pSubscribeDbName); +SNode* createTopicOptions(SAstCreateContext* pCxt); +SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery, const SToken* pSubscribeDbName, SNode* pOptions); SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pTopicName); SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue); SNode* createDefaultExplainOptions(SAstCreateContext* pCxt); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 9ff36c702f..b1cf5759b7 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -35,14 +35,11 @@ %left OR. %left AND. -//%right NOT. %left UNION ALL MINUS EXCEPT INTERSECT. %left NK_BITAND NK_BITOR NK_LSHIFT NK_RSHIFT. %left NK_PLUS NK_MINUS. -//%left DIVIDE TIMES. %left NK_STAR NK_SLASH NK_REM. %left NK_CONCAT. -//%right NK_BITNOT. /************************************************ create/alter account *****************************************/ cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options. { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } @@ -387,10 +384,17 @@ func_list(A) ::= func_list(B) NK_COMMA func(C). func(A) ::= function_name(B) NK_LP expression_list(C) NK_RP. { A = createFunctionNode(pCxt, &B, C); } /************************************************ create/drop topic ***************************************************/ -cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS query_expression(C). { pCxt->pRootNode = createCreateTopicStmt(pCxt, A, &B, C, NULL); } -cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS db_name(C). { pCxt->pRootNode = createCreateTopicStmt(pCxt, A, &B, NULL, &C); } +cmd ::= CREATE TOPIC not_exists_opt(A) + topic_name(B) topic_options(D) AS query_expression(C). { pCxt->pRootNode = createCreateTopicStmt(pCxt, A, &B, C, NULL, D); } +cmd ::= CREATE TOPIC not_exists_opt(A) + topic_name(B) topic_options(D) AS db_name(C). { pCxt->pRootNode = createCreateTopicStmt(pCxt, A, &B, NULL, &C, D); } cmd ::= DROP TOPIC exists_opt(A) topic_name(B). { pCxt->pRootNode = createDropTopicStmt(pCxt, A, &B); } +topic_options(A) ::= . { A = createTopicOptions(pCxt); } +topic_options(A) ::= topic_options(B) WITH TABLE. { ((STopicOptions*)B)->withTable = true; A = B; } +topic_options(A) ::= topic_options(B) WITH SCHEMA. { ((STopicOptions*)B)->withSchema = true; A = B; } +topic_options(A) ::= topic_options(B) WITH TAG. { ((STopicOptions*)B)->withTag = true; A = B; } + /************************************************ desc/describe *******************************************************/ cmd ::= DESC full_table_name(A). { pCxt->pRootNode = createDescribeStmt(pCxt, A); } cmd ::= DESCRIBE full_table_name(A). { pCxt->pRootNode = createDescribeStmt(pCxt, A); } @@ -887,3 +891,5 @@ ordering_specification_opt(A) ::= DESC. null_ordering_opt(A) ::= . { A = NULL_ORDER_DEFAULT; } null_ordering_opt(A) ::= NULLS FIRST. { A = NULL_ORDER_FIRST; } null_ordering_opt(A) ::= NULLS LAST. { A = NULL_ORDER_LAST; } + +%fallback ID NK_BITNOT INSERT VALUES IMPORT NK_SEMI FILE. diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index ea4e8cae50..cbd57595bb 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -146,44 +146,42 @@ static bool checkDbName(SAstCreateContext* pCxt, SToken* pDbName, bool query) { pCxt->valid = false; } } else { + trimEscape(pDbName); if (pDbName->n >= TSDB_DB_NAME_LEN) { generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pDbName->z); pCxt->valid = false; } } - if (pCxt->valid) { - trimEscape(pDbName); - } return pCxt->valid; } static bool checkTableName(SAstCreateContext* pCxt, SToken* pTableName) { + trimEscape(pTableName); if (NULL != pTableName && pTableName->n >= TSDB_TABLE_NAME_LEN) { generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pTableName->z); pCxt->valid = false; return false; } - trimEscape(pTableName); return true; } static bool checkColumnName(SAstCreateContext* pCxt, SToken* pColumnName) { + trimEscape(pColumnName); if (NULL != pColumnName && pColumnName->n >= TSDB_COL_NAME_LEN) { generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pColumnName->z); pCxt->valid = false; return false; } - trimEscape(pColumnName); return true; } static bool checkIndexName(SAstCreateContext* pCxt, SToken* pIndexName) { + trimEscape(pIndexName); if (NULL != pIndexName && pIndexName->n >= TSDB_INDEX_NAME_LEN) { generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pIndexName->z); pCxt->valid = false; return false; } - trimEscape(pIndexName); return true; } @@ -609,14 +607,14 @@ SNode* addOrderByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrder SNode* addSlimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit) { if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { - ((SSelectStmt*)pStmt)->pSlimit = pSlimit; + ((SSelectStmt*)pStmt)->pSlimit = (SLimitNode*)pSlimit; } return pStmt; } SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit) { if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { - ((SSelectStmt*)pStmt)->pLimit = pLimit; + ((SSelectStmt*)pStmt)->pLimit = (SLimitNode*)pLimit; } return pStmt; } @@ -771,7 +769,10 @@ SNode* setTableAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOptio return pOptions; } -SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment) { +SNode* createColumnDefNode(SAstCreateContext* pCxt, SToken* pColName, SDataType dataType, const SToken* pComment) { + if (!checkColumnName(pCxt, pColName)) { + return NULL; + } SColumnDefNode* pCol = (SColumnDefNode*)nodesMakeNode(QUERY_NODE_COLUMN_DEF); CHECK_OUT_OF_MEM(pCol); strncpy(pCol->colName, pColName->z, pColName->n); @@ -1101,7 +1102,17 @@ SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, cons return (SNode*)pStmt; } -SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery, const SToken* pSubscribeDbName) { +SNode* createTopicOptions(SAstCreateContext* pCxt) { + STopicOptions* pOptions = nodesMakeNode(QUERY_NODE_TOPIC_OPTIONS); + CHECK_OUT_OF_MEM(pOptions); + pOptions->withTable = false; + pOptions->withSchema = false; + pOptions->withTag = false; + return (SNode*)pOptions; +} + +SNode* createCreateTopicStmt(SAstCreateContext* pCxt, + bool ignoreExists, const SToken* pTopicName, SNode* pQuery, const SToken* pSubscribeDbName, SNode* pOptions) { SCreateTopicStmt* pStmt = nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT); CHECK_OUT_OF_MEM(pStmt); strncpy(pStmt->topicName, pTopicName->z, pTopicName->n); @@ -1110,6 +1121,7 @@ SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const S if (NULL != pSubscribeDbName) { strncpy(pStmt->subscribeDbName, pSubscribeDbName->z, pSubscribeDbName->n); } + pStmt->pOptions = (STopicOptions*)pOptions; return (SNode*)pStmt; } diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index ead3ec90ad..ef1f9ada01 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -16,12 +16,15 @@ #include "functionMgt.h" #include "parInt.h" #include "scalar.h" +#include "ttime.h" typedef struct SCalcConstContext { + SParseContext* pParseCxt; + SMsgBuf msgBuf; int32_t code; } SCalcConstContext; -static int32_t calcConstQuery(SNode* pStmt); +static int32_t calcConstQuery(SCalcConstContext* pCxt, SNode* pStmt); static EDealRes doCalcConst(SNode** pNode, SCalcConstContext* pCxt) { SNode* pNew = NULL; @@ -35,10 +38,45 @@ static EDealRes doCalcConst(SNode** pNode, SCalcConstContext* pCxt) { return DEAL_RES_CONTINUE; } +static bool isTimestampCol(SNode* pNode) { + if (NULL == pNode) { + return false; + } + return (QUERY_NODE_COLUMN == nodeType(pNode) && TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode*)pNode)->resType.type); +} + +static EDealRes stringToTimestamp(SCalcConstContext* pCxt, uint8_t precision, SValueNode* pVal) { + switch (pVal->node.resType.type) { + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARBINARY: { + int64_t val = 0; + if (TSDB_CODE_SUCCESS != convertStringToTimestamp(pVal->node.resType.type, pVal->datum.p, precision, &val)) { + pCxt->code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCORRECT_TIMESTAMP_VAL, varDataVal(pVal->datum.p)); + return DEAL_RES_ERROR; + } + pVal->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; + pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; + taosMemoryFreeClear(pVal->datum.p); + pVal->datum.i = val; + break; + } + default: + break; + } + return DEAL_RES_CONTINUE; +} + static EDealRes calcConstOperator(SOperatorNode** pNode, void* pContext) { + SCalcConstContext* pCxt = pContext; SOperatorNode* pOp = *pNode; if (QUERY_NODE_VALUE == nodeType(pOp->pLeft) && (NULL == pOp->pRight || QUERY_NODE_VALUE == nodeType(pOp->pRight))) { - return doCalcConst((SNode**)pNode, (SCalcConstContext*)pContext); + return doCalcConst((SNode**)pNode, pCxt); + } + if (isTimestampCol(pOp->pLeft) && (NULL == pOp->pRight || QUERY_NODE_VALUE == nodeType(pOp->pRight))) { + return stringToTimestamp(pCxt, ((SColumnNode*)pOp->pLeft)->node.resType.precision, (SValueNode*)pOp->pRight); + } else if (isTimestampCol(pOp->pRight) && QUERY_NODE_VALUE == nodeType(pOp->pLeft)) { + return stringToTimestamp(pCxt, ((SColumnNode*)pOp->pRight)->node.resType.precision, (SValueNode*)pOp->pLeft); } return DEAL_RES_CONTINUE; } @@ -71,7 +109,7 @@ static EDealRes calcConstLogicCond(SLogicConditionNode** pNode, void* pContext) static EDealRes calcConstSubquery(STempTableNode** pNode, void* pContext) { SCalcConstContext* pCxt = pContext; - pCxt->code = calcConstQuery((*pNode)->pSubquery); + pCxt->code = calcConstQuery(pCxt, (*pNode)->pSubquery); return (TSDB_CODE_SUCCESS == pCxt->code ? DEAL_RES_CONTINUE : DEAL_RES_ERROR); } @@ -185,39 +223,38 @@ static int32_t calcConstCondition(SCalcConstContext* pCxt, SSelectStmt* pSelect, return pCxt->code; } -static int32_t calcConstSelect(SSelectStmt* pSelect) { - SCalcConstContext cxt = { .code = TSDB_CODE_SUCCESS }; - nodesRewriteExprsPostOrder(pSelect->pProjectionList, calcConst, &cxt); - if (TSDB_CODE_SUCCESS == cxt.code) { - cxt.code = calcConstFromTable(&cxt, pSelect); +static int32_t calcConstSelect(SCalcConstContext* pCxt, SSelectStmt* pSelect) { + nodesRewriteExprsPostOrder(pSelect->pProjectionList, calcConst, pCxt); + if (TSDB_CODE_SUCCESS == pCxt->code) { + pCxt->code = calcConstFromTable(pCxt, pSelect); } - if (TSDB_CODE_SUCCESS == cxt.code) { - cxt.code = calcConstCondition(&cxt, pSelect, &pSelect->pWhere); + if (TSDB_CODE_SUCCESS == pCxt->code) { + pCxt->code = calcConstCondition(pCxt, pSelect, &pSelect->pWhere); } - if (TSDB_CODE_SUCCESS == cxt.code) { - nodesRewriteExprsPostOrder(pSelect->pPartitionByList, calcConst, &cxt); + if (TSDB_CODE_SUCCESS == pCxt->code) { + nodesRewriteExprsPostOrder(pSelect->pPartitionByList, calcConst, pCxt); } - if (TSDB_CODE_SUCCESS == cxt.code) { - nodesRewriteExprPostOrder(&pSelect->pWindow, calcConst, &cxt); + if (TSDB_CODE_SUCCESS == pCxt->code) { + nodesRewriteExprPostOrder(&pSelect->pWindow, calcConst, pCxt); } - if (TSDB_CODE_SUCCESS == cxt.code) { - nodesRewriteExprsPostOrder(pSelect->pGroupByList, calcConst, &cxt); + if (TSDB_CODE_SUCCESS == pCxt->code) { + nodesRewriteExprsPostOrder(pSelect->pGroupByList, calcConst, pCxt); } - if (TSDB_CODE_SUCCESS == cxt.code) { - cxt.code = calcConstCondition(&cxt, pSelect, &pSelect->pHaving); + if (TSDB_CODE_SUCCESS == pCxt->code) { + pCxt->code = calcConstCondition(pCxt, pSelect, &pSelect->pHaving); } - if (TSDB_CODE_SUCCESS == cxt.code) { - nodesRewriteExprsPostOrder(pSelect->pOrderByList, calcConst, &cxt); + if (TSDB_CODE_SUCCESS == pCxt->code) { + nodesRewriteExprsPostOrder(pSelect->pOrderByList, calcConst, pCxt); } - return cxt.code; + return pCxt->code; } -static int32_t calcConstQuery(SNode* pStmt) { +static int32_t calcConstQuery(SCalcConstContext* pCxt, SNode* pStmt) { switch (nodeType(pStmt)) { case QUERY_NODE_SELECT_STMT: - return calcConstSelect((SSelectStmt*)pStmt); + return calcConstSelect(pCxt, (SSelectStmt*)pStmt); case QUERY_NODE_EXPLAIN_STMT: - return calcConstQuery(((SExplainStmt*)pStmt)->pQuery); + return calcConstQuery(pCxt, ((SExplainStmt*)pStmt)->pQuery); default: break; } @@ -237,7 +274,13 @@ static bool isEmptyResultQuery(SNode* pStmt) { } int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery) { - int32_t code = calcConstQuery(pQuery->pRoot); + SCalcConstContext cxt = { + .pParseCxt = pParseCxt, + .msgBuf.buf = pParseCxt->pMsg, + .msgBuf.len = pParseCxt->msgLen, + .code = TSDB_CODE_SUCCESS + }; + int32_t code = calcConstQuery(&cxt, pQuery->pRoot); if (TSDB_CODE_SUCCESS == code) { pQuery->execMode = isEmptyResultQuery(pQuery->pRoot) ? QUERY_EXEC_MODE_EMPTY_RESULT : pQuery->execMode; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index f21a738032..eb6c068493 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1115,7 +1115,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { } // FILE csv_file_path - if (TK_NK_FILE == sToken.type) { + if (TK_FILE == sToken.type) { // pSql -> csv_file_path NEXT_TOKEN(pCxt->pSql, sToken); if (0 == sToken.n || (TK_NK_STRING != sToken.type && TK_NK_ID != sToken.type)) { diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 666cdbb6db..fe7835d355 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -148,6 +148,7 @@ static SKeyword keywordTable[] = { {"RESET", TK_RESET}, {"RETENTIONS", TK_RETENTIONS}, {"ROLLUP", TK_ROLLUP}, + {"SCHEMA", TK_SCHEMA}, {"SCORES", TK_SCORES}, {"SELECT", TK_SELECT}, {"SESSION", TK_SESSION}, @@ -202,6 +203,7 @@ static SKeyword keywordTable[] = { {"WATERMARK", TK_WATERMARK}, {"WHERE", TK_WHERE}, {"WINDOW_CLOSE", TK_WINDOW_CLOSE}, + {"WITH", TK_WITH}, {"_QENDTS", TK_QENDTS}, {"_QSTARTTS", TK_QSTARTTS}, {"_ROWTS", TK_ROWTS}, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 8506cfc33d..982703e5cb 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -117,10 +117,8 @@ static int32_t getTableMetaImpl(STranslateContext* pCxt, const SName* pName, STa } static int32_t getTableMeta(STranslateContext* pCxt, const char* pDbName, const char* pTableName, STableMeta** pMeta) { - SName name = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId}; - strcpy(name.dbname, pDbName); - strcpy(name.tname, pTableName); - return getTableMetaImpl(pCxt, &name, pMeta); + SName name; + return getTableMetaImpl(pCxt, toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name), pMeta); } static int32_t getTableDistVgInfo(STranslateContext* pCxt, const SName* pName, SArray** pVgInfo) { @@ -179,10 +177,8 @@ static int32_t getTableHashVgroupImpl(STranslateContext* pCxt, const SName* pNam static int32_t getTableHashVgroup(STranslateContext* pCxt, const char* pDbName, const char* pTableName, SVgroupInfo* pInfo) { - SName name = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId}; - strcpy(name.dbname, pDbName); - strcpy(name.tname, pTableName); - return getTableHashVgroupImpl(pCxt, &name, pInfo); + SName name; + return getTableHashVgroupImpl(pCxt, toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name), pInfo); } static int32_t getDBVgVersion(STranslateContext* pCxt, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, @@ -564,6 +560,9 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { TSDB_DATA_TYPE_BLOB == rdt.type) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); } + if (OP_TYPE_IN == pOp->opType || OP_TYPE_NOT_IN == pOp->opType) { + ((SExprNode*)pOp->pRight)->resType = ((SExprNode*)pOp->pLeft)->resType; + } pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; } else if (nodesIsJsonOp(pOp)){ @@ -933,7 +932,7 @@ static int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STabl return TSDB_CODE_SUCCESS; } } - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_TABLE_NOT_EXIST, pTableAlias); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TABLE_NOT_EXIST, pTableAlias); } static int32_t createTableAllCols(STranslateContext* pCxt, SColumnNode* pCol, SNodeList** pOutput) { @@ -1341,6 +1340,18 @@ static int32_t translateFrom(STranslateContext* pCxt, SSelectStmt* pSelect) { return code; } +static int32_t checkLimit(STranslateContext* pCxt, SSelectStmt* pSelect) { + if ((NULL != pSelect->pLimit && pSelect->pLimit->offset < 0) || (NULL != pSelect->pSlimit && pSelect->pSlimit->offset < 0)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_OFFSET_LESS_ZERO); + } + + if (NULL != pSelect->pSlimit && NULL == pSelect->pPartitionByList) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY); + } + + return TSDB_CODE_SUCCESS; +} + static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->pCurrStmt = pSelect; int32_t code = translateFrom(pCxt, pSelect); @@ -1368,6 +1379,9 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { if (TSDB_CODE_SUCCESS == code) { code = checkAggColCoexist(pCxt, pSelect); } + if (TSDB_CODE_SUCCESS == code) { + code = checkLimit(pCxt, pSelect); + } return code; } @@ -1532,11 +1546,11 @@ static int32_t checkRangeOption(STranslateContext* pCxt, const char* pName, SVal } if (pVal->isDuration && (TIME_UNIT_MINUTE != pVal->unit && TIME_UNIT_HOUR != pVal->unit && TIME_UNIT_DAY != pVal->unit)) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_OPTION_UNIT, pName, pVal->unit); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_OPTION_UNIT, pName, pVal->unit); } int64_t val = getBigintFromValueNode(pVal); if (val < minVal || val > maxVal) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_RANGE_OPTION, pName, val, minVal, maxVal); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_RANGE_OPTION, pName, val, minVal, maxVal); } } return TSDB_CODE_SUCCESS; @@ -1562,7 +1576,7 @@ static int32_t checkDbPrecisionOption(STranslateContext* pCxt, SValueNode* pVal) } else if (0 == strcmp(pRrecision, TSDB_TIME_PRECISION_NANO_STR)) { convertValueFromStrToInt(pVal, TSDB_TIME_PRECISION_NANO); } else { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STR_OPTION, "precision", pVal->datum.p); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STR_OPTION, "precision", pVal->datum.p); } } return TSDB_CODE_SUCCESS; @@ -1575,7 +1589,7 @@ static int32_t checkDbEnumOption(STranslateContext* pCxt, const char* pName, SVa } int64_t val = pVal->datum.i; if (val != v1 && val != v2) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_ENUM_OPTION, pName, val, v1, v2); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ENUM_OPTION, pName, val, v1, v2); } } return TSDB_CODE_SUCCESS; @@ -1588,7 +1602,7 @@ static int32_t checkTtlOption(STranslateContext* pCxt, SValueNode* pVal) { } int64_t val = pVal->datum.i; if (val < TSDB_MIN_DB_TTL) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_TTL_OPTION, val, TSDB_MIN_DB_TTL); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TTL_OPTION, val, TSDB_MIN_DB_TTL); } } return TSDB_CODE_SUCCESS; @@ -1601,7 +1615,7 @@ static int32_t checkKeepOption(STranslateContext* pCxt, SNodeList* pKeep) { int32_t numOfKeep = LIST_LENGTH(pKeep); if (numOfKeep > 3 || numOfKeep < 1) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_KEEP_NUM); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_NUM); } SNode* pNode = NULL; @@ -1632,7 +1646,7 @@ static int32_t checkKeepOption(STranslateContext* pCxt, SNodeList* pKeep) { (TIME_UNIT_MINUTE != pKeep1->unit && TIME_UNIT_HOUR != pKeep1->unit && TIME_UNIT_DAY != pKeep1->unit)) || (pKeep2->isDuration && (TIME_UNIT_MINUTE != pKeep2->unit && TIME_UNIT_HOUR != pKeep2->unit && TIME_UNIT_DAY != pKeep2->unit))) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_KEEP_UNIT, pKeep0->unit, pKeep1->unit, pKeep2->unit); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_UNIT, pKeep0->unit, pKeep1->unit, pKeep2->unit); } int32_t daysToKeep0 = getBigintFromValueNode(pKeep0); @@ -1640,12 +1654,12 @@ static int32_t checkKeepOption(STranslateContext* pCxt, SNodeList* pKeep) { int32_t daysToKeep2 = getBigintFromValueNode(pKeep2); if (daysToKeep0 < TSDB_MIN_KEEP || daysToKeep1 < TSDB_MIN_KEEP || daysToKeep2 < TSDB_MIN_KEEP || daysToKeep0 > TSDB_MAX_KEEP || daysToKeep1 > TSDB_MAX_KEEP || daysToKeep2 > TSDB_MAX_KEEP) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_KEEP_VALUE, daysToKeep0, daysToKeep1, daysToKeep2, + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_VALUE, daysToKeep0, daysToKeep1, daysToKeep2, TSDB_MIN_KEEP, TSDB_MAX_KEEP); } if (!((daysToKeep0 <= daysToKeep1) && (daysToKeep1 <= daysToKeep2))) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_KEEP_ORDER); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_ORDER); } return TSDB_CODE_SUCCESS; @@ -1657,7 +1671,7 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete } if (LIST_LENGTH(pRetentions) > 3) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION); } SNode* pNode = NULL; @@ -1673,7 +1687,28 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete return TSDB_CODE_SUCCESS; } -static int32_t checkDatabaseOptions(STranslateContext* pCxt, SDatabaseOptions* pOptions) { +static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions, bool alter) { + if (NULL == pOptions->pDaysPerFile && NULL == pOptions->pKeep) { + return TSDB_CODE_SUCCESS; + } + int64_t daysPerFile = GET_OPTION_VAL(pOptions->pDaysPerFile, alter ? -1 : TSDB_DEFAULT_DAYS_PER_FILE); + int64_t daysToKeep0 = GET_OPTION_VAL(nodesListGetNode(pOptions->pKeep, 0), alter ? -1 : TSDB_DEFAULT_KEEP); + if (alter && (-1 == daysPerFile || -1 == daysToKeep0)) { + SDbCfgInfo dbCfg; + int32_t code = getDBCfg(pCxt, pDbName, &dbCfg); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + daysPerFile = (-1 == daysPerFile ? dbCfg.daysPerFile : daysPerFile); + daysToKeep0 = (-1 == daysPerFile ? dbCfg.daysToKeep0 : daysToKeep0); + } + if (daysPerFile > daysToKeep0) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DAYS_VALUE); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions, bool alter) { int32_t code = checkRangeOption(pCxt, "totalBlocks", pOptions->pNumOfBlocks, TSDB_MIN_TOTAL_BLOCKS, TSDB_MAX_TOTAL_BLOCKS); if (TSDB_CODE_SUCCESS == code) { @@ -1734,11 +1769,14 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, SDatabaseOptions* p if (TSDB_CODE_SUCCESS == code) { code = checkDbEnumOption(pCxt, "strict", pOptions->pStrict, TSDB_DB_STRICT_OFF, TSDB_DB_STRICT_ON); } + if (TSDB_CODE_SUCCESS == code) { + code = checkOptionsDependency(pCxt, pDbName, pOptions, alter); + } return code; } static int32_t checkCreateDatabase(STranslateContext* pCxt, SCreateDatabaseStmt* pStmt) { - return checkDatabaseOptions(pCxt, pStmt->pOptions); + return checkDatabaseOptions(pCxt, pStmt->dbName, pStmt->pOptions, false); } typedef int32_t (*FSerializeFunc)(void* pBuf, int32_t bufLen, void* pReq); @@ -1802,7 +1840,7 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt, } static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt) { - int32_t code = checkDatabaseOptions(pCxt, pStmt->pOptions); + int32_t code = checkDatabaseOptions(pCxt, pStmt->dbName, pStmt->pOptions, true); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -1863,7 +1901,7 @@ static int32_t checkTableCommentOption(STranslateContext* pCxt, SValueNode* pVal return pCxt->errCode; } if (pVal->node.resType.bytes >= TSDB_STB_COMMENT_LEN) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COMMENT_OPTION, TSDB_STB_COMMENT_LEN - 1); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COMMENT_OPTION, TSDB_STB_COMMENT_LEN - 1); } } return TSDB_CODE_SUCCESS; @@ -1875,7 +1913,7 @@ static int32_t checTableFactorOption(STranslateContext* pCxt, SValueNode* pVal) return pCxt->errCode; } if (pVal->datum.d < TSDB_MIN_DB_FILE_FACTOR || pVal->datum.d > TSDB_MAX_DB_FILE_FACTOR) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_F_RANGE_OPTION, "file_factor", pVal->datum.d, + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_F_RANGE_OPTION, "file_factor", pVal->datum.d, TSDB_MIN_DB_FILE_FACTOR, TSDB_MAX_DB_FILE_FACTOR); } } @@ -1916,12 +1954,12 @@ static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs } if (1 != LIST_LENGTH(pFuncs)) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_ROLLUP_OPTION); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ROLLUP_OPTION); } SFunctionNode* pFunc = nodesListGetNode(pFuncs, 0); SFmGetFuncInfoParam param = { .pCtg = pCxt->pParseCxt->pCatalog, .pRpc = pCxt->pParseCxt->pTransporter, .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet}; if (TSDB_CODE_SUCCESS != fmGetFuncInfo(¶m, pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName); } return TSDB_CODE_SUCCESS; } @@ -2195,10 +2233,8 @@ static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStm pReq->numOfSmas = LIST_LENGTH(pStmt->pOptions->pSma); } - SName tableName = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId}; - strcpy(tableName.dbname, pStmt->dbName); - strcpy(tableName.tname, pStmt->tableName); - tNameExtractFullName(&tableName, pReq->name); + SName tableName; + tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pReq->name); return buildRollupAst(pCxt, pStmt, pReq); } @@ -2248,10 +2284,8 @@ static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt } static int32_t translateDropSuperTable(STranslateContext* pCxt, SDropSuperTableStmt* pStmt) { - SName tableName = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId}; - strcpy(tableName.dbname, pStmt->dbName); - strcpy(tableName.tname, pStmt->tableName); - return doTranslateDropSuperTable(pCxt, &tableName, pStmt->ignoreNotExists); + SName tableName; + return doTranslateDropSuperTable(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pStmt->ignoreNotExists); } static int32_t setAlterTableField(SAlterTableStmt* pStmt, SMAltertbReq* pAlterReq) { @@ -2291,10 +2325,8 @@ static int32_t setAlterTableField(SAlterTableStmt* pStmt, SMAltertbReq* pAlterRe static int32_t translateAlterTable(STranslateContext* pCxt, SAlterTableStmt* pStmt) { SMAltertbReq alterReq = {0}; - SName tableName = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId}; - strcpy(tableName.dbname, pStmt->dbName); - strcpy(tableName.tname, pStmt->tableName); - tNameExtractFullName(&tableName, alterReq.name); + SName tableName; + tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), alterReq.name); alterReq.alterType = pStmt->alterType; alterReq.numOfFields = 1; if (TSDB_ALTER_TABLE_UPDATE_OPTIONS == pStmt->alterType) { @@ -2384,8 +2416,6 @@ static int32_t nodeTypeToShowType(ENodeType nt) { return TSDB_MGMT_TABLE_GRANTS; case QUERY_NODE_SHOW_QUERIES_STMT: return TSDB_MGMT_TABLE_QUERIES; - case QUERY_NODE_SHOW_TOPICS_STMT: - return 0; // todo case QUERY_NODE_SHOW_VARIABLE_STMT: return 0; // todo default: @@ -2447,10 +2477,8 @@ static int32_t getSmaIndexAst(STranslateContext* pCxt, SCreateIndexStmt* pStmt, } static int32_t buildCreateSmaReq(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SMCreateSmaReq* pReq) { - SName name = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId}; - strcpy(name.dbname, pCxt->pParseCxt->db); - strcpy(name.tname, pStmt->indexName); - tNameExtractFullName(&name, pReq->name); + SName name; + tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->indexName, &name), pReq->name); strcpy(name.tname, pStmt->tableName); name.tname[strlen(pStmt->tableName)] = '\0'; tNameExtractFullName(&name, pReq->stb); @@ -2605,11 +2633,14 @@ static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* p return TSDB_CODE_OUT_OF_MEMORY; } - SName name = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId}; - strcpy(name.dbname, pCxt->pParseCxt->db); - strcpy(name.tname, pStmt->topicName); - tNameExtractFullName(&name, createReq.name); + SName name; + // tNameSetDbName(&name, pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, strlen(pCxt->pParseCxt->db)); + // tNameGetFullDbName(&name, createReq.name); + tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->topicName, &name), createReq.name); createReq.igExists = pStmt->ignoreExists; + createReq.withTbName = pStmt->pOptions->withTable; + createReq.withSchema = pStmt->pOptions->withSchema; + createReq.withTag = pStmt->pOptions->withTag; int32_t code = buildCmdMsg(pCxt, TDMT_MND_CREATE_TOPIC, (FSerializeFunc)tSerializeSCMCreateTopicReq, &createReq); tFreeSCMCreateTopicReq(&createReq); @@ -2619,10 +2650,8 @@ static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* p static int32_t translateDropTopic(STranslateContext* pCxt, SDropTopicStmt* pStmt) { SMDropTopicReq dropReq = {0}; - SName name = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId}; - strcpy(name.dbname, pCxt->pParseCxt->db); - strcpy(name.tname, pStmt->topicName); - tNameExtractFullName(&name, dropReq.name); + SName name; + tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->topicName, &name), dropReq.name); dropReq.igNotExists = pStmt->ignoreNotExists; return buildCmdMsg(pCxt, TDMT_MND_DROP_TOPIC, (FSerializeFunc)tSerializeSMDropTopicReq, &dropReq); @@ -2661,10 +2690,8 @@ static int32_t translateCreateStream(STranslateContext* pCxt, SCreateStreamStmt* createReq.igExists = pStmt->ignoreExists; - SName name = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId}; - strcpy(name.dbname, pCxt->pParseCxt->db); - strcpy(name.tname, pStmt->streamName); - tNameExtractFullName(&name, createReq.name); + SName name; + tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->streamName, &name), createReq.name); if ('\0' != pStmt->targetTabName[0]) { strcpy(name.dbname, pStmt->targetDbName); @@ -2979,6 +3006,7 @@ static const char* getSysDbName(ENodeType type) { return TSDB_INFORMATION_SCHEMA_DB; case QUERY_NODE_SHOW_CONNECTIONS_STMT: case QUERY_NODE_SHOW_QUERIES_STMT: + case QUERY_NODE_SHOW_TOPICS_STMT: return TSDB_PERFORMANCE_SCHEMA_DB; default: break; @@ -3024,6 +3052,8 @@ static const char* getSysTableName(ENodeType type) { return TSDB_PERFS_TABLE_CONNECTIONS; case QUERY_NODE_SHOW_QUERIES_STMT: return TSDB_PERFS_TABLE_QUERIES; + case QUERY_NODE_SHOW_TOPICS_STMT: + return TSDB_PERFS_TABLE_TOPICS; default: break; } @@ -3545,6 +3575,7 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { case QUERY_NODE_SHOW_CONNECTIONS_STMT: case QUERY_NODE_SHOW_QUERIES_STMT: case QUERY_NODE_SHOW_CLUSTER_STMT: + case QUERY_NODE_SHOW_TOPICS_STMT: code = rewriteShow(pCxt, pQuery); break; case QUERY_NODE_CREATE_TABLE_STMT: diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 075caa868c..7e41bbe3fd 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -23,15 +23,15 @@ static char* getSyntaxErrFormat(int32_t errCode) { case TSDB_CODE_PAR_INCOMPLETE_SQL: return "Incomplete SQL statement"; case TSDB_CODE_PAR_INVALID_COLUMN: - return "Invalid column name : %s"; + return "Invalid column name: %s"; case TSDB_CODE_PAR_TABLE_NOT_EXIST: - return "Table does not exist : %s"; + return "Table does not exist: %s"; case TSDB_CODE_PAR_AMBIGUOUS_COLUMN: - return "Column ambiguously defined : %s"; + return "Column ambiguously defined: %s"; case TSDB_CODE_PAR_WRONG_VALUE_TYPE: - return "Invalid value type : %s"; + return "Invalid value type: %s"; case TSDB_CODE_PAR_INVALID_FUNTION: - return "Invalid function name : %s"; + return "Invalid function name: %s"; case TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION: return "There mustn't be aggregation"; case TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT: @@ -45,7 +45,7 @@ static char* getSyntaxErrFormat(int32_t errCode) { case TSDB_CODE_PAR_TAGS_NOT_MATCHED: return "Tags number not matched"; case TSDB_CODE_PAR_INVALID_TAG_NAME: - return "Invalid tag name : %s"; + return "Invalid tag name: %s"; case TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG: return "Name or password too long"; case TSDB_CODE_PAR_PASSWD_EMPTY: @@ -61,7 +61,7 @@ static char* getSyntaxErrFormat(int32_t errCode) { case TSDB_CODE_PAR_DB_NOT_SPECIFIED: return "Database not specified"; case TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME: - return "Invalid identifier name : %s"; + return "Invalid identifier name: %s"; case TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR: return "Corresponding super table not in this db"; case TSDB_CODE_PAR_INVALID_RANGE_OPTION: @@ -120,6 +120,14 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Only one tag if there is a json tag"; case TSDB_CODE_PAR_INCORRECT_NUM_OF_COL: return "Query block has incorrect number of result columns"; + case TSDB_CODE_PAR_INCORRECT_TIMESTAMP_VAL: + return "Incorrect TIMESTAMP value: %s"; + case TSDB_CODE_PAR_INVALID_DAYS_VALUE: + return "Invalid days value, should be keep2 >= keep1 >= keep0 >= days"; + case TSDB_CODE_PAR_OFFSET_LESS_ZERO: + return "soffset/offset can not be less than 0"; + case TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY: + return "slimit/soffset only available for PARTITION BY query"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index b38e4092e3..5e91a15025 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -100,24 +100,24 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 336 +#define YYNOCODE 346 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SToken typedef union { int yyinit; ParseTOKENTYPE yy0; - int32_t yy4; - bool yy89; - EFillMode yy102; - SDataType yy112; - SAlterOption yy221; - ENullOrder yy361; - EJoinType yy372; - SNodeList* yy376; - EOperatorType yy380; - EOrder yy386; - SNode* yy392; - SToken yy449; + SAlterOption yy145; + EOrder yy250; + EOperatorType yy348; + int32_t yy376; + SDataType yy380; + SNode* yy456; + SToken yy517; + EFillMode yy534; + ENullOrder yy645; + SNodeList* yy652; + bool yy673; + EJoinType yy684; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -132,17 +132,18 @@ typedef union { #define ParseCTX_PARAM #define ParseCTX_FETCH #define ParseCTX_STORE -#define YYNSTATE 572 -#define YYNRULE 437 -#define YYNTOKEN 222 -#define YY_MAX_SHIFT 571 -#define YY_MIN_SHIFTREDUCE 850 -#define YY_MAX_SHIFTREDUCE 1286 -#define YY_ERROR_ACTION 1287 -#define YY_ACCEPT_ACTION 1288 -#define YY_NO_ACTION 1289 -#define YY_MIN_REDUCE 1290 -#define YY_MAX_REDUCE 1726 +#define YYFALLBACK 1 +#define YYNSTATE 574 +#define YYNRULE 441 +#define YYNTOKEN 231 +#define YY_MAX_SHIFT 573 +#define YY_MIN_SHIFTREDUCE 855 +#define YY_MAX_SHIFTREDUCE 1295 +#define YY_ERROR_ACTION 1296 +#define YY_ACCEPT_ACTION 1297 +#define YY_NO_ACTION 1298 +#define YY_MIN_REDUCE 1299 +#define YY_MAX_REDUCE 1739 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -209,553 +210,544 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (1960) +#define YY_ACTTAB_COUNT (1906) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 270, 1593, 287, 472, 1577, 485, 282, 325, 469, 1505, - /* 10 */ 1288, 1401, 33, 31, 78, 1577, 124, 1563, 1302, 1563, - /* 20 */ 279, 383, 1106, 34, 32, 30, 29, 28, 1593, 1563, - /* 30 */ 1412, 1559, 1565, 1559, 1565, 469, 125, 250, 1104, 1593, - /* 40 */ 1369, 1705, 438, 1559, 1566, 468, 469, 1389, 445, 1549, - /* 50 */ 12, 33, 31, 1228, 1704, 54, 468, 1112, 1702, 279, - /* 60 */ 1549, 1106, 484, 295, 319, 449, 1577, 243, 1578, 471, - /* 70 */ 1580, 1581, 467, 105, 462, 1, 1408, 1104, 73, 1578, - /* 80 */ 471, 1580, 1581, 467, 525, 462, 1384, 290, 1643, 12, - /* 90 */ 1593, 484, 251, 1639, 1496, 1498, 1112, 448, 568, 26, - /* 100 */ 205, 362, 1705, 1705, 1705, 441, 421, 468, 107, 103, - /* 110 */ 1105, 1549, 516, 485, 1, 137, 137, 137, 1129, 1702, - /* 120 */ 1702, 1702, 323, 447, 133, 1650, 1651, 445, 1655, 74, - /* 130 */ 1578, 471, 1580, 1581, 467, 519, 462, 568, 1412, 1643, - /* 140 */ 54, 144, 485, 272, 1639, 132, 36, 940, 1107, 1105, - /* 150 */ 422, 324, 105, 100, 515, 514, 513, 201, 512, 1127, - /* 160 */ 439, 1407, 1130, 428, 1670, 435, 942, 1412, 52, 1110, - /* 170 */ 1111, 51, 1155, 1156, 1157, 1158, 1159, 1160, 1161, 464, - /* 180 */ 1166, 1167, 1168, 1169, 1170, 1171, 1172, 1107, 103, 1705, - /* 190 */ 1461, 34, 32, 30, 29, 28, 269, 9, 8, 138, - /* 200 */ 69, 1459, 137, 134, 1650, 1651, 1702, 1655, 1110, 1111, - /* 210 */ 65, 1155, 1156, 1157, 1158, 1159, 1160, 1161, 464, 1166, - /* 220 */ 1167, 1168, 1169, 1170, 1171, 1172, 33, 31, 138, 252, - /* 230 */ 227, 440, 436, 1442, 279, 249, 1106, 1127, 1577, 399, - /* 240 */ 63, 393, 521, 1461, 341, 398, 70, 353, 102, 284, - /* 250 */ 394, 392, 1104, 395, 1459, 1142, 354, 318, 391, 317, - /* 260 */ 106, 1405, 1593, 1190, 12, 33, 31, 1404, 1227, 469, - /* 270 */ 138, 1112, 1204, 279, 22, 1106, 1657, 56, 268, 468, - /* 280 */ 1252, 175, 24, 1549, 34, 32, 30, 29, 28, 1, - /* 290 */ 1657, 1104, 34, 32, 30, 29, 28, 30, 29, 28, - /* 300 */ 1654, 74, 1578, 471, 1580, 1581, 467, 128, 462, 138, - /* 310 */ 1112, 1643, 568, 1191, 1653, 272, 1639, 1717, 1453, 432, - /* 320 */ 1250, 1251, 1253, 1254, 1105, 413, 1677, 1390, 7, 1290, - /* 330 */ 352, 1196, 1128, 347, 346, 345, 344, 343, 138, 340, - /* 340 */ 339, 338, 337, 336, 332, 331, 330, 329, 328, 327, - /* 350 */ 326, 568, 59, 98, 97, 96, 95, 94, 93, 92, - /* 360 */ 91, 90, 1107, 1105, 1705, 1705, 25, 277, 1185, 1186, - /* 370 */ 1187, 1188, 1189, 1193, 1194, 1195, 362, 137, 1703, 1403, - /* 380 */ 1131, 1702, 1702, 1110, 1111, 6, 1155, 1156, 1157, 1158, - /* 390 */ 1159, 1160, 1161, 464, 1166, 1167, 1168, 1169, 1170, 1171, - /* 400 */ 1172, 1107, 524, 484, 452, 34, 32, 30, 29, 28, - /* 410 */ 89, 564, 563, 88, 87, 86, 85, 84, 83, 82, - /* 420 */ 81, 80, 1110, 1111, 1549, 1155, 1156, 1157, 1158, 1159, - /* 430 */ 1160, 1161, 464, 1166, 1167, 1168, 1169, 1170, 1171, 1172, - /* 440 */ 33, 31, 1173, 252, 1291, 536, 1313, 283, 279, 313, - /* 450 */ 1106, 289, 138, 485, 472, 122, 262, 101, 36, 122, - /* 460 */ 1506, 386, 333, 1414, 1226, 89, 1104, 1414, 88, 87, - /* 470 */ 86, 85, 84, 83, 82, 81, 80, 1190, 1412, 33, - /* 480 */ 31, 48, 101, 1387, 389, 1112, 386, 279, 1312, 1106, - /* 490 */ 1577, 1549, 34, 32, 30, 29, 28, 485, 485, 263, - /* 500 */ 200, 261, 260, 7, 385, 1104, 78, 334, 511, 389, - /* 510 */ 388, 387, 292, 390, 1593, 34, 32, 30, 29, 28, - /* 520 */ 122, 469, 1412, 1412, 1112, 485, 568, 1191, 1414, 1311, - /* 530 */ 1461, 468, 1493, 1549, 361, 1549, 291, 485, 1105, 146, - /* 540 */ 1223, 1459, 7, 1192, 107, 1196, 1409, 138, 516, 453, - /* 550 */ 1412, 397, 396, 74, 1578, 471, 1580, 1581, 467, 1461, - /* 560 */ 462, 1197, 1412, 1643, 1342, 568, 1132, 272, 1639, 1717, - /* 570 */ 1460, 519, 348, 1397, 1549, 1657, 1107, 1105, 1700, 1242, - /* 580 */ 25, 277, 1185, 1186, 1187, 1188, 1189, 1193, 1194, 1195, - /* 590 */ 515, 514, 513, 1577, 512, 1538, 23, 1110, 1111, 1652, - /* 600 */ 1155, 1156, 1157, 1158, 1159, 1160, 1161, 464, 1166, 1167, - /* 610 */ 1168, 1169, 1170, 1171, 1172, 1107, 399, 1593, 393, 148, - /* 620 */ 147, 1283, 398, 1310, 448, 102, 166, 394, 392, 164, - /* 630 */ 395, 302, 518, 517, 468, 391, 1110, 1111, 1549, 1155, - /* 640 */ 1156, 1157, 1158, 1159, 1160, 1161, 464, 1166, 1167, 1168, - /* 650 */ 1169, 1170, 1171, 1172, 33, 31, 74, 1578, 471, 1580, - /* 660 */ 1581, 467, 279, 462, 1106, 1577, 1643, 1388, 1549, 485, - /* 670 */ 272, 1639, 132, 34, 32, 30, 29, 28, 1530, 485, - /* 680 */ 1104, 1309, 122, 485, 445, 887, 186, 886, 482, 1593, - /* 690 */ 1415, 1671, 293, 451, 1412, 1399, 469, 537, 535, 1112, - /* 700 */ 1282, 1235, 485, 886, 1412, 888, 468, 1129, 1412, 105, - /* 710 */ 1549, 483, 485, 1308, 1307, 310, 521, 1, 1339, 381, - /* 720 */ 1306, 219, 1305, 1304, 1301, 1395, 1549, 1412, 74, 1578, - /* 730 */ 471, 1580, 1581, 467, 312, 462, 168, 1412, 1643, 167, - /* 740 */ 568, 455, 272, 1639, 1717, 103, 34, 32, 30, 29, - /* 750 */ 28, 1300, 1105, 1661, 1299, 1298, 178, 1297, 1549, 1549, - /* 760 */ 135, 1650, 1651, 1296, 1655, 1549, 459, 1549, 1549, 1549, - /* 770 */ 544, 543, 542, 541, 294, 463, 540, 539, 538, 108, - /* 780 */ 533, 532, 531, 530, 529, 528, 527, 526, 115, 522, - /* 790 */ 1107, 1178, 1497, 1498, 1662, 1223, 1549, 1129, 510, 1549, - /* 800 */ 1549, 121, 1549, 1295, 1092, 1093, 1294, 1293, 1549, 113, - /* 810 */ 1142, 1110, 1111, 424, 1155, 1156, 1157, 1158, 1159, 1160, - /* 820 */ 1161, 464, 1166, 1167, 1168, 1169, 1170, 1171, 1172, 1570, - /* 830 */ 978, 508, 507, 506, 982, 505, 984, 985, 504, 987, - /* 840 */ 501, 1568, 993, 498, 995, 996, 495, 492, 1549, 123, - /* 850 */ 1577, 1549, 1549, 170, 233, 1329, 169, 172, 1324, 411, - /* 860 */ 171, 1322, 1115, 1577, 47, 189, 231, 420, 1249, 191, - /* 870 */ 9, 8, 409, 1114, 1593, 1285, 1286, 400, 35, 149, - /* 880 */ 402, 469, 1198, 405, 1303, 1182, 35, 1593, 456, 35, - /* 890 */ 1162, 468, 208, 1067, 469, 1549, 210, 110, 1370, 111, - /* 900 */ 449, 477, 113, 216, 468, 47, 971, 202, 1549, 966, - /* 910 */ 433, 1454, 414, 240, 1578, 471, 1580, 1581, 467, 490, - /* 920 */ 462, 111, 195, 999, 380, 1003, 75, 1578, 471, 1580, - /* 930 */ 1581, 467, 446, 462, 1118, 1673, 1643, 1594, 112, 1705, - /* 940 */ 1642, 1639, 1010, 72, 1577, 1117, 204, 912, 2, 301, - /* 950 */ 161, 113, 137, 131, 111, 1009, 1702, 1127, 114, 379, - /* 960 */ 297, 375, 371, 367, 160, 257, 913, 259, 1593, 940, - /* 970 */ 224, 335, 50, 49, 322, 469, 143, 1495, 1076, 342, - /* 980 */ 145, 316, 350, 349, 351, 468, 355, 1136, 356, 1549, - /* 990 */ 150, 55, 357, 258, 158, 308, 1577, 304, 300, 140, - /* 1000 */ 1135, 358, 153, 359, 1134, 360, 156, 75, 1578, 471, - /* 1010 */ 1580, 1581, 467, 53, 462, 363, 159, 1643, 1133, 571, - /* 1020 */ 1593, 458, 1639, 382, 384, 1402, 79, 466, 163, 267, - /* 1030 */ 138, 1398, 1112, 223, 165, 116, 99, 468, 117, 1400, - /* 1040 */ 1396, 1549, 560, 118, 556, 552, 548, 222, 119, 176, - /* 1050 */ 225, 1577, 157, 415, 152, 423, 154, 1534, 179, 247, - /* 1060 */ 1578, 471, 1580, 1581, 467, 465, 462, 460, 1615, 419, - /* 1070 */ 181, 425, 426, 151, 71, 1593, 416, 217, 1132, 434, - /* 1080 */ 184, 187, 469, 1684, 475, 1674, 431, 1683, 5, 190, - /* 1090 */ 442, 271, 468, 437, 430, 4, 1549, 1223, 104, 1131, - /* 1100 */ 1577, 273, 37, 1720, 457, 454, 197, 481, 1658, 1664, - /* 1110 */ 16, 473, 1624, 474, 126, 1578, 471, 1580, 1581, 467, - /* 1120 */ 194, 462, 130, 1504, 1593, 196, 478, 1503, 226, 212, - /* 1130 */ 281, 469, 479, 214, 480, 62, 427, 1701, 203, 182, - /* 1140 */ 1413, 468, 64, 488, 221, 1549, 1385, 228, 567, 129, - /* 1150 */ 230, 234, 1577, 43, 1084, 235, 177, 232, 450, 1718, - /* 1160 */ 445, 1543, 1542, 75, 1578, 471, 1580, 1581, 467, 296, - /* 1170 */ 462, 1539, 298, 1643, 299, 1100, 1593, 1101, 1640, 141, - /* 1180 */ 303, 1537, 1577, 469, 305, 105, 306, 307, 1536, 309, - /* 1190 */ 1535, 1520, 311, 468, 142, 314, 315, 1549, 1079, 1078, - /* 1200 */ 429, 1514, 1513, 320, 449, 321, 1593, 1512, 1511, 1050, - /* 1210 */ 1488, 1487, 1486, 469, 109, 248, 1578, 471, 1580, 1581, - /* 1220 */ 467, 103, 462, 468, 1485, 1484, 1483, 1549, 1482, 1481, - /* 1230 */ 1480, 1479, 1478, 1577, 1477, 1476, 198, 1650, 444, 1475, - /* 1240 */ 443, 1474, 1473, 1705, 1472, 126, 1578, 471, 1580, 1581, - /* 1250 */ 467, 1471, 462, 1470, 1469, 1468, 137, 1593, 1467, 1466, - /* 1260 */ 1702, 1465, 1052, 1577, 469, 1464, 1463, 1462, 1341, 1528, - /* 1270 */ 1522, 1510, 1501, 1391, 468, 162, 155, 1340, 1549, 1338, - /* 1280 */ 1336, 276, 365, 364, 369, 368, 366, 1593, 905, 1334, - /* 1290 */ 1719, 1577, 370, 374, 466, 372, 248, 1578, 471, 1580, - /* 1300 */ 1581, 467, 1332, 462, 468, 378, 1321, 1320, 1549, 1317, - /* 1310 */ 373, 1393, 1015, 1392, 1018, 1593, 376, 377, 1330, 264, - /* 1320 */ 77, 939, 469, 938, 534, 937, 247, 1578, 471, 1580, - /* 1330 */ 1581, 467, 468, 462, 936, 1616, 1549, 935, 932, 278, - /* 1340 */ 1577, 931, 286, 285, 1325, 1323, 265, 403, 266, 1106, - /* 1350 */ 406, 1316, 1120, 408, 248, 1578, 471, 1580, 1581, 467, - /* 1360 */ 536, 462, 1315, 410, 1593, 1104, 76, 1527, 1113, 1521, - /* 1370 */ 1086, 469, 120, 1509, 417, 180, 1508, 1500, 1577, 57, - /* 1380 */ 183, 468, 3, 13, 1112, 1549, 35, 1112, 280, 1577, - /* 1390 */ 14, 188, 41, 1248, 127, 38, 46, 192, 185, 11, - /* 1400 */ 418, 1241, 1593, 248, 1578, 471, 1580, 1581, 467, 469, - /* 1410 */ 462, 1220, 193, 1593, 20, 1568, 199, 58, 1271, 468, - /* 1420 */ 469, 21, 40, 1549, 1219, 568, 39, 15, 486, 1276, - /* 1430 */ 468, 1270, 274, 1275, 1549, 1274, 275, 1105, 8, 1577, - /* 1440 */ 1116, 236, 1578, 471, 1580, 1581, 467, 136, 462, 461, - /* 1450 */ 17, 1165, 242, 1578, 471, 1580, 1581, 467, 1164, 462, - /* 1460 */ 1183, 27, 10, 1593, 1163, 18, 139, 1150, 206, 470, - /* 1470 */ 469, 19, 476, 1499, 209, 1107, 207, 1246, 1121, 211, - /* 1480 */ 468, 60, 213, 61, 1549, 215, 42, 65, 1122, 1577, - /* 1490 */ 1567, 1000, 489, 288, 218, 487, 1110, 1111, 491, 1124, - /* 1500 */ 1577, 997, 244, 1578, 471, 1580, 1581, 467, 493, 462, - /* 1510 */ 494, 977, 496, 1593, 994, 497, 499, 988, 500, 502, - /* 1520 */ 469, 992, 986, 503, 1593, 509, 991, 990, 989, 1012, - /* 1530 */ 468, 469, 66, 67, 1549, 68, 1008, 1005, 903, 1011, - /* 1540 */ 520, 468, 928, 946, 523, 1549, 926, 220, 1577, 925, - /* 1550 */ 924, 923, 237, 1578, 471, 1580, 1581, 467, 922, 462, - /* 1560 */ 921, 1577, 404, 245, 1578, 471, 1580, 1581, 467, 920, - /* 1570 */ 462, 919, 1593, 943, 941, 916, 915, 412, 914, 469, - /* 1580 */ 911, 910, 909, 908, 1337, 1593, 545, 546, 1335, 468, - /* 1590 */ 547, 174, 469, 1549, 407, 549, 550, 551, 1333, 401, - /* 1600 */ 553, 554, 468, 1331, 555, 173, 1549, 557, 558, 559, - /* 1610 */ 1319, 238, 1578, 471, 1580, 1581, 467, 561, 462, 562, - /* 1620 */ 1318, 1314, 565, 566, 246, 1578, 471, 1580, 1581, 467, - /* 1630 */ 1577, 462, 45, 1108, 229, 44, 569, 570, 1289, 1577, - /* 1640 */ 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, - /* 1650 */ 1289, 1289, 1289, 1289, 1593, 1289, 1289, 1289, 1289, 1289, - /* 1660 */ 1289, 469, 1289, 1593, 1289, 1289, 1289, 1289, 1289, 1289, - /* 1670 */ 469, 468, 1289, 1289, 1289, 1549, 1289, 1289, 1289, 1289, - /* 1680 */ 468, 1289, 1289, 1289, 1549, 1289, 1289, 1289, 1289, 1577, - /* 1690 */ 1289, 1289, 1289, 239, 1578, 471, 1580, 1581, 467, 1289, - /* 1700 */ 462, 1289, 1589, 1578, 471, 1580, 1581, 467, 1289, 462, - /* 1710 */ 1289, 1289, 1289, 1593, 1289, 1289, 1289, 1289, 1289, 1289, - /* 1720 */ 469, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, - /* 1730 */ 468, 1289, 1289, 1289, 1549, 1289, 1289, 1289, 1289, 1577, - /* 1740 */ 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, - /* 1750 */ 1577, 1289, 1588, 1578, 471, 1580, 1581, 467, 1289, 462, - /* 1760 */ 1289, 1289, 1289, 1593, 1289, 1289, 1289, 1289, 1289, 1289, - /* 1770 */ 469, 1289, 1289, 1289, 1593, 1289, 1289, 1289, 1289, 1289, - /* 1780 */ 468, 469, 1289, 1289, 1549, 1289, 1289, 1289, 1289, 1289, - /* 1790 */ 1289, 468, 1289, 1289, 1289, 1549, 1289, 1289, 1577, 1289, - /* 1800 */ 1289, 1289, 1587, 1578, 471, 1580, 1581, 467, 1289, 462, - /* 1810 */ 1289, 1577, 1289, 255, 1578, 471, 1580, 1581, 467, 1289, - /* 1820 */ 462, 1289, 1593, 1289, 1289, 1289, 1289, 1289, 1289, 469, - /* 1830 */ 1289, 1289, 1289, 1289, 1289, 1593, 1289, 1289, 1289, 468, - /* 1840 */ 1289, 1289, 469, 1549, 1289, 1289, 1289, 1289, 1289, 1289, - /* 1850 */ 1289, 1289, 468, 1289, 1289, 1289, 1549, 1289, 1289, 1289, - /* 1860 */ 1289, 254, 1578, 471, 1580, 1581, 467, 1289, 462, 1289, - /* 1870 */ 1289, 1289, 1289, 1289, 256, 1578, 471, 1580, 1581, 467, - /* 1880 */ 1577, 462, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1577, - /* 1890 */ 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, - /* 1900 */ 1289, 1289, 1289, 1289, 1593, 1289, 1289, 1289, 1289, 1289, - /* 1910 */ 1289, 469, 1289, 1593, 1289, 1289, 1289, 1289, 1289, 1289, - /* 1920 */ 469, 468, 1289, 1289, 1289, 1549, 1289, 1289, 1289, 1289, - /* 1930 */ 468, 1289, 1289, 1289, 1549, 1289, 1289, 1289, 1289, 1289, - /* 1940 */ 1289, 1289, 1289, 253, 1578, 471, 1580, 1581, 467, 1289, - /* 1950 */ 462, 1289, 241, 1578, 471, 1580, 1581, 467, 1289, 462, + /* 0 */ 487, 1718, 1590, 1502, 273, 474, 322, 290, 285, 78, + /* 10 */ 146, 1514, 33, 31, 1717, 1410, 386, 407, 1715, 487, + /* 20 */ 282, 1576, 1115, 1470, 1576, 1421, 1606, 125, 78, 272, + /* 30 */ 447, 1378, 415, 450, 1468, 393, 1572, 1578, 1113, 1572, + /* 40 */ 1578, 26, 205, 470, 1421, 1718, 174, 1562, 1576, 410, + /* 50 */ 12, 33, 31, 1237, 404, 105, 486, 1121, 137, 282, + /* 60 */ 173, 1115, 1715, 1572, 1579, 54, 74, 1591, 473, 1593, + /* 70 */ 1594, 469, 486, 464, 59, 1, 1656, 1113, 100, 487, + /* 80 */ 275, 1652, 132, 30, 29, 28, 1416, 45, 326, 12, + /* 90 */ 44, 54, 103, 486, 201, 293, 1121, 286, 570, 1348, + /* 100 */ 430, 1683, 1505, 1507, 1421, 122, 449, 133, 1663, 1664, + /* 110 */ 1114, 1668, 1417, 1423, 1, 1300, 983, 510, 509, 508, + /* 120 */ 987, 507, 989, 990, 506, 992, 503, 36, 998, 500, + /* 130 */ 1000, 1001, 497, 494, 9, 8, 89, 570, 1136, 88, + /* 140 */ 87, 86, 85, 84, 83, 82, 81, 80, 36, 1114, + /* 150 */ 1116, 546, 545, 544, 543, 297, 365, 542, 541, 540, + /* 160 */ 108, 535, 534, 533, 532, 531, 530, 529, 528, 115, + /* 170 */ 524, 1119, 1120, 63, 1164, 1165, 1166, 1167, 1168, 1169, + /* 180 */ 1170, 466, 1175, 1176, 1177, 1178, 1179, 1180, 1181, 1116, + /* 190 */ 891, 1606, 292, 402, 1414, 396, 416, 1297, 471, 401, + /* 200 */ 122, 138, 102, 1140, 397, 395, 384, 398, 1423, 1213, + /* 210 */ 1119, 1120, 394, 1164, 1165, 1166, 1167, 1168, 1169, 1170, + /* 220 */ 466, 1175, 1176, 1177, 1178, 1179, 1180, 1181, 33, 31, + /* 230 */ 101, 1590, 414, 440, 389, 1718, 282, 250, 1115, 1136, + /* 240 */ 34, 32, 30, 29, 28, 412, 344, 1718, 137, 356, + /* 250 */ 1670, 298, 1715, 138, 1113, 1606, 1299, 392, 357, 1470, + /* 260 */ 1716, 321, 471, 320, 1715, 287, 12, 33, 31, 487, + /* 270 */ 1468, 138, 470, 1121, 1667, 282, 1562, 1115, 327, 138, + /* 280 */ 98, 97, 96, 95, 94, 93, 92, 91, 90, 1412, + /* 290 */ 1718, 1, 1138, 1113, 1421, 74, 1591, 473, 1593, 1594, + /* 300 */ 469, 328, 464, 137, 1251, 1656, 128, 1715, 69, 275, + /* 310 */ 1652, 1730, 1121, 138, 570, 1261, 1470, 1462, 65, 138, + /* 320 */ 1690, 1077, 294, 200, 56, 270, 1114, 1468, 175, 1079, + /* 330 */ 7, 251, 355, 264, 1562, 350, 349, 348, 347, 346, + /* 340 */ 6, 343, 342, 341, 340, 339, 335, 334, 333, 332, + /* 350 */ 331, 330, 329, 570, 434, 1259, 1260, 1262, 1263, 101, + /* 360 */ 892, 441, 891, 389, 89, 1114, 1116, 88, 87, 86, + /* 370 */ 85, 84, 83, 82, 81, 80, 265, 1322, 263, 262, + /* 380 */ 893, 388, 437, 124, 48, 1311, 392, 1119, 1120, 1078, + /* 390 */ 1164, 1165, 1166, 1167, 1168, 1169, 1170, 466, 1175, 1176, + /* 400 */ 1177, 1178, 1179, 1180, 1181, 1116, 1398, 22, 295, 34, + /* 410 */ 32, 30, 29, 28, 1351, 523, 122, 34, 32, 30, + /* 420 */ 29, 28, 1562, 1141, 1423, 228, 1119, 1120, 1451, 1164, + /* 430 */ 1165, 1166, 1167, 1168, 1169, 1170, 466, 1175, 1176, 1177, + /* 440 */ 1178, 1179, 1180, 1181, 33, 31, 1182, 254, 24, 1201, + /* 450 */ 442, 438, 282, 424, 1115, 1590, 138, 945, 34, 32, + /* 460 */ 30, 29, 28, 566, 565, 487, 402, 107, 396, 1206, + /* 470 */ 1113, 518, 401, 1151, 336, 102, 947, 397, 395, 1606, + /* 480 */ 398, 1199, 313, 33, 31, 394, 471, 391, 390, 1121, + /* 490 */ 1421, 282, 1139, 1115, 521, 1321, 470, 1232, 425, 474, + /* 500 */ 1562, 315, 487, 1320, 23, 1515, 451, 7, 1590, 1113, + /* 510 */ 1137, 337, 1319, 517, 516, 515, 487, 514, 1318, 73, + /* 520 */ 1591, 473, 1593, 1594, 469, 364, 464, 1421, 1121, 1656, + /* 530 */ 570, 1200, 1606, 253, 1652, 487, 351, 1718, 70, 471, + /* 540 */ 1562, 1421, 1114, 186, 1418, 1718, 7, 122, 1562, 470, + /* 550 */ 137, 1205, 106, 1562, 1715, 1424, 431, 1562, 137, 1413, + /* 560 */ 1421, 513, 1715, 1562, 34, 32, 30, 29, 28, 570, + /* 570 */ 144, 487, 249, 1591, 473, 1593, 1594, 469, 1317, 464, + /* 580 */ 1543, 1114, 1116, 148, 147, 1316, 25, 280, 1194, 1195, + /* 590 */ 1196, 1197, 1198, 1202, 1203, 1204, 1421, 52, 453, 1315, + /* 600 */ 51, 400, 399, 1119, 1120, 526, 1164, 1165, 1166, 1167, + /* 610 */ 1168, 1169, 1170, 466, 1175, 1176, 1177, 1178, 1179, 1180, + /* 620 */ 1181, 1116, 1314, 1562, 1313, 34, 32, 30, 29, 28, + /* 630 */ 1562, 457, 1310, 34, 32, 30, 29, 28, 1399, 520, + /* 640 */ 519, 1236, 1119, 1120, 1562, 1164, 1165, 1166, 1167, 1168, + /* 650 */ 1169, 1170, 466, 1175, 1176, 1177, 1178, 1179, 1180, 1181, + /* 660 */ 33, 31, 1309, 254, 1670, 1590, 1670, 1562, 282, 1562, + /* 670 */ 1115, 34, 32, 30, 29, 28, 447, 1562, 34, 32, + /* 680 */ 30, 29, 28, 289, 288, 1406, 1113, 365, 1666, 1606, + /* 690 */ 1665, 1397, 1583, 1129, 487, 1292, 450, 1199, 527, 487, + /* 700 */ 1393, 105, 487, 484, 1581, 1121, 470, 1562, 485, 1122, + /* 710 */ 1562, 219, 539, 537, 1506, 1507, 487, 1408, 1590, 1421, + /* 720 */ 1308, 451, 1404, 1, 1421, 296, 1307, 1421, 1121, 74, + /* 730 */ 1591, 473, 1593, 1594, 469, 1151, 464, 178, 103, 1656, + /* 740 */ 523, 1421, 1606, 275, 1652, 132, 570, 1200, 1306, 471, + /* 750 */ 1305, 1101, 1102, 198, 1663, 446, 1304, 445, 1114, 470, + /* 760 */ 1718, 121, 538, 1562, 1684, 1562, 316, 1205, 1303, 488, + /* 770 */ 1302, 1562, 1244, 137, 1291, 1675, 1232, 1715, 1138, 1470, + /* 780 */ 458, 1125, 75, 1591, 473, 1593, 1594, 469, 1187, 464, + /* 790 */ 1469, 454, 1656, 1562, 1138, 1562, 1655, 1652, 1116, 9, + /* 800 */ 8, 1562, 25, 280, 1194, 1195, 1196, 1197, 1198, 1202, + /* 810 */ 1203, 1204, 465, 1562, 1551, 1562, 1590, 40, 252, 1119, + /* 820 */ 1120, 1130, 1164, 1165, 1166, 1167, 1168, 1169, 1170, 466, + /* 830 */ 1175, 1176, 1177, 1178, 1179, 1180, 1181, 1396, 123, 1235, + /* 840 */ 1606, 166, 1133, 234, 164, 1338, 168, 471, 1590, 167, + /* 850 */ 305, 170, 1333, 1115, 169, 232, 172, 470, 113, 171, + /* 860 */ 47, 1562, 426, 512, 1258, 1294, 1295, 403, 149, 1113, + /* 870 */ 1331, 189, 1606, 35, 405, 191, 461, 1207, 1312, 471, + /* 880 */ 74, 1591, 473, 1593, 1594, 469, 423, 464, 1121, 470, + /* 890 */ 1656, 1379, 408, 1562, 275, 1652, 1730, 35, 107, 35, + /* 900 */ 208, 1171, 518, 1072, 210, 1713, 110, 111, 202, 435, + /* 910 */ 479, 216, 74, 1591, 473, 1593, 1594, 469, 1463, 464, + /* 920 */ 113, 1124, 1656, 1123, 976, 521, 275, 1652, 1730, 570, + /* 930 */ 195, 47, 72, 417, 383, 971, 917, 1674, 455, 492, + /* 940 */ 448, 1114, 1686, 1004, 517, 516, 515, 111, 514, 112, + /* 950 */ 447, 1008, 113, 1015, 111, 918, 1014, 1607, 114, 2, + /* 960 */ 204, 1136, 300, 50, 49, 325, 304, 143, 1590, 259, + /* 970 */ 945, 261, 319, 224, 145, 105, 338, 1590, 1085, 1504, + /* 980 */ 345, 1116, 352, 353, 260, 354, 311, 358, 307, 303, + /* 990 */ 140, 1145, 1606, 1127, 359, 1126, 150, 1191, 360, 471, + /* 1000 */ 1144, 1606, 1119, 1120, 361, 153, 362, 1143, 471, 470, + /* 1010 */ 156, 363, 103, 1562, 1142, 53, 366, 159, 470, 451, + /* 1020 */ 385, 138, 1562, 387, 1411, 79, 269, 134, 1663, 1664, + /* 1030 */ 1121, 1668, 241, 1591, 473, 1593, 1594, 469, 1590, 464, + /* 1040 */ 163, 75, 1591, 473, 1593, 1594, 469, 1407, 464, 165, + /* 1050 */ 116, 1656, 117, 1409, 1405, 460, 1652, 1547, 1718, 573, + /* 1060 */ 118, 119, 1606, 428, 447, 418, 1141, 1687, 176, 468, + /* 1070 */ 225, 137, 419, 223, 427, 1715, 99, 179, 181, 470, + /* 1080 */ 422, 184, 562, 1562, 558, 554, 550, 222, 436, 105, + /* 1090 */ 226, 1697, 1590, 187, 477, 433, 444, 190, 1696, 194, + /* 1100 */ 274, 439, 248, 1591, 473, 1593, 1594, 469, 467, 464, + /* 1110 */ 462, 1628, 5, 432, 71, 4, 1606, 217, 1232, 104, + /* 1120 */ 1140, 130, 37, 471, 1590, 197, 103, 196, 1677, 1671, + /* 1130 */ 276, 459, 456, 470, 16, 1513, 475, 1562, 481, 476, + /* 1140 */ 1590, 135, 1663, 1664, 1512, 1668, 480, 483, 1606, 212, + /* 1150 */ 203, 1733, 1714, 284, 1637, 471, 126, 1591, 473, 1593, + /* 1160 */ 1594, 469, 214, 464, 1606, 470, 62, 482, 1422, 1562, + /* 1170 */ 227, 471, 64, 1394, 490, 229, 429, 221, 569, 182, + /* 1180 */ 43, 470, 233, 129, 271, 1562, 235, 231, 244, 1591, + /* 1190 */ 473, 1593, 1594, 469, 1556, 464, 1093, 236, 177, 1590, + /* 1200 */ 452, 1731, 1555, 299, 75, 1591, 473, 1593, 1594, 469, + /* 1210 */ 301, 464, 1590, 1552, 1656, 302, 1109, 1110, 141, 1653, + /* 1220 */ 306, 1550, 308, 1606, 309, 310, 443, 1549, 312, 1548, + /* 1230 */ 471, 314, 1533, 142, 317, 318, 1606, 1088, 1087, 1527, + /* 1240 */ 470, 1526, 323, 471, 1562, 324, 1525, 1524, 1055, 1497, + /* 1250 */ 1496, 1495, 1494, 470, 1493, 1492, 1491, 1562, 109, 1480, + /* 1260 */ 279, 1490, 1489, 126, 1591, 473, 1593, 1594, 469, 1590, + /* 1270 */ 464, 155, 1400, 910, 1349, 1057, 249, 1591, 473, 1593, + /* 1280 */ 1594, 469, 1488, 464, 1487, 1486, 1485, 1484, 1483, 1482, + /* 1290 */ 1481, 1479, 1478, 1606, 1477, 1476, 1475, 1474, 1473, 1472, + /* 1300 */ 468, 1471, 1350, 1541, 1535, 1519, 1510, 1347, 1732, 369, + /* 1310 */ 470, 367, 368, 1345, 1562, 371, 372, 1590, 373, 1343, + /* 1320 */ 1341, 375, 377, 1330, 381, 1329, 1326, 1402, 77, 376, + /* 1330 */ 162, 1401, 536, 248, 1591, 473, 1593, 1594, 469, 538, + /* 1340 */ 464, 1606, 1629, 1020, 380, 1339, 1590, 379, 471, 1023, + /* 1350 */ 944, 266, 943, 942, 941, 940, 937, 1334, 470, 267, + /* 1360 */ 936, 1332, 1562, 406, 268, 281, 409, 1325, 411, 1324, + /* 1370 */ 1606, 413, 76, 1540, 46, 1534, 1095, 471, 120, 180, + /* 1380 */ 420, 249, 1591, 473, 1593, 1594, 469, 470, 464, 1518, + /* 1390 */ 1517, 1562, 161, 183, 283, 131, 1590, 421, 1509, 185, + /* 1400 */ 13, 382, 3, 378, 374, 370, 160, 57, 35, 188, + /* 1410 */ 249, 1591, 473, 1593, 1594, 469, 41, 464, 38, 14, + /* 1420 */ 1606, 11, 1581, 1257, 1250, 1280, 127, 471, 192, 20, + /* 1430 */ 58, 8, 193, 55, 21, 1590, 158, 470, 39, 1229, + /* 1440 */ 1228, 1562, 1279, 199, 1590, 277, 1285, 1284, 15, 1283, + /* 1450 */ 278, 17, 136, 1174, 1173, 463, 27, 1159, 139, 1606, + /* 1460 */ 237, 1591, 473, 1593, 1594, 469, 471, 464, 1606, 1508, + /* 1470 */ 1172, 1192, 10, 206, 18, 471, 470, 19, 478, 207, + /* 1480 */ 1562, 1255, 213, 472, 1131, 470, 65, 1580, 209, 1562, + /* 1490 */ 211, 60, 1590, 61, 157, 218, 152, 42, 154, 243, + /* 1500 */ 1591, 473, 1593, 1594, 469, 1590, 464, 215, 245, 1591, + /* 1510 */ 473, 1593, 1594, 469, 489, 464, 1606, 151, 491, 1005, + /* 1520 */ 291, 493, 495, 471, 1002, 496, 498, 999, 499, 1606, + /* 1530 */ 993, 501, 502, 470, 991, 504, 471, 1562, 505, 982, + /* 1540 */ 997, 996, 66, 511, 995, 994, 470, 1017, 1016, 67, + /* 1550 */ 1562, 68, 1013, 1010, 908, 522, 238, 1591, 473, 1593, + /* 1560 */ 1594, 469, 933, 464, 951, 525, 220, 931, 1590, 246, + /* 1570 */ 1591, 473, 1593, 1594, 469, 930, 464, 1590, 929, 928, + /* 1580 */ 927, 926, 925, 924, 948, 946, 921, 920, 919, 1346, + /* 1590 */ 916, 915, 1606, 914, 913, 547, 548, 549, 1344, 471, + /* 1600 */ 551, 1606, 552, 1342, 556, 555, 553, 557, 471, 470, + /* 1610 */ 1340, 559, 560, 1562, 561, 1328, 1590, 563, 470, 564, + /* 1620 */ 1327, 1323, 1562, 567, 1117, 568, 571, 1590, 230, 572, + /* 1630 */ 1298, 1298, 239, 1591, 473, 1593, 1594, 469, 1298, 464, + /* 1640 */ 1606, 247, 1591, 473, 1593, 1594, 469, 471, 464, 1298, + /* 1650 */ 1298, 1606, 1298, 1298, 1298, 1298, 1298, 470, 471, 1298, + /* 1660 */ 1298, 1562, 1298, 1298, 1298, 1298, 1298, 1298, 470, 1298, + /* 1670 */ 1298, 1298, 1562, 1298, 1298, 1298, 1590, 1298, 1298, 1298, + /* 1680 */ 240, 1591, 473, 1593, 1594, 469, 1298, 464, 1298, 1298, + /* 1690 */ 1298, 1602, 1591, 473, 1593, 1594, 469, 1298, 464, 1298, + /* 1700 */ 1606, 1298, 1298, 1298, 1298, 1298, 1298, 471, 1298, 1298, + /* 1710 */ 1298, 1298, 1298, 1298, 1298, 1590, 1298, 470, 1298, 1298, + /* 1720 */ 1298, 1562, 1298, 1298, 1590, 1298, 1298, 1298, 1298, 1298, + /* 1730 */ 1298, 1298, 1298, 1590, 1298, 1298, 1298, 1298, 1298, 1606, + /* 1740 */ 1601, 1591, 473, 1593, 1594, 469, 471, 464, 1606, 1298, + /* 1750 */ 1298, 1298, 1298, 1298, 1298, 471, 470, 1606, 1298, 1298, + /* 1760 */ 1562, 1298, 1298, 1298, 471, 470, 1298, 1298, 1298, 1562, + /* 1770 */ 1298, 1298, 1590, 1298, 470, 1298, 1298, 1298, 1562, 1600, + /* 1780 */ 1591, 473, 1593, 1594, 469, 1590, 464, 1298, 257, 1591, + /* 1790 */ 473, 1593, 1594, 469, 1298, 464, 1606, 256, 1591, 473, + /* 1800 */ 1593, 1594, 469, 471, 464, 1298, 1298, 1298, 1298, 1606, + /* 1810 */ 1298, 1298, 1298, 470, 1298, 1298, 471, 1562, 1298, 1298, + /* 1820 */ 1298, 1298, 1298, 1298, 1298, 1298, 470, 1298, 1298, 1298, + /* 1830 */ 1562, 1298, 1298, 1298, 1590, 1298, 258, 1591, 473, 1593, + /* 1840 */ 1594, 469, 1298, 464, 1298, 1298, 1298, 1298, 1298, 255, + /* 1850 */ 1591, 473, 1593, 1594, 469, 1298, 464, 1298, 1606, 1298, + /* 1860 */ 1298, 1298, 1298, 1298, 1298, 471, 1298, 1298, 1298, 1298, + /* 1870 */ 1298, 1298, 1298, 1298, 1298, 470, 1298, 1298, 1298, 1562, + /* 1880 */ 1298, 1298, 1298, 1298, 1298, 1298, 1298, 1298, 1298, 1298, + /* 1890 */ 1298, 1298, 1298, 1298, 1298, 1298, 1298, 1298, 242, 1591, + /* 1900 */ 473, 1593, 1594, 469, 1298, 464, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 253, 249, 253, 266, 225, 231, 269, 231, 256, 272, - /* 10 */ 222, 250, 12, 13, 240, 225, 224, 270, 226, 270, - /* 20 */ 20, 247, 22, 12, 13, 14, 15, 16, 249, 270, - /* 30 */ 256, 284, 285, 284, 285, 256, 234, 261, 38, 249, - /* 40 */ 238, 314, 290, 284, 285, 266, 256, 0, 231, 270, - /* 50 */ 50, 12, 13, 14, 327, 233, 266, 57, 331, 20, - /* 60 */ 270, 22, 20, 275, 275, 275, 225, 288, 289, 290, - /* 70 */ 291, 292, 293, 256, 295, 75, 254, 38, 288, 289, - /* 80 */ 290, 291, 292, 293, 237, 295, 239, 258, 298, 50, - /* 90 */ 249, 20, 302, 303, 265, 266, 57, 256, 98, 299, - /* 100 */ 300, 49, 314, 314, 314, 326, 231, 266, 61, 292, - /* 110 */ 110, 270, 65, 231, 75, 327, 327, 327, 20, 331, - /* 120 */ 331, 331, 240, 306, 307, 308, 309, 231, 311, 288, - /* 130 */ 289, 290, 291, 292, 293, 88, 295, 98, 256, 298, - /* 140 */ 233, 47, 231, 302, 303, 304, 75, 38, 148, 110, - /* 150 */ 275, 240, 256, 246, 107, 108, 109, 316, 111, 20, - /* 160 */ 20, 254, 20, 322, 323, 138, 57, 256, 74, 169, - /* 170 */ 170, 77, 172, 173, 174, 175, 176, 177, 178, 179, - /* 180 */ 180, 181, 182, 183, 184, 185, 186, 148, 292, 314, - /* 190 */ 249, 12, 13, 14, 15, 16, 255, 1, 2, 199, - /* 200 */ 75, 260, 327, 307, 308, 309, 331, 311, 169, 170, - /* 210 */ 85, 172, 173, 174, 175, 176, 177, 178, 179, 180, - /* 220 */ 181, 182, 183, 184, 185, 186, 12, 13, 199, 50, - /* 230 */ 242, 204, 205, 245, 20, 18, 22, 20, 225, 52, - /* 240 */ 230, 54, 49, 249, 27, 58, 230, 30, 61, 255, - /* 250 */ 63, 64, 38, 66, 260, 76, 39, 147, 71, 149, - /* 260 */ 244, 251, 249, 84, 50, 12, 13, 251, 4, 256, - /* 270 */ 199, 57, 76, 20, 2, 22, 286, 157, 158, 266, - /* 280 */ 169, 161, 2, 270, 12, 13, 14, 15, 16, 75, - /* 290 */ 286, 38, 12, 13, 14, 15, 16, 14, 15, 16, - /* 300 */ 310, 288, 289, 290, 291, 292, 293, 248, 295, 199, - /* 310 */ 57, 298, 98, 134, 310, 302, 303, 304, 259, 208, - /* 320 */ 209, 210, 211, 212, 110, 275, 313, 0, 75, 0, - /* 330 */ 113, 152, 20, 116, 117, 118, 119, 120, 199, 122, - /* 340 */ 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, - /* 350 */ 133, 98, 4, 24, 25, 26, 27, 28, 29, 30, - /* 360 */ 31, 32, 148, 110, 314, 314, 187, 188, 189, 190, - /* 370 */ 191, 192, 193, 194, 195, 196, 49, 327, 327, 225, - /* 380 */ 20, 331, 331, 169, 170, 43, 172, 173, 174, 175, - /* 390 */ 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, - /* 400 */ 186, 148, 57, 20, 72, 12, 13, 14, 15, 16, - /* 410 */ 21, 228, 229, 24, 25, 26, 27, 28, 29, 30, - /* 420 */ 31, 32, 169, 170, 270, 172, 173, 174, 175, 176, - /* 430 */ 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, - /* 440 */ 12, 13, 14, 50, 0, 72, 225, 241, 20, 76, - /* 450 */ 22, 241, 199, 231, 266, 249, 35, 61, 75, 249, - /* 460 */ 272, 65, 240, 257, 200, 21, 38, 257, 24, 25, - /* 470 */ 26, 27, 28, 29, 30, 31, 32, 84, 256, 12, - /* 480 */ 13, 3, 61, 0, 88, 57, 65, 20, 225, 22, - /* 490 */ 225, 270, 12, 13, 14, 15, 16, 231, 231, 78, - /* 500 */ 140, 80, 81, 75, 83, 38, 240, 240, 86, 88, - /* 510 */ 235, 236, 241, 247, 249, 12, 13, 14, 15, 16, - /* 520 */ 249, 256, 256, 256, 57, 231, 98, 134, 257, 225, - /* 530 */ 249, 266, 256, 270, 240, 270, 255, 231, 110, 263, - /* 540 */ 198, 260, 75, 134, 61, 152, 240, 199, 65, 217, - /* 550 */ 256, 235, 236, 288, 289, 290, 291, 292, 293, 249, - /* 560 */ 295, 152, 256, 298, 0, 98, 20, 302, 303, 304, - /* 570 */ 260, 88, 67, 250, 270, 286, 148, 110, 313, 76, - /* 580 */ 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, - /* 590 */ 107, 108, 109, 225, 111, 0, 187, 169, 170, 310, - /* 600 */ 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, - /* 610 */ 182, 183, 184, 185, 186, 148, 52, 249, 54, 114, - /* 620 */ 115, 141, 58, 225, 256, 61, 79, 63, 64, 82, - /* 630 */ 66, 36, 235, 236, 266, 71, 169, 170, 270, 172, - /* 640 */ 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, - /* 650 */ 183, 184, 185, 186, 12, 13, 288, 289, 290, 291, - /* 660 */ 292, 293, 20, 295, 22, 225, 298, 0, 270, 231, - /* 670 */ 302, 303, 304, 12, 13, 14, 15, 16, 240, 231, - /* 680 */ 38, 225, 249, 231, 231, 20, 140, 22, 240, 249, - /* 690 */ 257, 323, 240, 215, 256, 250, 256, 235, 236, 57, - /* 700 */ 220, 14, 231, 22, 256, 40, 266, 20, 256, 256, - /* 710 */ 270, 240, 231, 225, 225, 144, 49, 75, 0, 38, - /* 720 */ 225, 240, 225, 225, 225, 250, 270, 256, 288, 289, - /* 730 */ 290, 291, 292, 293, 163, 295, 79, 256, 298, 82, - /* 740 */ 98, 72, 302, 303, 304, 292, 12, 13, 14, 15, - /* 750 */ 16, 225, 110, 313, 225, 225, 250, 225, 270, 270, - /* 760 */ 307, 308, 309, 225, 311, 270, 50, 270, 270, 270, - /* 770 */ 52, 53, 54, 55, 56, 250, 58, 59, 60, 61, - /* 780 */ 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, - /* 790 */ 148, 14, 265, 266, 197, 198, 270, 20, 250, 270, - /* 800 */ 270, 140, 270, 225, 159, 160, 225, 225, 270, 72, - /* 810 */ 76, 169, 170, 76, 172, 173, 174, 175, 176, 177, - /* 820 */ 178, 179, 180, 181, 182, 183, 184, 185, 186, 75, - /* 830 */ 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, - /* 840 */ 99, 87, 101, 102, 103, 104, 105, 106, 270, 18, - /* 850 */ 225, 270, 270, 79, 23, 0, 82, 79, 0, 21, - /* 860 */ 82, 0, 38, 225, 72, 72, 35, 278, 76, 76, - /* 870 */ 1, 2, 34, 38, 249, 184, 185, 22, 72, 48, - /* 880 */ 22, 256, 76, 22, 226, 169, 72, 249, 219, 72, - /* 890 */ 76, 266, 72, 76, 256, 270, 76, 72, 238, 72, - /* 900 */ 275, 76, 72, 76, 266, 72, 76, 334, 270, 76, - /* 910 */ 325, 259, 282, 288, 289, 290, 291, 292, 293, 72, - /* 920 */ 295, 72, 319, 76, 228, 76, 288, 289, 290, 291, - /* 930 */ 292, 293, 312, 295, 110, 287, 298, 249, 72, 314, - /* 940 */ 302, 303, 76, 112, 225, 110, 328, 38, 315, 36, - /* 950 */ 33, 72, 327, 36, 72, 76, 331, 20, 76, 42, - /* 960 */ 231, 44, 45, 46, 47, 283, 57, 235, 249, 38, - /* 970 */ 276, 231, 141, 142, 143, 256, 145, 231, 146, 264, - /* 980 */ 121, 150, 134, 262, 262, 266, 231, 20, 280, 270, - /* 990 */ 233, 74, 266, 162, 77, 164, 225, 166, 167, 168, - /* 1000 */ 20, 274, 233, 256, 20, 267, 233, 288, 289, 290, - /* 1010 */ 291, 292, 293, 233, 295, 231, 233, 298, 20, 19, - /* 1020 */ 249, 302, 303, 227, 249, 249, 231, 256, 249, 227, - /* 1030 */ 199, 249, 57, 33, 249, 249, 36, 266, 249, 249, - /* 1040 */ 249, 270, 42, 249, 44, 45, 46, 47, 249, 230, - /* 1050 */ 280, 225, 135, 155, 137, 274, 139, 270, 230, 288, - /* 1060 */ 289, 290, 291, 292, 293, 294, 295, 296, 297, 266, - /* 1070 */ 230, 256, 267, 156, 74, 249, 279, 77, 20, 207, - /* 1080 */ 230, 271, 256, 324, 206, 287, 270, 324, 214, 271, - /* 1090 */ 213, 270, 266, 270, 202, 201, 270, 198, 256, 20, - /* 1100 */ 225, 221, 121, 335, 218, 216, 305, 107, 286, 321, - /* 1110 */ 75, 270, 301, 270, 288, 289, 290, 291, 292, 293, - /* 1120 */ 320, 295, 318, 271, 249, 317, 137, 271, 245, 256, - /* 1130 */ 270, 256, 268, 230, 267, 230, 136, 330, 329, 139, - /* 1140 */ 256, 266, 75, 252, 230, 270, 239, 231, 227, 281, - /* 1150 */ 232, 243, 225, 277, 154, 243, 156, 223, 332, 333, - /* 1160 */ 231, 0, 0, 288, 289, 290, 291, 292, 293, 64, - /* 1170 */ 295, 0, 38, 298, 165, 38, 249, 38, 303, 38, - /* 1180 */ 165, 0, 225, 256, 38, 256, 38, 165, 0, 38, - /* 1190 */ 0, 0, 38, 266, 75, 152, 151, 270, 110, 148, - /* 1200 */ 273, 0, 0, 53, 275, 144, 249, 0, 0, 87, - /* 1210 */ 0, 0, 0, 256, 121, 288, 289, 290, 291, 292, - /* 1220 */ 293, 292, 295, 266, 0, 0, 0, 270, 0, 0, - /* 1230 */ 0, 0, 0, 225, 0, 0, 307, 308, 309, 0, - /* 1240 */ 311, 0, 0, 314, 0, 288, 289, 290, 291, 292, - /* 1250 */ 293, 0, 295, 0, 0, 0, 327, 249, 0, 0, - /* 1260 */ 331, 0, 22, 225, 256, 0, 0, 0, 0, 0, - /* 1270 */ 0, 0, 0, 0, 266, 82, 43, 0, 270, 0, - /* 1280 */ 0, 273, 36, 38, 36, 38, 43, 249, 51, 0, - /* 1290 */ 333, 225, 43, 43, 256, 38, 288, 289, 290, 291, - /* 1300 */ 292, 293, 0, 295, 266, 43, 0, 0, 270, 0, - /* 1310 */ 36, 0, 22, 0, 38, 249, 38, 36, 0, 22, - /* 1320 */ 84, 38, 256, 38, 72, 38, 288, 289, 290, 291, - /* 1330 */ 292, 293, 266, 295, 38, 297, 270, 38, 38, 273, - /* 1340 */ 225, 38, 12, 13, 0, 0, 22, 39, 22, 22, - /* 1350 */ 38, 0, 22, 22, 288, 289, 290, 291, 292, 293, - /* 1360 */ 72, 295, 0, 22, 249, 38, 20, 0, 38, 0, - /* 1370 */ 38, 256, 153, 0, 22, 137, 0, 0, 225, 75, - /* 1380 */ 43, 266, 72, 203, 57, 270, 72, 57, 273, 225, - /* 1390 */ 203, 76, 72, 76, 75, 197, 140, 75, 135, 203, - /* 1400 */ 140, 76, 249, 288, 289, 290, 291, 292, 293, 256, - /* 1410 */ 295, 76, 72, 249, 75, 87, 87, 75, 38, 266, - /* 1420 */ 256, 72, 140, 270, 76, 98, 72, 72, 98, 76, - /* 1430 */ 266, 38, 38, 38, 270, 38, 38, 110, 2, 225, - /* 1440 */ 110, 288, 289, 290, 291, 292, 293, 87, 295, 75, - /* 1450 */ 72, 76, 288, 289, 290, 291, 292, 293, 76, 295, - /* 1460 */ 169, 75, 75, 249, 76, 75, 87, 22, 87, 171, - /* 1470 */ 256, 75, 138, 0, 75, 148, 76, 76, 148, 75, - /* 1480 */ 266, 75, 43, 75, 270, 135, 75, 85, 22, 225, - /* 1490 */ 87, 76, 38, 38, 87, 86, 169, 170, 75, 169, - /* 1500 */ 225, 76, 288, 289, 290, 291, 292, 293, 38, 295, - /* 1510 */ 75, 22, 38, 249, 76, 75, 38, 76, 75, 38, - /* 1520 */ 256, 100, 76, 75, 249, 88, 100, 100, 100, 38, - /* 1530 */ 266, 256, 75, 75, 270, 75, 38, 22, 51, 110, - /* 1540 */ 50, 266, 38, 57, 73, 270, 38, 72, 225, 38, - /* 1550 */ 38, 38, 288, 289, 290, 291, 292, 293, 38, 295, - /* 1560 */ 38, 225, 4, 288, 289, 290, 291, 292, 293, 38, - /* 1570 */ 295, 22, 249, 57, 38, 38, 38, 19, 38, 256, - /* 1580 */ 38, 38, 38, 38, 0, 249, 38, 36, 0, 266, - /* 1590 */ 43, 33, 256, 270, 36, 38, 36, 43, 0, 41, - /* 1600 */ 38, 36, 266, 0, 43, 47, 270, 38, 36, 43, - /* 1610 */ 0, 288, 289, 290, 291, 292, 293, 38, 295, 37, - /* 1620 */ 0, 0, 22, 21, 288, 289, 290, 291, 292, 293, - /* 1630 */ 225, 295, 74, 22, 22, 77, 21, 20, 336, 225, - /* 1640 */ 336, 336, 336, 336, 336, 336, 336, 336, 336, 336, - /* 1650 */ 336, 336, 336, 336, 249, 336, 336, 336, 336, 336, - /* 1660 */ 336, 256, 336, 249, 336, 336, 336, 336, 336, 336, - /* 1670 */ 256, 266, 336, 336, 336, 270, 336, 336, 336, 336, - /* 1680 */ 266, 336, 336, 336, 270, 336, 336, 336, 336, 225, - /* 1690 */ 336, 336, 336, 288, 289, 290, 291, 292, 293, 336, - /* 1700 */ 295, 336, 288, 289, 290, 291, 292, 293, 336, 295, - /* 1710 */ 336, 336, 336, 249, 336, 336, 336, 336, 336, 336, - /* 1720 */ 256, 336, 336, 336, 336, 336, 336, 336, 336, 336, - /* 1730 */ 266, 336, 336, 336, 270, 336, 336, 336, 336, 225, - /* 1740 */ 336, 336, 336, 336, 336, 336, 336, 336, 336, 336, - /* 1750 */ 225, 336, 288, 289, 290, 291, 292, 293, 336, 295, - /* 1760 */ 336, 336, 336, 249, 336, 336, 336, 336, 336, 336, - /* 1770 */ 256, 336, 336, 336, 249, 336, 336, 336, 336, 336, - /* 1780 */ 266, 256, 336, 336, 270, 336, 336, 336, 336, 336, - /* 1790 */ 336, 266, 336, 336, 336, 270, 336, 336, 225, 336, - /* 1800 */ 336, 336, 288, 289, 290, 291, 292, 293, 336, 295, - /* 1810 */ 336, 225, 336, 288, 289, 290, 291, 292, 293, 336, - /* 1820 */ 295, 336, 249, 336, 336, 336, 336, 336, 336, 256, - /* 1830 */ 336, 336, 336, 336, 336, 249, 336, 336, 336, 266, - /* 1840 */ 336, 336, 256, 270, 336, 336, 336, 336, 336, 336, - /* 1850 */ 336, 336, 266, 336, 336, 336, 270, 336, 336, 336, - /* 1860 */ 336, 288, 289, 290, 291, 292, 293, 336, 295, 336, - /* 1870 */ 336, 336, 336, 336, 288, 289, 290, 291, 292, 293, - /* 1880 */ 225, 295, 336, 336, 336, 336, 336, 336, 336, 225, - /* 1890 */ 336, 336, 336, 336, 336, 336, 336, 336, 336, 336, - /* 1900 */ 336, 336, 336, 336, 249, 336, 336, 336, 336, 336, - /* 1910 */ 336, 256, 336, 249, 336, 336, 336, 336, 336, 336, - /* 1920 */ 256, 266, 336, 336, 336, 270, 336, 336, 336, 336, - /* 1930 */ 266, 336, 336, 336, 270, 336, 336, 336, 336, 336, - /* 1940 */ 336, 336, 336, 288, 289, 290, 291, 292, 293, 336, - /* 1950 */ 295, 336, 288, 289, 290, 291, 292, 293, 336, 295, + /* 0 */ 240, 324, 234, 265, 262, 275, 285, 262, 278, 249, + /* 10 */ 272, 281, 12, 13, 337, 259, 256, 4, 341, 240, + /* 20 */ 20, 279, 22, 258, 279, 265, 258, 243, 249, 264, + /* 30 */ 240, 247, 19, 265, 269, 256, 294, 295, 38, 294, + /* 40 */ 295, 309, 310, 275, 265, 324, 33, 279, 279, 36, + /* 50 */ 50, 12, 13, 14, 41, 265, 20, 57, 337, 20, + /* 60 */ 47, 22, 341, 294, 295, 242, 298, 299, 300, 301, + /* 70 */ 302, 303, 20, 305, 4, 75, 308, 38, 255, 240, + /* 80 */ 312, 313, 314, 14, 15, 16, 263, 74, 249, 50, + /* 90 */ 77, 242, 302, 20, 326, 267, 57, 250, 98, 0, + /* 100 */ 332, 333, 274, 275, 265, 258, 316, 317, 318, 319, + /* 110 */ 110, 321, 263, 266, 75, 0, 89, 90, 91, 92, + /* 120 */ 93, 94, 95, 96, 97, 98, 99, 75, 101, 102, + /* 130 */ 103, 104, 105, 106, 1, 2, 21, 98, 20, 24, + /* 140 */ 25, 26, 27, 28, 29, 30, 31, 32, 75, 110, + /* 150 */ 150, 52, 53, 54, 55, 56, 49, 58, 59, 60, + /* 160 */ 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, + /* 170 */ 71, 171, 172, 239, 174, 175, 176, 177, 178, 179, + /* 180 */ 180, 181, 182, 183, 184, 185, 186, 187, 188, 150, + /* 190 */ 22, 258, 250, 52, 260, 54, 285, 231, 265, 58, + /* 200 */ 258, 201, 61, 20, 63, 64, 38, 66, 266, 76, + /* 210 */ 171, 172, 71, 174, 175, 176, 177, 178, 179, 180, + /* 220 */ 181, 182, 183, 184, 185, 186, 187, 188, 12, 13, + /* 230 */ 61, 234, 21, 300, 65, 324, 20, 18, 22, 20, + /* 240 */ 12, 13, 14, 15, 16, 34, 27, 324, 337, 30, + /* 250 */ 296, 285, 341, 201, 38, 258, 0, 88, 39, 258, + /* 260 */ 337, 149, 265, 151, 341, 264, 50, 12, 13, 240, + /* 270 */ 269, 201, 275, 57, 320, 20, 279, 22, 249, 201, + /* 280 */ 24, 25, 26, 27, 28, 29, 30, 31, 32, 234, + /* 290 */ 324, 75, 20, 38, 265, 298, 299, 300, 301, 302, + /* 300 */ 303, 240, 305, 337, 76, 308, 257, 341, 75, 312, + /* 310 */ 313, 314, 57, 201, 98, 171, 258, 268, 85, 201, + /* 320 */ 323, 74, 264, 140, 159, 160, 110, 269, 163, 82, + /* 330 */ 75, 270, 113, 35, 279, 116, 117, 118, 119, 120, + /* 340 */ 43, 122, 123, 124, 125, 126, 127, 128, 129, 130, + /* 350 */ 131, 132, 133, 98, 210, 211, 212, 213, 214, 61, + /* 360 */ 20, 20, 22, 65, 21, 110, 150, 24, 25, 26, + /* 370 */ 27, 28, 29, 30, 31, 32, 78, 234, 80, 81, + /* 380 */ 40, 83, 138, 233, 3, 235, 88, 171, 172, 142, + /* 390 */ 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, + /* 400 */ 184, 185, 186, 187, 188, 150, 0, 2, 250, 12, + /* 410 */ 13, 14, 15, 16, 0, 49, 258, 12, 13, 14, + /* 420 */ 15, 16, 279, 20, 266, 251, 171, 172, 254, 174, + /* 430 */ 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, + /* 440 */ 185, 186, 187, 188, 12, 13, 14, 50, 2, 134, + /* 450 */ 206, 207, 20, 240, 22, 234, 201, 38, 12, 13, + /* 460 */ 14, 15, 16, 237, 238, 240, 52, 61, 54, 154, + /* 470 */ 38, 65, 58, 76, 249, 61, 57, 63, 64, 258, + /* 480 */ 66, 84, 146, 12, 13, 71, 265, 244, 245, 57, + /* 490 */ 265, 20, 20, 22, 88, 234, 275, 200, 285, 275, + /* 500 */ 279, 165, 240, 234, 189, 281, 285, 75, 234, 38, + /* 510 */ 20, 249, 234, 107, 108, 109, 240, 111, 234, 298, + /* 520 */ 299, 300, 301, 302, 303, 249, 305, 265, 57, 308, + /* 530 */ 98, 134, 258, 312, 313, 240, 67, 324, 239, 265, + /* 540 */ 279, 265, 110, 140, 249, 324, 75, 258, 279, 275, + /* 550 */ 337, 154, 253, 279, 341, 266, 282, 279, 337, 260, + /* 560 */ 265, 86, 341, 279, 12, 13, 14, 15, 16, 98, + /* 570 */ 47, 240, 298, 299, 300, 301, 302, 303, 234, 305, + /* 580 */ 249, 110, 150, 114, 115, 234, 189, 190, 191, 192, + /* 590 */ 193, 194, 195, 196, 197, 198, 265, 74, 217, 234, + /* 600 */ 77, 244, 245, 171, 172, 57, 174, 175, 176, 177, + /* 610 */ 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, + /* 620 */ 188, 150, 234, 279, 234, 12, 13, 14, 15, 16, + /* 630 */ 279, 72, 234, 12, 13, 14, 15, 16, 0, 244, + /* 640 */ 245, 4, 171, 172, 279, 174, 175, 176, 177, 178, + /* 650 */ 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, + /* 660 */ 12, 13, 234, 50, 296, 234, 296, 279, 20, 279, + /* 670 */ 22, 12, 13, 14, 15, 16, 240, 279, 12, 13, + /* 680 */ 14, 15, 16, 12, 13, 259, 38, 49, 320, 258, + /* 690 */ 320, 0, 75, 22, 240, 143, 265, 84, 246, 240, + /* 700 */ 248, 265, 240, 249, 87, 57, 275, 279, 249, 38, + /* 710 */ 279, 249, 244, 245, 274, 275, 240, 259, 234, 265, + /* 720 */ 234, 285, 259, 75, 265, 249, 234, 265, 57, 298, + /* 730 */ 299, 300, 301, 302, 303, 76, 305, 259, 302, 308, + /* 740 */ 49, 265, 258, 312, 313, 314, 98, 134, 234, 265, + /* 750 */ 234, 161, 162, 317, 318, 319, 234, 321, 110, 275, + /* 760 */ 324, 140, 72, 279, 333, 279, 76, 154, 234, 98, + /* 770 */ 234, 279, 14, 337, 222, 199, 200, 341, 20, 258, + /* 780 */ 221, 110, 298, 299, 300, 301, 302, 303, 14, 305, + /* 790 */ 269, 72, 308, 279, 20, 279, 312, 313, 150, 1, + /* 800 */ 2, 279, 189, 190, 191, 192, 193, 194, 195, 196, + /* 810 */ 197, 198, 259, 279, 0, 279, 234, 140, 141, 171, + /* 820 */ 172, 150, 174, 175, 176, 177, 178, 179, 180, 181, + /* 830 */ 182, 183, 184, 185, 186, 187, 188, 0, 18, 202, + /* 840 */ 258, 79, 171, 23, 82, 0, 79, 265, 234, 82, + /* 850 */ 36, 79, 0, 22, 82, 35, 79, 275, 72, 82, + /* 860 */ 72, 279, 76, 259, 76, 186, 187, 22, 48, 38, + /* 870 */ 0, 72, 258, 72, 22, 76, 50, 76, 235, 265, + /* 880 */ 298, 299, 300, 301, 302, 303, 288, 305, 57, 275, + /* 890 */ 308, 247, 22, 279, 312, 313, 314, 72, 61, 72, + /* 900 */ 72, 76, 65, 76, 76, 323, 72, 72, 344, 335, + /* 910 */ 76, 76, 298, 299, 300, 301, 302, 303, 268, 305, + /* 920 */ 72, 38, 308, 38, 76, 88, 312, 313, 314, 98, + /* 930 */ 329, 72, 112, 292, 237, 76, 38, 323, 219, 72, + /* 940 */ 322, 110, 297, 76, 107, 108, 109, 72, 111, 72, + /* 950 */ 240, 76, 72, 76, 72, 57, 76, 258, 76, 325, + /* 960 */ 338, 20, 240, 143, 144, 145, 36, 147, 234, 293, + /* 970 */ 38, 244, 152, 286, 121, 265, 240, 234, 148, 240, + /* 980 */ 273, 150, 271, 134, 164, 271, 166, 240, 168, 169, + /* 990 */ 170, 20, 258, 110, 290, 110, 242, 171, 275, 265, + /* 1000 */ 20, 258, 171, 172, 283, 242, 265, 20, 265, 275, + /* 1010 */ 242, 276, 302, 279, 20, 242, 240, 242, 275, 285, + /* 1020 */ 236, 201, 279, 258, 258, 240, 236, 317, 318, 319, + /* 1030 */ 57, 321, 298, 299, 300, 301, 302, 303, 234, 305, + /* 1040 */ 258, 298, 299, 300, 301, 302, 303, 258, 305, 258, + /* 1050 */ 258, 308, 258, 258, 258, 312, 313, 279, 324, 19, + /* 1060 */ 258, 258, 258, 276, 240, 157, 20, 297, 239, 265, + /* 1070 */ 290, 337, 289, 33, 265, 341, 36, 239, 239, 275, + /* 1080 */ 275, 239, 42, 279, 44, 45, 46, 47, 209, 265, + /* 1090 */ 283, 334, 234, 280, 208, 279, 215, 280, 334, 330, + /* 1100 */ 279, 279, 298, 299, 300, 301, 302, 303, 304, 305, + /* 1110 */ 306, 307, 216, 204, 74, 203, 258, 77, 200, 265, + /* 1120 */ 20, 328, 121, 265, 234, 315, 302, 327, 331, 296, + /* 1130 */ 223, 220, 218, 275, 75, 280, 279, 279, 277, 279, + /* 1140 */ 234, 317, 318, 319, 280, 321, 137, 107, 258, 265, + /* 1150 */ 339, 345, 340, 279, 311, 265, 298, 299, 300, 301, + /* 1160 */ 302, 303, 239, 305, 258, 275, 239, 276, 265, 279, + /* 1170 */ 254, 265, 75, 248, 261, 240, 136, 239, 236, 139, + /* 1180 */ 287, 275, 232, 291, 284, 279, 252, 241, 298, 299, + /* 1190 */ 300, 301, 302, 303, 0, 305, 156, 252, 158, 234, + /* 1200 */ 342, 343, 0, 64, 298, 299, 300, 301, 302, 303, + /* 1210 */ 38, 305, 234, 0, 308, 167, 38, 38, 38, 313, + /* 1220 */ 167, 0, 38, 258, 38, 167, 336, 0, 38, 0, + /* 1230 */ 265, 38, 0, 75, 154, 153, 258, 110, 150, 0, + /* 1240 */ 275, 0, 53, 265, 279, 146, 0, 0, 87, 0, + /* 1250 */ 0, 0, 0, 275, 0, 0, 0, 279, 121, 0, + /* 1260 */ 282, 0, 0, 298, 299, 300, 301, 302, 303, 234, + /* 1270 */ 305, 43, 0, 51, 0, 22, 298, 299, 300, 301, + /* 1280 */ 302, 303, 0, 305, 0, 0, 0, 0, 0, 0, + /* 1290 */ 0, 0, 0, 258, 0, 0, 0, 0, 0, 0, + /* 1300 */ 265, 0, 0, 0, 0, 0, 0, 0, 343, 43, + /* 1310 */ 275, 38, 36, 0, 279, 38, 36, 234, 43, 0, + /* 1320 */ 0, 38, 43, 0, 43, 0, 0, 0, 84, 36, + /* 1330 */ 82, 0, 72, 298, 299, 300, 301, 302, 303, 72, + /* 1340 */ 305, 258, 307, 22, 36, 0, 234, 38, 265, 38, + /* 1350 */ 38, 22, 38, 38, 38, 38, 38, 0, 275, 22, + /* 1360 */ 38, 0, 279, 39, 22, 282, 38, 0, 22, 0, + /* 1370 */ 258, 22, 20, 0, 140, 0, 38, 265, 155, 137, + /* 1380 */ 22, 298, 299, 300, 301, 302, 303, 275, 305, 0, + /* 1390 */ 0, 279, 33, 43, 282, 36, 234, 140, 0, 135, + /* 1400 */ 205, 42, 72, 44, 45, 46, 47, 75, 72, 76, + /* 1410 */ 298, 299, 300, 301, 302, 303, 72, 305, 199, 205, + /* 1420 */ 258, 205, 87, 76, 76, 38, 75, 265, 75, 75, + /* 1430 */ 75, 2, 72, 74, 72, 234, 77, 275, 72, 76, + /* 1440 */ 76, 279, 38, 87, 234, 38, 76, 38, 72, 38, + /* 1450 */ 38, 72, 87, 76, 76, 75, 75, 22, 87, 258, + /* 1460 */ 298, 299, 300, 301, 302, 303, 265, 305, 258, 0, + /* 1470 */ 76, 171, 75, 87, 75, 265, 275, 75, 138, 76, + /* 1480 */ 279, 76, 43, 173, 22, 275, 85, 87, 75, 279, + /* 1490 */ 75, 75, 234, 75, 135, 87, 137, 75, 139, 298, + /* 1500 */ 299, 300, 301, 302, 303, 234, 305, 135, 298, 299, + /* 1510 */ 300, 301, 302, 303, 86, 305, 258, 158, 38, 76, + /* 1520 */ 38, 75, 38, 265, 76, 75, 38, 76, 75, 258, + /* 1530 */ 76, 38, 75, 275, 76, 38, 265, 279, 75, 22, + /* 1540 */ 100, 100, 75, 88, 100, 100, 275, 38, 110, 75, + /* 1550 */ 279, 75, 38, 22, 51, 50, 298, 299, 300, 301, + /* 1560 */ 302, 303, 38, 305, 57, 73, 72, 38, 234, 298, + /* 1570 */ 299, 300, 301, 302, 303, 38, 305, 234, 38, 38, + /* 1580 */ 38, 38, 38, 22, 57, 38, 38, 38, 38, 0, + /* 1590 */ 38, 38, 258, 38, 38, 38, 36, 43, 0, 265, + /* 1600 */ 38, 258, 36, 0, 36, 38, 43, 43, 265, 275, + /* 1610 */ 0, 38, 36, 279, 43, 0, 234, 38, 275, 37, + /* 1620 */ 0, 0, 279, 22, 22, 21, 21, 234, 22, 20, + /* 1630 */ 346, 346, 298, 299, 300, 301, 302, 303, 346, 305, + /* 1640 */ 258, 298, 299, 300, 301, 302, 303, 265, 305, 346, + /* 1650 */ 346, 258, 346, 346, 346, 346, 346, 275, 265, 346, + /* 1660 */ 346, 279, 346, 346, 346, 346, 346, 346, 275, 346, + /* 1670 */ 346, 346, 279, 346, 346, 346, 234, 346, 346, 346, + /* 1680 */ 298, 299, 300, 301, 302, 303, 346, 305, 346, 346, + /* 1690 */ 346, 298, 299, 300, 301, 302, 303, 346, 305, 346, + /* 1700 */ 258, 346, 346, 346, 346, 346, 346, 265, 346, 346, + /* 1710 */ 346, 346, 346, 346, 346, 234, 346, 275, 346, 346, + /* 1720 */ 346, 279, 346, 346, 234, 346, 346, 346, 346, 346, + /* 1730 */ 346, 346, 346, 234, 346, 346, 346, 346, 346, 258, + /* 1740 */ 298, 299, 300, 301, 302, 303, 265, 305, 258, 346, + /* 1750 */ 346, 346, 346, 346, 346, 265, 275, 258, 346, 346, + /* 1760 */ 279, 346, 346, 346, 265, 275, 346, 346, 346, 279, + /* 1770 */ 346, 346, 234, 346, 275, 346, 346, 346, 279, 298, + /* 1780 */ 299, 300, 301, 302, 303, 234, 305, 346, 298, 299, + /* 1790 */ 300, 301, 302, 303, 346, 305, 258, 298, 299, 300, + /* 1800 */ 301, 302, 303, 265, 305, 346, 346, 346, 346, 258, + /* 1810 */ 346, 346, 346, 275, 346, 346, 265, 279, 346, 346, + /* 1820 */ 346, 346, 346, 346, 346, 346, 275, 346, 346, 346, + /* 1830 */ 279, 346, 346, 346, 234, 346, 298, 299, 300, 301, + /* 1840 */ 302, 303, 346, 305, 346, 346, 346, 346, 346, 298, + /* 1850 */ 299, 300, 301, 302, 303, 346, 305, 346, 258, 346, + /* 1860 */ 346, 346, 346, 346, 346, 265, 346, 346, 346, 346, + /* 1870 */ 346, 346, 346, 346, 346, 275, 346, 346, 346, 279, + /* 1880 */ 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, + /* 1890 */ 346, 346, 346, 346, 346, 346, 346, 346, 298, 299, + /* 1900 */ 300, 301, 302, 303, 346, 305, }; -#define YY_SHIFT_COUNT (571) +#define YY_SHIFT_COUNT (573) #define YY_SHIFT_MIN (0) #define YY_SHIFT_MAX (1621) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 831, 0, 39, 214, 214, 214, 214, 253, 214, 214, - /* 10 */ 428, 467, 642, 467, 467, 467, 467, 467, 467, 467, - /* 20 */ 467, 467, 467, 467, 467, 467, 467, 467, 467, 467, - /* 30 */ 467, 467, 467, 467, 467, 467, 71, 383, 383, 383, - /* 40 */ 139, 1330, 1330, 110, 42, 42, 29, 1330, 348, 42, - /* 50 */ 42, 42, 42, 42, 42, 52, 42, 98, 140, 29, - /* 60 */ 142, 98, 42, 42, 98, 42, 98, 142, 98, 98, - /* 70 */ 42, 193, 217, 179, 393, 393, 389, 1327, 421, 187, - /* 80 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 90 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 665, - /* 100 */ 327, 109, 109, 360, 360, 360, 667, 109, 109, 312, - /* 110 */ 142, 98, 142, 98, 422, 345, 741, 741, 741, 741, - /* 120 */ 741, 741, 741, 1000, 444, 564, 480, 111, 396, 120, - /* 130 */ 27, 681, 546, 597, 342, 597, 687, 478, 264, 777, - /* 140 */ 937, 913, 931, 832, 937, 937, 859, 848, 848, 937, - /* 150 */ 967, 52, 142, 980, 52, 312, 984, 52, 52, 937, - /* 160 */ 52, 998, 98, 98, 98, 98, 98, 98, 98, 98, - /* 170 */ 98, 98, 98, 937, 998, 975, 967, 193, 898, 142, - /* 180 */ 193, 980, 193, 312, 984, 193, 1058, 872, 878, 975, - /* 190 */ 872, 878, 975, 975, 874, 877, 892, 894, 899, 312, - /* 200 */ 1079, 981, 880, 886, 889, 1035, 98, 878, 975, 975, - /* 210 */ 878, 975, 989, 312, 984, 193, 422, 193, 312, 1067, - /* 220 */ 345, 937, 193, 998, 1960, 1960, 1960, 1960, 1960, 1960, - /* 230 */ 718, 917, 329, 1558, 47, 483, 503, 272, 280, 661, - /* 240 */ 734, 11, 11, 11, 11, 11, 11, 11, 11, 94, - /* 250 */ 505, 196, 409, 283, 283, 283, 283, 595, 571, 373, - /* 260 */ 547, 657, 774, 778, 855, 858, 861, 838, 645, 737, - /* 270 */ 792, 793, 869, 691, 332, 669, 806, 716, 814, 754, - /* 280 */ 817, 820, 825, 827, 830, 824, 835, 833, 847, 849, - /* 290 */ 866, 879, 882, 125, 909, 1161, 1162, 1105, 1171, 1134, - /* 300 */ 1009, 1137, 1139, 1141, 1015, 1181, 1146, 1148, 1022, 1188, - /* 310 */ 1151, 1190, 1154, 1191, 1119, 1043, 1045, 1088, 1051, 1201, - /* 320 */ 1202, 1150, 1061, 1207, 1208, 1122, 1210, 1211, 1212, 1224, - /* 330 */ 1225, 1226, 1228, 1229, 1230, 1231, 1232, 1234, 1235, 1239, - /* 340 */ 1241, 1242, 1244, 1093, 1251, 1253, 1254, 1255, 1258, 1259, - /* 350 */ 1240, 1261, 1265, 1266, 1267, 1268, 1269, 1270, 1271, 1272, - /* 360 */ 1233, 1273, 1237, 1277, 1279, 1245, 1246, 1243, 1280, 1247, - /* 370 */ 1248, 1249, 1289, 1257, 1274, 1250, 1302, 1278, 1281, 1262, - /* 380 */ 1306, 1307, 1309, 1311, 1236, 1193, 1276, 1252, 1288, 1290, - /* 390 */ 1313, 1283, 1285, 1287, 1296, 1299, 1252, 1288, 1300, 1303, - /* 400 */ 1318, 1297, 1344, 1324, 1308, 1345, 1326, 1312, 1351, 1331, - /* 410 */ 1362, 1341, 1346, 1367, 1256, 1332, 1369, 1219, 1352, 1260, - /* 420 */ 1238, 1373, 1376, 1282, 1377, 1304, 1337, 1263, 1310, 1314, - /* 430 */ 1180, 1315, 1320, 1317, 1319, 1322, 1339, 1325, 1340, 1328, - /* 440 */ 1342, 1349, 1187, 1335, 1348, 1329, 1198, 1354, 1360, 1353, - /* 450 */ 1355, 1196, 1380, 1393, 1394, 1395, 1397, 1398, 1436, 1291, - /* 460 */ 1378, 1375, 1374, 1382, 1386, 1388, 1379, 1387, 1390, 1381, - /* 470 */ 1445, 1298, 1396, 1400, 1401, 1399, 1404, 1334, 1406, 1473, - /* 480 */ 1439, 1350, 1408, 1402, 1403, 1407, 1466, 1411, 1409, 1415, - /* 490 */ 1454, 1455, 1423, 1425, 1470, 1435, 1438, 1474, 1440, 1441, - /* 500 */ 1478, 1443, 1446, 1481, 1448, 1421, 1426, 1427, 1428, 1489, - /* 510 */ 1437, 1457, 1491, 1429, 1458, 1460, 1498, 1252, 1288, 1515, - /* 520 */ 1487, 1490, 1504, 1486, 1471, 1475, 1508, 1511, 1512, 1513, - /* 530 */ 1520, 1522, 1531, 1549, 1516, 1252, 1536, 1288, 1537, 1538, - /* 540 */ 1540, 1542, 1543, 1544, 1545, 1584, 1548, 1551, 1547, 1588, - /* 550 */ 1557, 1560, 1554, 1598, 1562, 1565, 1561, 1603, 1569, 1572, - /* 560 */ 1566, 1610, 1579, 1582, 1620, 1621, 1600, 1602, 1611, 1612, - /* 570 */ 1615, 1617, + /* 0 */ 820, 0, 39, 216, 216, 216, 216, 255, 216, 216, + /* 10 */ 432, 471, 648, 471, 471, 471, 471, 471, 471, 471, + /* 20 */ 471, 471, 471, 471, 471, 471, 471, 471, 471, 471, + /* 30 */ 471, 471, 471, 471, 471, 471, 52, 73, 73, 73, + /* 40 */ 118, 671, 671, 112, 36, 36, 78, 671, 70, 36, + /* 50 */ 36, 36, 36, 36, 36, 107, 36, 272, 341, 78, + /* 60 */ 472, 272, 36, 36, 272, 36, 272, 472, 272, 272, + /* 70 */ 36, 366, 219, 397, 613, 613, 343, 831, 298, 141, + /* 80 */ 831, 831, 831, 831, 831, 831, 831, 831, 831, 831, + /* 90 */ 831, 831, 831, 831, 831, 831, 831, 831, 831, 340, + /* 100 */ 638, 419, 419, 183, 183, 183, 691, 419, 419, 490, + /* 110 */ 472, 272, 472, 272, 475, 548, 27, 27, 27, 27, + /* 120 */ 27, 27, 27, 1040, 115, 414, 552, 144, 169, 165, + /* 130 */ 244, 168, 403, 576, 297, 576, 758, 381, 637, 774, + /* 140 */ 941, 930, 932, 830, 941, 941, 853, 849, 849, 941, + /* 150 */ 971, 107, 472, 980, 107, 490, 987, 107, 107, 941, + /* 160 */ 107, 994, 272, 272, 272, 272, 272, 272, 272, 272, + /* 170 */ 272, 272, 272, 941, 994, 973, 971, 366, 908, 472, + /* 180 */ 366, 980, 366, 490, 987, 366, 1046, 879, 886, 973, + /* 190 */ 879, 886, 973, 973, 896, 881, 909, 912, 918, 490, + /* 200 */ 1100, 1001, 907, 911, 914, 1059, 272, 886, 973, 973, + /* 210 */ 886, 973, 1009, 490, 987, 366, 475, 366, 490, 1097, + /* 220 */ 548, 941, 366, 994, 1906, 1906, 1906, 1906, 1906, 1906, + /* 230 */ 1906, 99, 1359, 256, 13, 406, 837, 228, 405, 446, + /* 240 */ 621, 659, 666, 666, 666, 666, 666, 666, 666, 666, + /* 250 */ 523, 469, 247, 133, 315, 69, 69, 69, 69, 814, + /* 260 */ 336, 690, 762, 767, 772, 777, 845, 852, 870, 211, + /* 270 */ 590, 677, 786, 788, 799, 798, 679, 719, 559, 801, + /* 280 */ 826, 825, 617, 827, 828, 834, 835, 848, 883, 885, + /* 290 */ 859, 867, 875, 877, 880, 882, 233, 898, 1194, 1202, + /* 300 */ 1139, 1213, 1172, 1048, 1178, 1179, 1180, 1053, 1221, 1184, + /* 310 */ 1186, 1058, 1227, 1190, 1229, 1193, 1232, 1158, 1080, 1082, + /* 320 */ 1127, 1088, 1239, 1241, 1189, 1099, 1246, 1247, 1161, 1249, + /* 330 */ 1250, 1251, 1252, 1254, 1255, 1256, 1261, 1262, 1282, 1284, + /* 340 */ 1285, 1286, 1287, 1288, 1289, 1290, 1137, 1259, 1291, 1292, + /* 350 */ 1294, 1295, 1296, 1253, 1297, 1298, 1299, 1301, 1302, 1303, + /* 360 */ 1304, 1305, 1306, 1228, 1272, 1222, 1274, 1307, 1273, 1276, + /* 370 */ 1266, 1313, 1277, 1280, 1275, 1319, 1283, 1293, 1279, 1320, + /* 380 */ 1309, 1308, 1281, 1323, 1325, 1326, 1327, 1244, 1248, 1311, + /* 390 */ 1260, 1267, 1321, 1331, 1312, 1314, 1315, 1316, 1317, 1260, + /* 400 */ 1267, 1318, 1322, 1345, 1329, 1357, 1337, 1324, 1361, 1342, + /* 410 */ 1328, 1367, 1346, 1369, 1349, 1352, 1373, 1234, 1338, 1375, + /* 420 */ 1223, 1358, 1257, 1242, 1389, 1390, 1398, 1332, 1350, 1264, + /* 430 */ 1330, 1336, 1195, 1333, 1344, 1347, 1351, 1353, 1354, 1348, + /* 440 */ 1360, 1335, 1355, 1362, 1214, 1363, 1364, 1356, 1219, 1366, + /* 450 */ 1365, 1370, 1376, 1216, 1387, 1404, 1407, 1409, 1411, 1412, + /* 460 */ 1429, 1300, 1379, 1377, 1380, 1378, 1381, 1394, 1371, 1397, + /* 470 */ 1399, 1386, 1435, 1310, 1402, 1403, 1405, 1413, 1415, 1340, + /* 480 */ 1416, 1469, 1439, 1372, 1418, 1401, 1400, 1408, 1462, 1422, + /* 490 */ 1428, 1443, 1480, 1482, 1446, 1448, 1484, 1450, 1451, 1488, + /* 500 */ 1453, 1454, 1493, 1457, 1458, 1497, 1463, 1440, 1441, 1444, + /* 510 */ 1445, 1517, 1455, 1467, 1509, 1438, 1474, 1476, 1514, 1260, + /* 520 */ 1267, 1531, 1503, 1505, 1524, 1507, 1492, 1494, 1529, 1537, + /* 530 */ 1540, 1541, 1542, 1543, 1544, 1561, 1527, 1260, 1547, 1267, + /* 540 */ 1548, 1549, 1550, 1552, 1553, 1555, 1556, 1589, 1557, 1560, + /* 550 */ 1554, 1598, 1562, 1566, 1563, 1603, 1567, 1568, 1564, 1610, + /* 560 */ 1573, 1576, 1571, 1615, 1579, 1582, 1620, 1621, 1601, 1604, + /* 570 */ 1602, 1606, 1605, 1609, }; -#define YY_REDUCE_COUNT (229) -#define YY_REDUCE_MIN (-273) -#define YY_REDUCE_MAX (1664) +#define YY_REDUCE_COUNT (230) +#define YY_REDUCE_MIN (-323) +#define YY_REDUCE_MAX (1600) static const short yy_reduce_ofst[] = { - /* 0 */ -212, -210, -159, 368, 13, 265, 440, 625, 638, 719, - /* 10 */ 771, 826, 875, 927, -221, 957, 1008, 1038, 1066, 1115, - /* 20 */ 1153, 1164, 1214, 1264, 1275, 1323, 1336, 1405, 1414, 1464, - /* 30 */ 1514, 1525, 1573, 1586, 1655, 1664, 929, -183, -104, 453, - /* 40 */ -125, -253, -251, -211, -226, 266, 50, -241, -273, -118, - /* 50 */ -89, 222, 267, 294, 306, -93, 438, -59, -248, 51, - /* 60 */ -263, 206, 448, 471, -6, 481, 210, -171, 281, 271, - /* 70 */ 452, 16, -224, -200, -200, -200, -208, 154, 59, -198, - /* 80 */ 221, 263, 304, 398, 456, 488, 489, 495, 497, 498, - /* 90 */ 499, 526, 529, 530, 532, 538, 578, 581, 582, 183, - /* 100 */ -178, 275, 316, -10, 4, 289, 10, 397, 462, 276, - /* 110 */ 188, 433, 527, 310, -12, -153, -239, 323, 445, 475, - /* 120 */ 506, 525, 548, 589, 658, 660, 573, 585, 652, 630, - /* 130 */ 603, 696, 648, 620, 620, 620, 688, 618, 633, 688, - /* 140 */ 729, 682, 732, 694, 740, 746, 715, 721, 722, 755, - /* 150 */ 708, 757, 726, 727, 769, 747, 738, 773, 780, 784, - /* 160 */ 783, 796, 775, 776, 779, 782, 785, 786, 789, 790, - /* 170 */ 791, 794, 799, 795, 802, 787, 770, 819, 797, 803, - /* 180 */ 828, 781, 840, 815, 805, 850, 798, 759, 810, 816, - /* 190 */ 763, 818, 821, 823, 788, 800, 804, 808, 620, 842, - /* 200 */ 822, 801, 768, 807, 809, 811, 688, 852, 841, 843, - /* 210 */ 856, 860, 864, 873, 867, 903, 883, 905, 884, 891, - /* 220 */ 907, 916, 914, 921, 876, 868, 908, 912, 918, 934, + /* 0 */ -34, 221, -232, 431, -3, 582, 614, 734, 484, 743, + /* 10 */ 804, 858, 906, 274, 890, 965, 978, 1035, 1083, 1112, + /* 20 */ 1162, 1201, 1210, 1258, 1271, 1334, 1343, 1382, 1393, 1442, + /* 30 */ 1481, 1490, 1499, 1538, 1551, 1600, 436, -210, 710, 824, + /* 40 */ 213, -258, -255, -279, -240, -221, -89, -231, -323, -161, + /* 50 */ 29, 225, 262, 276, 295, -177, 331, -235, -67, -77, + /* 60 */ -270, -153, 454, 459, 1, 462, -58, -172, 58, 158, + /* 70 */ 476, 299, 61, -268, -268, -268, 150, 55, 49, -216, + /* 80 */ 143, 261, 269, 278, 284, 344, 351, 365, 388, 390, + /* 90 */ 398, 428, 486, 492, 514, 516, 522, 534, 536, 226, + /* 100 */ -151, 243, 357, -46, 368, 370, -66, 395, 468, -262, + /* 110 */ 224, 289, 440, 521, 174, 452, -244, 426, 458, 463, + /* 120 */ 478, 553, 604, 598, 643, 644, 564, 574, 650, 641, + /* 130 */ 601, 697, 645, 618, 618, 618, 699, 622, 634, 699, + /* 140 */ 722, 676, 727, 687, 736, 739, 707, 711, 714, 747, + /* 150 */ 704, 754, 723, 721, 763, 741, 735, 768, 773, 776, + /* 160 */ 775, 784, 765, 766, 782, 789, 791, 792, 794, 795, + /* 170 */ 796, 802, 803, 785, 790, 778, 780, 829, 783, 805, + /* 180 */ 838, 807, 839, 809, 787, 842, 770, 757, 813, 816, + /* 190 */ 764, 817, 821, 822, 797, 769, 793, 800, 618, 854, + /* 200 */ 833, 810, 806, 812, 811, 843, 699, 855, 857, 860, + /* 210 */ 864, 874, 861, 884, 891, 923, 916, 927, 903, 913, + /* 220 */ 925, 935, 938, 942, 893, 892, 900, 934, 945, 946, + /* 230 */ 950, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 10 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 20 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 30 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 40 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 50 */ 1287, 1287, 1287, 1287, 1287, 1346, 1287, 1287, 1287, 1287, - /* 60 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 70 */ 1287, 1344, 1489, 1287, 1645, 1287, 1287, 1287, 1287, 1287, - /* 80 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 90 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 100 */ 1346, 1287, 1287, 1656, 1656, 1656, 1344, 1287, 1287, 1287, - /* 110 */ 1287, 1287, 1287, 1287, 1441, 1287, 1287, 1287, 1287, 1287, - /* 120 */ 1287, 1287, 1287, 1523, 1287, 1287, 1721, 1287, 1394, 1529, - /* 130 */ 1680, 1287, 1672, 1648, 1662, 1649, 1287, 1706, 1665, 1287, - /* 140 */ 1287, 1287, 1287, 1515, 1287, 1287, 1494, 1491, 1491, 1287, - /* 150 */ 1287, 1346, 1287, 1287, 1346, 1287, 1287, 1346, 1346, 1287, - /* 160 */ 1346, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 170 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1344, 1525, 1287, - /* 180 */ 1344, 1287, 1344, 1287, 1287, 1344, 1287, 1687, 1685, 1287, - /* 190 */ 1687, 1685, 1287, 1287, 1699, 1695, 1678, 1676, 1662, 1287, - /* 200 */ 1287, 1287, 1724, 1712, 1708, 1287, 1287, 1685, 1287, 1287, - /* 210 */ 1685, 1287, 1502, 1287, 1287, 1344, 1287, 1344, 1287, 1410, - /* 220 */ 1287, 1287, 1344, 1287, 1517, 1531, 1444, 1444, 1347, 1292, - /* 230 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 240 */ 1287, 1592, 1698, 1697, 1621, 1620, 1619, 1617, 1591, 1287, - /* 250 */ 1287, 1287, 1287, 1585, 1586, 1584, 1583, 1287, 1287, 1287, - /* 260 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 270 */ 1287, 1287, 1646, 1287, 1709, 1713, 1287, 1287, 1287, 1569, - /* 280 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 290 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 300 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 310 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 320 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 330 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 340 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 350 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 360 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 370 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 380 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1457, 1456, 1287, - /* 390 */ 1287, 1287, 1287, 1287, 1287, 1287, 1374, 1373, 1287, 1287, - /* 400 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 410 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 420 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1669, 1679, - /* 430 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1569, - /* 440 */ 1287, 1696, 1287, 1655, 1651, 1287, 1287, 1647, 1287, 1287, - /* 450 */ 1707, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1641, 1287, - /* 460 */ 1614, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 470 */ 1287, 1579, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 480 */ 1287, 1287, 1287, 1287, 1568, 1287, 1287, 1287, 1287, 1287, - /* 490 */ 1287, 1287, 1438, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 500 */ 1287, 1287, 1287, 1287, 1287, 1423, 1421, 1420, 1419, 1287, - /* 510 */ 1416, 1287, 1287, 1287, 1287, 1287, 1287, 1447, 1446, 1287, - /* 520 */ 1287, 1287, 1287, 1287, 1287, 1367, 1287, 1287, 1287, 1287, - /* 530 */ 1287, 1287, 1287, 1287, 1287, 1358, 1287, 1357, 1287, 1287, - /* 540 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 550 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 560 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 570 */ 1287, 1287, + /* 0 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 10 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 20 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 30 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 40 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 50 */ 1296, 1296, 1296, 1296, 1296, 1355, 1296, 1296, 1296, 1296, + /* 60 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 70 */ 1296, 1353, 1498, 1296, 1658, 1296, 1296, 1296, 1296, 1296, + /* 80 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 90 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 100 */ 1355, 1296, 1296, 1669, 1669, 1669, 1353, 1296, 1296, 1296, + /* 110 */ 1296, 1296, 1296, 1296, 1450, 1296, 1296, 1296, 1296, 1296, + /* 120 */ 1296, 1296, 1296, 1536, 1296, 1296, 1734, 1296, 1403, 1542, + /* 130 */ 1693, 1296, 1685, 1661, 1675, 1662, 1296, 1719, 1678, 1296, + /* 140 */ 1296, 1296, 1296, 1528, 1296, 1296, 1503, 1500, 1500, 1296, + /* 150 */ 1296, 1355, 1296, 1296, 1355, 1296, 1296, 1355, 1355, 1296, + /* 160 */ 1355, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 170 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1353, 1538, 1296, + /* 180 */ 1353, 1296, 1353, 1296, 1296, 1353, 1296, 1700, 1698, 1296, + /* 190 */ 1700, 1698, 1296, 1296, 1712, 1708, 1691, 1689, 1675, 1296, + /* 200 */ 1296, 1296, 1737, 1725, 1721, 1296, 1296, 1698, 1296, 1296, + /* 210 */ 1698, 1296, 1511, 1296, 1296, 1353, 1296, 1353, 1296, 1419, + /* 220 */ 1296, 1296, 1353, 1296, 1530, 1544, 1520, 1453, 1453, 1356, + /* 230 */ 1301, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 240 */ 1296, 1296, 1605, 1711, 1710, 1634, 1633, 1632, 1630, 1604, + /* 250 */ 1296, 1296, 1296, 1296, 1296, 1598, 1599, 1597, 1596, 1296, + /* 260 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 270 */ 1296, 1296, 1296, 1296, 1296, 1659, 1296, 1722, 1726, 1296, + /* 280 */ 1296, 1296, 1582, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 290 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 300 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 310 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 320 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 330 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 340 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 350 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 360 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 370 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 380 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 390 */ 1466, 1465, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1383, + /* 400 */ 1382, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 410 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 420 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 430 */ 1682, 1692, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 440 */ 1296, 1582, 1296, 1709, 1296, 1668, 1664, 1296, 1296, 1660, + /* 450 */ 1296, 1296, 1720, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 460 */ 1654, 1296, 1627, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 470 */ 1296, 1296, 1296, 1592, 1296, 1296, 1296, 1296, 1296, 1296, + /* 480 */ 1296, 1296, 1296, 1296, 1296, 1296, 1581, 1296, 1296, 1296, + /* 490 */ 1296, 1296, 1296, 1296, 1447, 1296, 1296, 1296, 1296, 1296, + /* 500 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1432, 1430, 1429, + /* 510 */ 1428, 1296, 1425, 1296, 1296, 1296, 1296, 1296, 1296, 1456, + /* 520 */ 1455, 1296, 1296, 1296, 1296, 1296, 1296, 1376, 1296, 1296, + /* 530 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1367, 1296, 1366, + /* 540 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 550 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 560 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, + /* 570 */ 1296, 1296, 1296, 1296, }; /********** End of lemon-generated parsing tables *****************************/ @@ -775,6 +767,237 @@ static const YYACTIONTYPE yy_default[] = { */ #ifdef YYFALLBACK static const YYCODETYPE yyFallback[] = { + 0, /* $ => nothing */ + 0, /* OR => nothing */ + 0, /* AND => nothing */ + 0, /* UNION => nothing */ + 0, /* ALL => nothing */ + 0, /* MINUS => nothing */ + 0, /* EXCEPT => nothing */ + 0, /* INTERSECT => nothing */ + 0, /* NK_BITAND => nothing */ + 0, /* NK_BITOR => nothing */ + 0, /* NK_LSHIFT => nothing */ + 0, /* NK_RSHIFT => nothing */ + 0, /* NK_PLUS => nothing */ + 0, /* NK_MINUS => nothing */ + 0, /* NK_STAR => nothing */ + 0, /* NK_SLASH => nothing */ + 0, /* NK_REM => nothing */ + 0, /* NK_CONCAT => nothing */ + 0, /* CREATE => nothing */ + 0, /* ACCOUNT => nothing */ + 0, /* NK_ID => nothing */ + 0, /* PASS => nothing */ + 0, /* NK_STRING => nothing */ + 0, /* ALTER => nothing */ + 0, /* PPS => nothing */ + 0, /* TSERIES => nothing */ + 0, /* STORAGE => nothing */ + 0, /* STREAMS => nothing */ + 0, /* QTIME => nothing */ + 0, /* DBS => nothing */ + 0, /* USERS => nothing */ + 0, /* CONNS => nothing */ + 0, /* STATE => nothing */ + 0, /* USER => nothing */ + 0, /* PRIVILEGE => nothing */ + 0, /* DROP => nothing */ + 0, /* DNODE => nothing */ + 0, /* PORT => nothing */ + 0, /* NK_INTEGER => nothing */ + 0, /* DNODES => nothing */ + 0, /* NK_IPTOKEN => nothing */ + 0, /* LOCAL => nothing */ + 0, /* QNODE => nothing */ + 0, /* ON => nothing */ + 0, /* BNODE => nothing */ + 0, /* SNODE => nothing */ + 0, /* MNODE => nothing */ + 0, /* DATABASE => nothing */ + 0, /* USE => nothing */ + 0, /* IF => nothing */ + 0, /* NOT => nothing */ + 0, /* EXISTS => nothing */ + 0, /* BLOCKS => nothing */ + 0, /* CACHE => nothing */ + 0, /* CACHELAST => nothing */ + 0, /* COMP => nothing */ + 0, /* DAYS => nothing */ + 0, /* NK_VARIABLE => nothing */ + 0, /* FSYNC => nothing */ + 0, /* MAXROWS => nothing */ + 0, /* MINROWS => nothing */ + 0, /* KEEP => nothing */ + 0, /* PRECISION => nothing */ + 0, /* QUORUM => nothing */ + 0, /* REPLICA => nothing */ + 0, /* TTL => nothing */ + 0, /* WAL => nothing */ + 0, /* VGROUPS => nothing */ + 0, /* SINGLE_STABLE => nothing */ + 0, /* STREAM_MODE => nothing */ + 0, /* RETENTIONS => nothing */ + 0, /* STRICT => nothing */ + 0, /* NK_COMMA => nothing */ + 0, /* NK_COLON => nothing */ + 0, /* TABLE => nothing */ + 0, /* NK_LP => nothing */ + 0, /* NK_RP => nothing */ + 0, /* STABLE => nothing */ + 0, /* ADD => nothing */ + 0, /* COLUMN => nothing */ + 0, /* MODIFY => nothing */ + 0, /* RENAME => nothing */ + 0, /* TAG => nothing */ + 0, /* SET => nothing */ + 0, /* NK_EQ => nothing */ + 0, /* USING => nothing */ + 0, /* TAGS => nothing */ + 0, /* NK_DOT => nothing */ + 0, /* COMMENT => nothing */ + 0, /* BOOL => nothing */ + 0, /* TINYINT => nothing */ + 0, /* SMALLINT => nothing */ + 0, /* INT => nothing */ + 0, /* INTEGER => nothing */ + 0, /* BIGINT => nothing */ + 0, /* FLOAT => nothing */ + 0, /* DOUBLE => nothing */ + 0, /* BINARY => nothing */ + 0, /* TIMESTAMP => nothing */ + 0, /* NCHAR => nothing */ + 0, /* UNSIGNED => nothing */ + 0, /* JSON => nothing */ + 0, /* VARCHAR => nothing */ + 0, /* MEDIUMBLOB => nothing */ + 0, /* BLOB => nothing */ + 0, /* VARBINARY => nothing */ + 0, /* DECIMAL => nothing */ + 0, /* SMA => nothing */ + 0, /* ROLLUP => nothing */ + 0, /* FILE_FACTOR => nothing */ + 0, /* NK_FLOAT => nothing */ + 0, /* DELAY => nothing */ + 0, /* SHOW => nothing */ + 0, /* DATABASES => nothing */ + 0, /* TABLES => nothing */ + 0, /* STABLES => nothing */ + 0, /* MNODES => nothing */ + 0, /* MODULES => nothing */ + 0, /* QNODES => nothing */ + 0, /* FUNCTIONS => nothing */ + 0, /* INDEXES => nothing */ + 0, /* FROM => nothing */ + 0, /* ACCOUNTS => nothing */ + 0, /* APPS => nothing */ + 0, /* CONNECTIONS => nothing */ + 0, /* LICENCE => nothing */ + 0, /* GRANTS => nothing */ + 0, /* QUERIES => nothing */ + 0, /* SCORES => nothing */ + 0, /* TOPICS => nothing */ + 0, /* VARIABLES => nothing */ + 0, /* BNODES => nothing */ + 0, /* SNODES => nothing */ + 0, /* CLUSTER => nothing */ + 0, /* LIKE => nothing */ + 0, /* INDEX => nothing */ + 0, /* FULLTEXT => nothing */ + 0, /* FUNCTION => nothing */ + 0, /* INTERVAL => nothing */ + 0, /* TOPIC => nothing */ + 0, /* AS => nothing */ + 0, /* WITH => nothing */ + 0, /* SCHEMA => nothing */ + 0, /* DESC => nothing */ + 0, /* DESCRIBE => nothing */ + 0, /* RESET => nothing */ + 0, /* QUERY => nothing */ + 0, /* EXPLAIN => nothing */ + 0, /* ANALYZE => nothing */ + 0, /* VERBOSE => nothing */ + 0, /* NK_BOOL => nothing */ + 0, /* RATIO => nothing */ + 0, /* COMPACT => nothing */ + 0, /* VNODES => nothing */ + 0, /* IN => nothing */ + 0, /* OUTPUTTYPE => nothing */ + 0, /* AGGREGATE => nothing */ + 0, /* BUFSIZE => nothing */ + 0, /* STREAM => nothing */ + 0, /* INTO => nothing */ + 0, /* TRIGGER => nothing */ + 0, /* AT_ONCE => nothing */ + 0, /* WINDOW_CLOSE => nothing */ + 0, /* WATERMARK => nothing */ + 0, /* KILL => nothing */ + 0, /* CONNECTION => nothing */ + 0, /* MERGE => nothing */ + 0, /* VGROUP => nothing */ + 0, /* REDISTRIBUTE => nothing */ + 0, /* SPLIT => nothing */ + 0, /* SYNCDB => nothing */ + 0, /* NULL => nothing */ + 0, /* NK_QUESTION => nothing */ + 0, /* NK_ARROW => nothing */ + 0, /* ROWTS => nothing */ + 0, /* TBNAME => nothing */ + 0, /* QSTARTTS => nothing */ + 0, /* QENDTS => nothing */ + 0, /* WSTARTTS => nothing */ + 0, /* WENDTS => nothing */ + 0, /* WDURATION => nothing */ + 0, /* CAST => nothing */ + 0, /* NOW => nothing */ + 0, /* TODAY => nothing */ + 0, /* TIMEZONE => nothing */ + 0, /* COUNT => nothing */ + 0, /* FIRST => nothing */ + 0, /* LAST => nothing */ + 0, /* LAST_ROW => nothing */ + 0, /* BETWEEN => nothing */ + 0, /* IS => nothing */ + 0, /* NK_LT => nothing */ + 0, /* NK_GT => nothing */ + 0, /* NK_LE => nothing */ + 0, /* NK_GE => nothing */ + 0, /* NK_NE => nothing */ + 0, /* MATCH => nothing */ + 0, /* NMATCH => nothing */ + 0, /* CONTAINS => nothing */ + 0, /* JOIN => nothing */ + 0, /* INNER => nothing */ + 0, /* SELECT => nothing */ + 0, /* DISTINCT => nothing */ + 0, /* WHERE => nothing */ + 0, /* PARTITION => nothing */ + 0, /* BY => nothing */ + 0, /* SESSION => nothing */ + 0, /* STATE_WINDOW => nothing */ + 0, /* SLIDING => nothing */ + 0, /* FILL => nothing */ + 0, /* VALUE => nothing */ + 0, /* NONE => nothing */ + 0, /* PREV => nothing */ + 0, /* LINEAR => nothing */ + 0, /* NEXT => nothing */ + 0, /* GROUP => nothing */ + 0, /* HAVING => nothing */ + 0, /* ORDER => nothing */ + 0, /* SLIMIT => nothing */ + 0, /* SOFFSET => nothing */ + 0, /* LIMIT => nothing */ + 0, /* OFFSET => nothing */ + 0, /* ASC => nothing */ + 0, /* NULLS => nothing */ + 0, /* ID => nothing */ + 224, /* NK_BITNOT => ID */ + 224, /* INSERT => ID */ + 224, /* VALUES => ID */ + 224, /* IMPORT => ID */ + 224, /* NK_SEMI => ID */ + 224, /* FILE => ID */ }; #endif /* YYFALLBACK */ @@ -1003,201 +1226,211 @@ static const char *const yyTokenName[] = { /* 138 */ "INTERVAL", /* 139 */ "TOPIC", /* 140 */ "AS", - /* 141 */ "DESC", - /* 142 */ "DESCRIBE", - /* 143 */ "RESET", - /* 144 */ "QUERY", - /* 145 */ "EXPLAIN", - /* 146 */ "ANALYZE", - /* 147 */ "VERBOSE", - /* 148 */ "NK_BOOL", - /* 149 */ "RATIO", - /* 150 */ "COMPACT", - /* 151 */ "VNODES", - /* 152 */ "IN", - /* 153 */ "OUTPUTTYPE", - /* 154 */ "AGGREGATE", - /* 155 */ "BUFSIZE", - /* 156 */ "STREAM", - /* 157 */ "INTO", - /* 158 */ "TRIGGER", - /* 159 */ "AT_ONCE", - /* 160 */ "WINDOW_CLOSE", - /* 161 */ "WATERMARK", - /* 162 */ "KILL", - /* 163 */ "CONNECTION", - /* 164 */ "MERGE", - /* 165 */ "VGROUP", - /* 166 */ "REDISTRIBUTE", - /* 167 */ "SPLIT", - /* 168 */ "SYNCDB", - /* 169 */ "NULL", - /* 170 */ "NK_QUESTION", - /* 171 */ "NK_ARROW", - /* 172 */ "ROWTS", - /* 173 */ "TBNAME", - /* 174 */ "QSTARTTS", - /* 175 */ "QENDTS", - /* 176 */ "WSTARTTS", - /* 177 */ "WENDTS", - /* 178 */ "WDURATION", - /* 179 */ "CAST", - /* 180 */ "NOW", - /* 181 */ "TODAY", - /* 182 */ "TIMEZONE", - /* 183 */ "COUNT", - /* 184 */ "FIRST", - /* 185 */ "LAST", - /* 186 */ "LAST_ROW", - /* 187 */ "BETWEEN", - /* 188 */ "IS", - /* 189 */ "NK_LT", - /* 190 */ "NK_GT", - /* 191 */ "NK_LE", - /* 192 */ "NK_GE", - /* 193 */ "NK_NE", - /* 194 */ "MATCH", - /* 195 */ "NMATCH", - /* 196 */ "CONTAINS", - /* 197 */ "JOIN", - /* 198 */ "INNER", - /* 199 */ "SELECT", - /* 200 */ "DISTINCT", - /* 201 */ "WHERE", - /* 202 */ "PARTITION", - /* 203 */ "BY", - /* 204 */ "SESSION", - /* 205 */ "STATE_WINDOW", - /* 206 */ "SLIDING", - /* 207 */ "FILL", - /* 208 */ "VALUE", - /* 209 */ "NONE", - /* 210 */ "PREV", - /* 211 */ "LINEAR", - /* 212 */ "NEXT", - /* 213 */ "GROUP", - /* 214 */ "HAVING", - /* 215 */ "ORDER", - /* 216 */ "SLIMIT", - /* 217 */ "SOFFSET", - /* 218 */ "LIMIT", - /* 219 */ "OFFSET", - /* 220 */ "ASC", - /* 221 */ "NULLS", - /* 222 */ "cmd", - /* 223 */ "account_options", - /* 224 */ "alter_account_options", - /* 225 */ "literal", - /* 226 */ "alter_account_option", - /* 227 */ "user_name", - /* 228 */ "dnode_endpoint", - /* 229 */ "dnode_host_name", - /* 230 */ "not_exists_opt", - /* 231 */ "db_name", - /* 232 */ "db_options", - /* 233 */ "exists_opt", - /* 234 */ "alter_db_options", - /* 235 */ "integer_list", - /* 236 */ "variable_list", - /* 237 */ "retention_list", - /* 238 */ "alter_db_option", - /* 239 */ "retention", - /* 240 */ "full_table_name", - /* 241 */ "column_def_list", - /* 242 */ "tags_def_opt", - /* 243 */ "table_options", - /* 244 */ "multi_create_clause", - /* 245 */ "tags_def", - /* 246 */ "multi_drop_clause", - /* 247 */ "alter_table_clause", - /* 248 */ "alter_table_options", - /* 249 */ "column_name", - /* 250 */ "type_name", - /* 251 */ "create_subtable_clause", - /* 252 */ "specific_tags_opt", - /* 253 */ "literal_list", - /* 254 */ "drop_table_clause", - /* 255 */ "col_name_list", - /* 256 */ "table_name", - /* 257 */ "column_def", - /* 258 */ "func_name_list", - /* 259 */ "alter_table_option", - /* 260 */ "col_name", - /* 261 */ "db_name_cond_opt", - /* 262 */ "like_pattern_opt", - /* 263 */ "table_name_cond", - /* 264 */ "from_db_opt", - /* 265 */ "func_name", - /* 266 */ "function_name", - /* 267 */ "index_name", - /* 268 */ "index_options", - /* 269 */ "func_list", - /* 270 */ "duration_literal", - /* 271 */ "sliding_opt", - /* 272 */ "func", - /* 273 */ "expression_list", - /* 274 */ "topic_name", - /* 275 */ "query_expression", - /* 276 */ "analyze_opt", - /* 277 */ "explain_options", - /* 278 */ "agg_func_opt", - /* 279 */ "bufsize_opt", - /* 280 */ "stream_name", - /* 281 */ "stream_options", - /* 282 */ "into_opt", - /* 283 */ "dnode_list", - /* 284 */ "signed", - /* 285 */ "signed_literal", - /* 286 */ "table_alias", - /* 287 */ "column_alias", - /* 288 */ "expression", - /* 289 */ "pseudo_column", - /* 290 */ "column_reference", - /* 291 */ "function_expression", - /* 292 */ "subquery", - /* 293 */ "star_func", - /* 294 */ "star_func_para_list", - /* 295 */ "noarg_func", - /* 296 */ "other_para_list", - /* 297 */ "star_func_para", - /* 298 */ "predicate", - /* 299 */ "compare_op", - /* 300 */ "in_op", - /* 301 */ "in_predicate_value", - /* 302 */ "boolean_value_expression", - /* 303 */ "boolean_primary", - /* 304 */ "common_expression", - /* 305 */ "from_clause", - /* 306 */ "table_reference_list", - /* 307 */ "table_reference", - /* 308 */ "table_primary", - /* 309 */ "joined_table", - /* 310 */ "alias_opt", - /* 311 */ "parenthesized_joined_table", - /* 312 */ "join_type", - /* 313 */ "search_condition", - /* 314 */ "query_specification", - /* 315 */ "set_quantifier_opt", - /* 316 */ "select_list", - /* 317 */ "where_clause_opt", - /* 318 */ "partition_by_clause_opt", - /* 319 */ "twindow_clause_opt", - /* 320 */ "group_by_clause_opt", - /* 321 */ "having_clause_opt", - /* 322 */ "select_sublist", - /* 323 */ "select_item", - /* 324 */ "fill_opt", - /* 325 */ "fill_mode", - /* 326 */ "group_by_list", - /* 327 */ "query_expression_body", - /* 328 */ "order_by_clause_opt", - /* 329 */ "slimit_clause_opt", - /* 330 */ "limit_clause_opt", - /* 331 */ "query_primary", - /* 332 */ "sort_specification_list", - /* 333 */ "sort_specification", - /* 334 */ "ordering_specification_opt", - /* 335 */ "null_ordering_opt", + /* 141 */ "WITH", + /* 142 */ "SCHEMA", + /* 143 */ "DESC", + /* 144 */ "DESCRIBE", + /* 145 */ "RESET", + /* 146 */ "QUERY", + /* 147 */ "EXPLAIN", + /* 148 */ "ANALYZE", + /* 149 */ "VERBOSE", + /* 150 */ "NK_BOOL", + /* 151 */ "RATIO", + /* 152 */ "COMPACT", + /* 153 */ "VNODES", + /* 154 */ "IN", + /* 155 */ "OUTPUTTYPE", + /* 156 */ "AGGREGATE", + /* 157 */ "BUFSIZE", + /* 158 */ "STREAM", + /* 159 */ "INTO", + /* 160 */ "TRIGGER", + /* 161 */ "AT_ONCE", + /* 162 */ "WINDOW_CLOSE", + /* 163 */ "WATERMARK", + /* 164 */ "KILL", + /* 165 */ "CONNECTION", + /* 166 */ "MERGE", + /* 167 */ "VGROUP", + /* 168 */ "REDISTRIBUTE", + /* 169 */ "SPLIT", + /* 170 */ "SYNCDB", + /* 171 */ "NULL", + /* 172 */ "NK_QUESTION", + /* 173 */ "NK_ARROW", + /* 174 */ "ROWTS", + /* 175 */ "TBNAME", + /* 176 */ "QSTARTTS", + /* 177 */ "QENDTS", + /* 178 */ "WSTARTTS", + /* 179 */ "WENDTS", + /* 180 */ "WDURATION", + /* 181 */ "CAST", + /* 182 */ "NOW", + /* 183 */ "TODAY", + /* 184 */ "TIMEZONE", + /* 185 */ "COUNT", + /* 186 */ "FIRST", + /* 187 */ "LAST", + /* 188 */ "LAST_ROW", + /* 189 */ "BETWEEN", + /* 190 */ "IS", + /* 191 */ "NK_LT", + /* 192 */ "NK_GT", + /* 193 */ "NK_LE", + /* 194 */ "NK_GE", + /* 195 */ "NK_NE", + /* 196 */ "MATCH", + /* 197 */ "NMATCH", + /* 198 */ "CONTAINS", + /* 199 */ "JOIN", + /* 200 */ "INNER", + /* 201 */ "SELECT", + /* 202 */ "DISTINCT", + /* 203 */ "WHERE", + /* 204 */ "PARTITION", + /* 205 */ "BY", + /* 206 */ "SESSION", + /* 207 */ "STATE_WINDOW", + /* 208 */ "SLIDING", + /* 209 */ "FILL", + /* 210 */ "VALUE", + /* 211 */ "NONE", + /* 212 */ "PREV", + /* 213 */ "LINEAR", + /* 214 */ "NEXT", + /* 215 */ "GROUP", + /* 216 */ "HAVING", + /* 217 */ "ORDER", + /* 218 */ "SLIMIT", + /* 219 */ "SOFFSET", + /* 220 */ "LIMIT", + /* 221 */ "OFFSET", + /* 222 */ "ASC", + /* 223 */ "NULLS", + /* 224 */ "ID", + /* 225 */ "NK_BITNOT", + /* 226 */ "INSERT", + /* 227 */ "VALUES", + /* 228 */ "IMPORT", + /* 229 */ "NK_SEMI", + /* 230 */ "FILE", + /* 231 */ "cmd", + /* 232 */ "account_options", + /* 233 */ "alter_account_options", + /* 234 */ "literal", + /* 235 */ "alter_account_option", + /* 236 */ "user_name", + /* 237 */ "dnode_endpoint", + /* 238 */ "dnode_host_name", + /* 239 */ "not_exists_opt", + /* 240 */ "db_name", + /* 241 */ "db_options", + /* 242 */ "exists_opt", + /* 243 */ "alter_db_options", + /* 244 */ "integer_list", + /* 245 */ "variable_list", + /* 246 */ "retention_list", + /* 247 */ "alter_db_option", + /* 248 */ "retention", + /* 249 */ "full_table_name", + /* 250 */ "column_def_list", + /* 251 */ "tags_def_opt", + /* 252 */ "table_options", + /* 253 */ "multi_create_clause", + /* 254 */ "tags_def", + /* 255 */ "multi_drop_clause", + /* 256 */ "alter_table_clause", + /* 257 */ "alter_table_options", + /* 258 */ "column_name", + /* 259 */ "type_name", + /* 260 */ "create_subtable_clause", + /* 261 */ "specific_tags_opt", + /* 262 */ "literal_list", + /* 263 */ "drop_table_clause", + /* 264 */ "col_name_list", + /* 265 */ "table_name", + /* 266 */ "column_def", + /* 267 */ "func_name_list", + /* 268 */ "alter_table_option", + /* 269 */ "col_name", + /* 270 */ "db_name_cond_opt", + /* 271 */ "like_pattern_opt", + /* 272 */ "table_name_cond", + /* 273 */ "from_db_opt", + /* 274 */ "func_name", + /* 275 */ "function_name", + /* 276 */ "index_name", + /* 277 */ "index_options", + /* 278 */ "func_list", + /* 279 */ "duration_literal", + /* 280 */ "sliding_opt", + /* 281 */ "func", + /* 282 */ "expression_list", + /* 283 */ "topic_name", + /* 284 */ "topic_options", + /* 285 */ "query_expression", + /* 286 */ "analyze_opt", + /* 287 */ "explain_options", + /* 288 */ "agg_func_opt", + /* 289 */ "bufsize_opt", + /* 290 */ "stream_name", + /* 291 */ "stream_options", + /* 292 */ "into_opt", + /* 293 */ "dnode_list", + /* 294 */ "signed", + /* 295 */ "signed_literal", + /* 296 */ "table_alias", + /* 297 */ "column_alias", + /* 298 */ "expression", + /* 299 */ "pseudo_column", + /* 300 */ "column_reference", + /* 301 */ "function_expression", + /* 302 */ "subquery", + /* 303 */ "star_func", + /* 304 */ "star_func_para_list", + /* 305 */ "noarg_func", + /* 306 */ "other_para_list", + /* 307 */ "star_func_para", + /* 308 */ "predicate", + /* 309 */ "compare_op", + /* 310 */ "in_op", + /* 311 */ "in_predicate_value", + /* 312 */ "boolean_value_expression", + /* 313 */ "boolean_primary", + /* 314 */ "common_expression", + /* 315 */ "from_clause", + /* 316 */ "table_reference_list", + /* 317 */ "table_reference", + /* 318 */ "table_primary", + /* 319 */ "joined_table", + /* 320 */ "alias_opt", + /* 321 */ "parenthesized_joined_table", + /* 322 */ "join_type", + /* 323 */ "search_condition", + /* 324 */ "query_specification", + /* 325 */ "set_quantifier_opt", + /* 326 */ "select_list", + /* 327 */ "where_clause_opt", + /* 328 */ "partition_by_clause_opt", + /* 329 */ "twindow_clause_opt", + /* 330 */ "group_by_clause_opt", + /* 331 */ "having_clause_opt", + /* 332 */ "select_sublist", + /* 333 */ "select_item", + /* 334 */ "fill_opt", + /* 335 */ "fill_mode", + /* 336 */ "group_by_list", + /* 337 */ "query_expression_body", + /* 338 */ "order_by_clause_opt", + /* 339 */ "slimit_clause_opt", + /* 340 */ "limit_clause_opt", + /* 341 */ "query_primary", + /* 342 */ "sort_specification_list", + /* 343 */ "sort_specification", + /* 344 */ "ordering_specification_opt", + /* 345 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -1423,225 +1656,229 @@ static const char *const yyRuleName[] = { /* 215 */ "func_list ::= func", /* 216 */ "func_list ::= func_list NK_COMMA func", /* 217 */ "func ::= function_name NK_LP expression_list NK_RP", - /* 218 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression", - /* 219 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name", + /* 218 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name topic_options AS query_expression", + /* 219 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name topic_options AS db_name", /* 220 */ "cmd ::= DROP TOPIC exists_opt topic_name", - /* 221 */ "cmd ::= DESC full_table_name", - /* 222 */ "cmd ::= DESCRIBE full_table_name", - /* 223 */ "cmd ::= RESET QUERY CACHE", - /* 224 */ "cmd ::= EXPLAIN analyze_opt explain_options query_expression", - /* 225 */ "analyze_opt ::=", - /* 226 */ "analyze_opt ::= ANALYZE", - /* 227 */ "explain_options ::=", - /* 228 */ "explain_options ::= explain_options VERBOSE NK_BOOL", - /* 229 */ "explain_options ::= explain_options RATIO NK_FLOAT", - /* 230 */ "cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP", - /* 231 */ "cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt", - /* 232 */ "cmd ::= DROP FUNCTION function_name", - /* 233 */ "agg_func_opt ::=", - /* 234 */ "agg_func_opt ::= AGGREGATE", - /* 235 */ "bufsize_opt ::=", - /* 236 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", - /* 237 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression", - /* 238 */ "cmd ::= DROP STREAM exists_opt stream_name", - /* 239 */ "into_opt ::=", - /* 240 */ "into_opt ::= INTO full_table_name", - /* 241 */ "stream_options ::=", - /* 242 */ "stream_options ::= stream_options TRIGGER AT_ONCE", - /* 243 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", - /* 244 */ "stream_options ::= stream_options WATERMARK duration_literal", - /* 245 */ "cmd ::= KILL CONNECTION NK_INTEGER", - /* 246 */ "cmd ::= KILL QUERY NK_INTEGER", - /* 247 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", - /* 248 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", - /* 249 */ "cmd ::= SPLIT VGROUP NK_INTEGER", - /* 250 */ "dnode_list ::= DNODE NK_INTEGER", - /* 251 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", - /* 252 */ "cmd ::= SYNCDB db_name REPLICA", - /* 253 */ "cmd ::= query_expression", - /* 254 */ "literal ::= NK_INTEGER", - /* 255 */ "literal ::= NK_FLOAT", - /* 256 */ "literal ::= NK_STRING", - /* 257 */ "literal ::= NK_BOOL", - /* 258 */ "literal ::= TIMESTAMP NK_STRING", - /* 259 */ "literal ::= duration_literal", - /* 260 */ "literal ::= NULL", - /* 261 */ "literal ::= NK_QUESTION", - /* 262 */ "duration_literal ::= NK_VARIABLE", - /* 263 */ "signed ::= NK_INTEGER", - /* 264 */ "signed ::= NK_PLUS NK_INTEGER", - /* 265 */ "signed ::= NK_MINUS NK_INTEGER", - /* 266 */ "signed ::= NK_FLOAT", - /* 267 */ "signed ::= NK_PLUS NK_FLOAT", - /* 268 */ "signed ::= NK_MINUS NK_FLOAT", - /* 269 */ "signed_literal ::= signed", - /* 270 */ "signed_literal ::= NK_STRING", - /* 271 */ "signed_literal ::= NK_BOOL", - /* 272 */ "signed_literal ::= TIMESTAMP NK_STRING", - /* 273 */ "signed_literal ::= duration_literal", - /* 274 */ "signed_literal ::= NULL", - /* 275 */ "literal_list ::= signed_literal", - /* 276 */ "literal_list ::= literal_list NK_COMMA signed_literal", - /* 277 */ "db_name ::= NK_ID", - /* 278 */ "table_name ::= NK_ID", - /* 279 */ "column_name ::= NK_ID", - /* 280 */ "function_name ::= NK_ID", - /* 281 */ "table_alias ::= NK_ID", - /* 282 */ "column_alias ::= NK_ID", - /* 283 */ "user_name ::= NK_ID", - /* 284 */ "index_name ::= NK_ID", - /* 285 */ "topic_name ::= NK_ID", - /* 286 */ "stream_name ::= NK_ID", - /* 287 */ "expression ::= literal", - /* 288 */ "expression ::= pseudo_column", - /* 289 */ "expression ::= column_reference", - /* 290 */ "expression ::= function_expression", - /* 291 */ "expression ::= subquery", - /* 292 */ "expression ::= NK_LP expression NK_RP", - /* 293 */ "expression ::= NK_PLUS expression", - /* 294 */ "expression ::= NK_MINUS expression", - /* 295 */ "expression ::= expression NK_PLUS expression", - /* 296 */ "expression ::= expression NK_MINUS expression", - /* 297 */ "expression ::= expression NK_STAR expression", - /* 298 */ "expression ::= expression NK_SLASH expression", - /* 299 */ "expression ::= expression NK_REM expression", - /* 300 */ "expression ::= column_reference NK_ARROW NK_STRING", - /* 301 */ "expression_list ::= expression", - /* 302 */ "expression_list ::= expression_list NK_COMMA expression", - /* 303 */ "column_reference ::= column_name", - /* 304 */ "column_reference ::= table_name NK_DOT column_name", - /* 305 */ "pseudo_column ::= ROWTS", - /* 306 */ "pseudo_column ::= TBNAME", - /* 307 */ "pseudo_column ::= QSTARTTS", - /* 308 */ "pseudo_column ::= QENDTS", - /* 309 */ "pseudo_column ::= WSTARTTS", - /* 310 */ "pseudo_column ::= WENDTS", - /* 311 */ "pseudo_column ::= WDURATION", - /* 312 */ "function_expression ::= function_name NK_LP expression_list NK_RP", - /* 313 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", - /* 314 */ "function_expression ::= CAST NK_LP expression AS type_name NK_RP", - /* 315 */ "function_expression ::= noarg_func NK_LP NK_RP", - /* 316 */ "noarg_func ::= NOW", - /* 317 */ "noarg_func ::= TODAY", - /* 318 */ "noarg_func ::= TIMEZONE", - /* 319 */ "star_func ::= COUNT", - /* 320 */ "star_func ::= FIRST", - /* 321 */ "star_func ::= LAST", - /* 322 */ "star_func ::= LAST_ROW", - /* 323 */ "star_func_para_list ::= NK_STAR", - /* 324 */ "star_func_para_list ::= other_para_list", - /* 325 */ "other_para_list ::= star_func_para", - /* 326 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", - /* 327 */ "star_func_para ::= expression", - /* 328 */ "star_func_para ::= table_name NK_DOT NK_STAR", - /* 329 */ "predicate ::= expression compare_op expression", - /* 330 */ "predicate ::= expression BETWEEN expression AND expression", - /* 331 */ "predicate ::= expression NOT BETWEEN expression AND expression", - /* 332 */ "predicate ::= expression IS NULL", - /* 333 */ "predicate ::= expression IS NOT NULL", - /* 334 */ "predicate ::= expression in_op in_predicate_value", - /* 335 */ "compare_op ::= NK_LT", - /* 336 */ "compare_op ::= NK_GT", - /* 337 */ "compare_op ::= NK_LE", - /* 338 */ "compare_op ::= NK_GE", - /* 339 */ "compare_op ::= NK_NE", - /* 340 */ "compare_op ::= NK_EQ", - /* 341 */ "compare_op ::= LIKE", - /* 342 */ "compare_op ::= NOT LIKE", - /* 343 */ "compare_op ::= MATCH", - /* 344 */ "compare_op ::= NMATCH", - /* 345 */ "compare_op ::= CONTAINS", - /* 346 */ "in_op ::= IN", - /* 347 */ "in_op ::= NOT IN", - /* 348 */ "in_predicate_value ::= NK_LP expression_list NK_RP", - /* 349 */ "boolean_value_expression ::= boolean_primary", - /* 350 */ "boolean_value_expression ::= NOT boolean_primary", - /* 351 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", - /* 352 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", - /* 353 */ "boolean_primary ::= predicate", - /* 354 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 355 */ "common_expression ::= expression", - /* 356 */ "common_expression ::= boolean_value_expression", - /* 357 */ "from_clause ::= FROM table_reference_list", - /* 358 */ "table_reference_list ::= table_reference", - /* 359 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 360 */ "table_reference ::= table_primary", - /* 361 */ "table_reference ::= joined_table", - /* 362 */ "table_primary ::= table_name alias_opt", - /* 363 */ "table_primary ::= db_name NK_DOT table_name alias_opt", - /* 364 */ "table_primary ::= subquery alias_opt", - /* 365 */ "table_primary ::= parenthesized_joined_table", - /* 366 */ "alias_opt ::=", - /* 367 */ "alias_opt ::= table_alias", - /* 368 */ "alias_opt ::= AS table_alias", - /* 369 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 370 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 371 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 372 */ "join_type ::=", - /* 373 */ "join_type ::= INNER", - /* 374 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 375 */ "set_quantifier_opt ::=", - /* 376 */ "set_quantifier_opt ::= DISTINCT", - /* 377 */ "set_quantifier_opt ::= ALL", - /* 378 */ "select_list ::= NK_STAR", - /* 379 */ "select_list ::= select_sublist", - /* 380 */ "select_sublist ::= select_item", - /* 381 */ "select_sublist ::= select_sublist NK_COMMA select_item", - /* 382 */ "select_item ::= common_expression", - /* 383 */ "select_item ::= common_expression column_alias", - /* 384 */ "select_item ::= common_expression AS column_alias", - /* 385 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 386 */ "where_clause_opt ::=", - /* 387 */ "where_clause_opt ::= WHERE search_condition", - /* 388 */ "partition_by_clause_opt ::=", - /* 389 */ "partition_by_clause_opt ::= PARTITION BY expression_list", - /* 390 */ "twindow_clause_opt ::=", - /* 391 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", - /* 392 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP", - /* 393 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 394 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 395 */ "sliding_opt ::=", - /* 396 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 397 */ "fill_opt ::=", - /* 398 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 399 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 400 */ "fill_mode ::= NONE", - /* 401 */ "fill_mode ::= PREV", - /* 402 */ "fill_mode ::= NULL", - /* 403 */ "fill_mode ::= LINEAR", - /* 404 */ "fill_mode ::= NEXT", - /* 405 */ "group_by_clause_opt ::=", - /* 406 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 407 */ "group_by_list ::= expression", - /* 408 */ "group_by_list ::= group_by_list NK_COMMA expression", - /* 409 */ "having_clause_opt ::=", - /* 410 */ "having_clause_opt ::= HAVING search_condition", - /* 411 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 412 */ "query_expression_body ::= query_primary", - /* 413 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", - /* 414 */ "query_expression_body ::= query_expression_body UNION query_expression_body", - /* 415 */ "query_primary ::= query_specification", - /* 416 */ "order_by_clause_opt ::=", - /* 417 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 418 */ "slimit_clause_opt ::=", - /* 419 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 420 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 421 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 422 */ "limit_clause_opt ::=", - /* 423 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 424 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 425 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 426 */ "subquery ::= NK_LP query_expression NK_RP", - /* 427 */ "search_condition ::= common_expression", - /* 428 */ "sort_specification_list ::= sort_specification", - /* 429 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 430 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", - /* 431 */ "ordering_specification_opt ::=", - /* 432 */ "ordering_specification_opt ::= ASC", - /* 433 */ "ordering_specification_opt ::= DESC", - /* 434 */ "null_ordering_opt ::=", - /* 435 */ "null_ordering_opt ::= NULLS FIRST", - /* 436 */ "null_ordering_opt ::= NULLS LAST", + /* 221 */ "topic_options ::=", + /* 222 */ "topic_options ::= topic_options WITH TABLE", + /* 223 */ "topic_options ::= topic_options WITH SCHEMA", + /* 224 */ "topic_options ::= topic_options WITH TAG", + /* 225 */ "cmd ::= DESC full_table_name", + /* 226 */ "cmd ::= DESCRIBE full_table_name", + /* 227 */ "cmd ::= RESET QUERY CACHE", + /* 228 */ "cmd ::= EXPLAIN analyze_opt explain_options query_expression", + /* 229 */ "analyze_opt ::=", + /* 230 */ "analyze_opt ::= ANALYZE", + /* 231 */ "explain_options ::=", + /* 232 */ "explain_options ::= explain_options VERBOSE NK_BOOL", + /* 233 */ "explain_options ::= explain_options RATIO NK_FLOAT", + /* 234 */ "cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP", + /* 235 */ "cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt", + /* 236 */ "cmd ::= DROP FUNCTION function_name", + /* 237 */ "agg_func_opt ::=", + /* 238 */ "agg_func_opt ::= AGGREGATE", + /* 239 */ "bufsize_opt ::=", + /* 240 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", + /* 241 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression", + /* 242 */ "cmd ::= DROP STREAM exists_opt stream_name", + /* 243 */ "into_opt ::=", + /* 244 */ "into_opt ::= INTO full_table_name", + /* 245 */ "stream_options ::=", + /* 246 */ "stream_options ::= stream_options TRIGGER AT_ONCE", + /* 247 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", + /* 248 */ "stream_options ::= stream_options WATERMARK duration_literal", + /* 249 */ "cmd ::= KILL CONNECTION NK_INTEGER", + /* 250 */ "cmd ::= KILL QUERY NK_INTEGER", + /* 251 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", + /* 252 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", + /* 253 */ "cmd ::= SPLIT VGROUP NK_INTEGER", + /* 254 */ "dnode_list ::= DNODE NK_INTEGER", + /* 255 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", + /* 256 */ "cmd ::= SYNCDB db_name REPLICA", + /* 257 */ "cmd ::= query_expression", + /* 258 */ "literal ::= NK_INTEGER", + /* 259 */ "literal ::= NK_FLOAT", + /* 260 */ "literal ::= NK_STRING", + /* 261 */ "literal ::= NK_BOOL", + /* 262 */ "literal ::= TIMESTAMP NK_STRING", + /* 263 */ "literal ::= duration_literal", + /* 264 */ "literal ::= NULL", + /* 265 */ "literal ::= NK_QUESTION", + /* 266 */ "duration_literal ::= NK_VARIABLE", + /* 267 */ "signed ::= NK_INTEGER", + /* 268 */ "signed ::= NK_PLUS NK_INTEGER", + /* 269 */ "signed ::= NK_MINUS NK_INTEGER", + /* 270 */ "signed ::= NK_FLOAT", + /* 271 */ "signed ::= NK_PLUS NK_FLOAT", + /* 272 */ "signed ::= NK_MINUS NK_FLOAT", + /* 273 */ "signed_literal ::= signed", + /* 274 */ "signed_literal ::= NK_STRING", + /* 275 */ "signed_literal ::= NK_BOOL", + /* 276 */ "signed_literal ::= TIMESTAMP NK_STRING", + /* 277 */ "signed_literal ::= duration_literal", + /* 278 */ "signed_literal ::= NULL", + /* 279 */ "literal_list ::= signed_literal", + /* 280 */ "literal_list ::= literal_list NK_COMMA signed_literal", + /* 281 */ "db_name ::= NK_ID", + /* 282 */ "table_name ::= NK_ID", + /* 283 */ "column_name ::= NK_ID", + /* 284 */ "function_name ::= NK_ID", + /* 285 */ "table_alias ::= NK_ID", + /* 286 */ "column_alias ::= NK_ID", + /* 287 */ "user_name ::= NK_ID", + /* 288 */ "index_name ::= NK_ID", + /* 289 */ "topic_name ::= NK_ID", + /* 290 */ "stream_name ::= NK_ID", + /* 291 */ "expression ::= literal", + /* 292 */ "expression ::= pseudo_column", + /* 293 */ "expression ::= column_reference", + /* 294 */ "expression ::= function_expression", + /* 295 */ "expression ::= subquery", + /* 296 */ "expression ::= NK_LP expression NK_RP", + /* 297 */ "expression ::= NK_PLUS expression", + /* 298 */ "expression ::= NK_MINUS expression", + /* 299 */ "expression ::= expression NK_PLUS expression", + /* 300 */ "expression ::= expression NK_MINUS expression", + /* 301 */ "expression ::= expression NK_STAR expression", + /* 302 */ "expression ::= expression NK_SLASH expression", + /* 303 */ "expression ::= expression NK_REM expression", + /* 304 */ "expression ::= column_reference NK_ARROW NK_STRING", + /* 305 */ "expression_list ::= expression", + /* 306 */ "expression_list ::= expression_list NK_COMMA expression", + /* 307 */ "column_reference ::= column_name", + /* 308 */ "column_reference ::= table_name NK_DOT column_name", + /* 309 */ "pseudo_column ::= ROWTS", + /* 310 */ "pseudo_column ::= TBNAME", + /* 311 */ "pseudo_column ::= QSTARTTS", + /* 312 */ "pseudo_column ::= QENDTS", + /* 313 */ "pseudo_column ::= WSTARTTS", + /* 314 */ "pseudo_column ::= WENDTS", + /* 315 */ "pseudo_column ::= WDURATION", + /* 316 */ "function_expression ::= function_name NK_LP expression_list NK_RP", + /* 317 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", + /* 318 */ "function_expression ::= CAST NK_LP expression AS type_name NK_RP", + /* 319 */ "function_expression ::= noarg_func NK_LP NK_RP", + /* 320 */ "noarg_func ::= NOW", + /* 321 */ "noarg_func ::= TODAY", + /* 322 */ "noarg_func ::= TIMEZONE", + /* 323 */ "star_func ::= COUNT", + /* 324 */ "star_func ::= FIRST", + /* 325 */ "star_func ::= LAST", + /* 326 */ "star_func ::= LAST_ROW", + /* 327 */ "star_func_para_list ::= NK_STAR", + /* 328 */ "star_func_para_list ::= other_para_list", + /* 329 */ "other_para_list ::= star_func_para", + /* 330 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", + /* 331 */ "star_func_para ::= expression", + /* 332 */ "star_func_para ::= table_name NK_DOT NK_STAR", + /* 333 */ "predicate ::= expression compare_op expression", + /* 334 */ "predicate ::= expression BETWEEN expression AND expression", + /* 335 */ "predicate ::= expression NOT BETWEEN expression AND expression", + /* 336 */ "predicate ::= expression IS NULL", + /* 337 */ "predicate ::= expression IS NOT NULL", + /* 338 */ "predicate ::= expression in_op in_predicate_value", + /* 339 */ "compare_op ::= NK_LT", + /* 340 */ "compare_op ::= NK_GT", + /* 341 */ "compare_op ::= NK_LE", + /* 342 */ "compare_op ::= NK_GE", + /* 343 */ "compare_op ::= NK_NE", + /* 344 */ "compare_op ::= NK_EQ", + /* 345 */ "compare_op ::= LIKE", + /* 346 */ "compare_op ::= NOT LIKE", + /* 347 */ "compare_op ::= MATCH", + /* 348 */ "compare_op ::= NMATCH", + /* 349 */ "compare_op ::= CONTAINS", + /* 350 */ "in_op ::= IN", + /* 351 */ "in_op ::= NOT IN", + /* 352 */ "in_predicate_value ::= NK_LP expression_list NK_RP", + /* 353 */ "boolean_value_expression ::= boolean_primary", + /* 354 */ "boolean_value_expression ::= NOT boolean_primary", + /* 355 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 356 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 357 */ "boolean_primary ::= predicate", + /* 358 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 359 */ "common_expression ::= expression", + /* 360 */ "common_expression ::= boolean_value_expression", + /* 361 */ "from_clause ::= FROM table_reference_list", + /* 362 */ "table_reference_list ::= table_reference", + /* 363 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 364 */ "table_reference ::= table_primary", + /* 365 */ "table_reference ::= joined_table", + /* 366 */ "table_primary ::= table_name alias_opt", + /* 367 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 368 */ "table_primary ::= subquery alias_opt", + /* 369 */ "table_primary ::= parenthesized_joined_table", + /* 370 */ "alias_opt ::=", + /* 371 */ "alias_opt ::= table_alias", + /* 372 */ "alias_opt ::= AS table_alias", + /* 373 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 374 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 375 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 376 */ "join_type ::=", + /* 377 */ "join_type ::= INNER", + /* 378 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 379 */ "set_quantifier_opt ::=", + /* 380 */ "set_quantifier_opt ::= DISTINCT", + /* 381 */ "set_quantifier_opt ::= ALL", + /* 382 */ "select_list ::= NK_STAR", + /* 383 */ "select_list ::= select_sublist", + /* 384 */ "select_sublist ::= select_item", + /* 385 */ "select_sublist ::= select_sublist NK_COMMA select_item", + /* 386 */ "select_item ::= common_expression", + /* 387 */ "select_item ::= common_expression column_alias", + /* 388 */ "select_item ::= common_expression AS column_alias", + /* 389 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 390 */ "where_clause_opt ::=", + /* 391 */ "where_clause_opt ::= WHERE search_condition", + /* 392 */ "partition_by_clause_opt ::=", + /* 393 */ "partition_by_clause_opt ::= PARTITION BY expression_list", + /* 394 */ "twindow_clause_opt ::=", + /* 395 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", + /* 396 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP", + /* 397 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 398 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 399 */ "sliding_opt ::=", + /* 400 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 401 */ "fill_opt ::=", + /* 402 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 403 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 404 */ "fill_mode ::= NONE", + /* 405 */ "fill_mode ::= PREV", + /* 406 */ "fill_mode ::= NULL", + /* 407 */ "fill_mode ::= LINEAR", + /* 408 */ "fill_mode ::= NEXT", + /* 409 */ "group_by_clause_opt ::=", + /* 410 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 411 */ "group_by_list ::= expression", + /* 412 */ "group_by_list ::= group_by_list NK_COMMA expression", + /* 413 */ "having_clause_opt ::=", + /* 414 */ "having_clause_opt ::= HAVING search_condition", + /* 415 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 416 */ "query_expression_body ::= query_primary", + /* 417 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", + /* 418 */ "query_expression_body ::= query_expression_body UNION query_expression_body", + /* 419 */ "query_primary ::= query_specification", + /* 420 */ "order_by_clause_opt ::=", + /* 421 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 422 */ "slimit_clause_opt ::=", + /* 423 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 424 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 425 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 426 */ "limit_clause_opt ::=", + /* 427 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 428 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 429 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 430 */ "subquery ::= NK_LP query_expression NK_RP", + /* 431 */ "search_condition ::= common_expression", + /* 432 */ "sort_specification_list ::= sort_specification", + /* 433 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 434 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", + /* 435 */ "ordering_specification_opt ::=", + /* 436 */ "ordering_specification_opt ::= ASC", + /* 437 */ "ordering_specification_opt ::= DESC", + /* 438 */ "null_ordering_opt ::=", + /* 439 */ "null_ordering_opt ::= NULLS FIRST", + /* 440 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -1768,164 +2005,165 @@ static void yy_destructor( */ /********* Begin destructor definitions ***************************************/ /* Default NON-TERMINAL Destructor */ - case 222: /* cmd */ - case 225: /* literal */ - case 232: /* db_options */ - case 234: /* alter_db_options */ - case 239: /* retention */ - case 240: /* full_table_name */ - case 243: /* table_options */ - case 247: /* alter_table_clause */ - case 248: /* alter_table_options */ - case 251: /* create_subtable_clause */ - case 254: /* drop_table_clause */ - case 257: /* column_def */ - case 260: /* col_name */ - case 261: /* db_name_cond_opt */ - case 262: /* like_pattern_opt */ - case 263: /* table_name_cond */ - case 264: /* from_db_opt */ - case 265: /* func_name */ - case 268: /* index_options */ - case 270: /* duration_literal */ - case 271: /* sliding_opt */ - case 272: /* func */ - case 275: /* query_expression */ - case 277: /* explain_options */ - case 281: /* stream_options */ - case 282: /* into_opt */ - case 284: /* signed */ - case 285: /* signed_literal */ - case 288: /* expression */ - case 289: /* pseudo_column */ - case 290: /* column_reference */ - case 291: /* function_expression */ - case 292: /* subquery */ - case 297: /* star_func_para */ - case 298: /* predicate */ - case 301: /* in_predicate_value */ - case 302: /* boolean_value_expression */ - case 303: /* boolean_primary */ - case 304: /* common_expression */ - case 305: /* from_clause */ - case 306: /* table_reference_list */ - case 307: /* table_reference */ - case 308: /* table_primary */ - case 309: /* joined_table */ - case 311: /* parenthesized_joined_table */ - case 313: /* search_condition */ - case 314: /* query_specification */ - case 317: /* where_clause_opt */ - case 319: /* twindow_clause_opt */ - case 321: /* having_clause_opt */ - case 323: /* select_item */ - case 324: /* fill_opt */ - case 327: /* query_expression_body */ - case 329: /* slimit_clause_opt */ - case 330: /* limit_clause_opt */ - case 331: /* query_primary */ - case 333: /* sort_specification */ + case 231: /* cmd */ + case 234: /* literal */ + case 241: /* db_options */ + case 243: /* alter_db_options */ + case 248: /* retention */ + case 249: /* full_table_name */ + case 252: /* table_options */ + case 256: /* alter_table_clause */ + case 257: /* alter_table_options */ + case 260: /* create_subtable_clause */ + case 263: /* drop_table_clause */ + case 266: /* column_def */ + case 269: /* col_name */ + case 270: /* db_name_cond_opt */ + case 271: /* like_pattern_opt */ + case 272: /* table_name_cond */ + case 273: /* from_db_opt */ + case 274: /* func_name */ + case 277: /* index_options */ + case 279: /* duration_literal */ + case 280: /* sliding_opt */ + case 281: /* func */ + case 284: /* topic_options */ + case 285: /* query_expression */ + case 287: /* explain_options */ + case 291: /* stream_options */ + case 292: /* into_opt */ + case 294: /* signed */ + case 295: /* signed_literal */ + case 298: /* expression */ + case 299: /* pseudo_column */ + case 300: /* column_reference */ + case 301: /* function_expression */ + case 302: /* subquery */ + case 307: /* star_func_para */ + case 308: /* predicate */ + case 311: /* in_predicate_value */ + case 312: /* boolean_value_expression */ + case 313: /* boolean_primary */ + case 314: /* common_expression */ + case 315: /* from_clause */ + case 316: /* table_reference_list */ + case 317: /* table_reference */ + case 318: /* table_primary */ + case 319: /* joined_table */ + case 321: /* parenthesized_joined_table */ + case 323: /* search_condition */ + case 324: /* query_specification */ + case 327: /* where_clause_opt */ + case 329: /* twindow_clause_opt */ + case 331: /* having_clause_opt */ + case 333: /* select_item */ + case 334: /* fill_opt */ + case 337: /* query_expression_body */ + case 339: /* slimit_clause_opt */ + case 340: /* limit_clause_opt */ + case 341: /* query_primary */ + case 343: /* sort_specification */ { - nodesDestroyNode((yypminor->yy392)); + nodesDestroyNode((yypminor->yy456)); } break; - case 223: /* account_options */ - case 224: /* alter_account_options */ - case 226: /* alter_account_option */ - case 279: /* bufsize_opt */ + case 232: /* account_options */ + case 233: /* alter_account_options */ + case 235: /* alter_account_option */ + case 289: /* bufsize_opt */ { } break; - case 227: /* user_name */ - case 228: /* dnode_endpoint */ - case 229: /* dnode_host_name */ - case 231: /* db_name */ - case 249: /* column_name */ - case 256: /* table_name */ - case 266: /* function_name */ - case 267: /* index_name */ - case 274: /* topic_name */ - case 280: /* stream_name */ - case 286: /* table_alias */ - case 287: /* column_alias */ - case 293: /* star_func */ - case 295: /* noarg_func */ - case 310: /* alias_opt */ + case 236: /* user_name */ + case 237: /* dnode_endpoint */ + case 238: /* dnode_host_name */ + case 240: /* db_name */ + case 258: /* column_name */ + case 265: /* table_name */ + case 275: /* function_name */ + case 276: /* index_name */ + case 283: /* topic_name */ + case 290: /* stream_name */ + case 296: /* table_alias */ + case 297: /* column_alias */ + case 303: /* star_func */ + case 305: /* noarg_func */ + case 320: /* alias_opt */ { } break; - case 230: /* not_exists_opt */ - case 233: /* exists_opt */ - case 276: /* analyze_opt */ - case 278: /* agg_func_opt */ - case 315: /* set_quantifier_opt */ + case 239: /* not_exists_opt */ + case 242: /* exists_opt */ + case 286: /* analyze_opt */ + case 288: /* agg_func_opt */ + case 325: /* set_quantifier_opt */ { } break; - case 235: /* integer_list */ - case 236: /* variable_list */ - case 237: /* retention_list */ - case 241: /* column_def_list */ - case 242: /* tags_def_opt */ - case 244: /* multi_create_clause */ - case 245: /* tags_def */ - case 246: /* multi_drop_clause */ - case 252: /* specific_tags_opt */ - case 253: /* literal_list */ - case 255: /* col_name_list */ - case 258: /* func_name_list */ - case 269: /* func_list */ - case 273: /* expression_list */ - case 283: /* dnode_list */ - case 294: /* star_func_para_list */ - case 296: /* other_para_list */ - case 316: /* select_list */ - case 318: /* partition_by_clause_opt */ - case 320: /* group_by_clause_opt */ - case 322: /* select_sublist */ - case 326: /* group_by_list */ - case 328: /* order_by_clause_opt */ - case 332: /* sort_specification_list */ + case 244: /* integer_list */ + case 245: /* variable_list */ + case 246: /* retention_list */ + case 250: /* column_def_list */ + case 251: /* tags_def_opt */ + case 253: /* multi_create_clause */ + case 254: /* tags_def */ + case 255: /* multi_drop_clause */ + case 261: /* specific_tags_opt */ + case 262: /* literal_list */ + case 264: /* col_name_list */ + case 267: /* func_name_list */ + case 278: /* func_list */ + case 282: /* expression_list */ + case 293: /* dnode_list */ + case 304: /* star_func_para_list */ + case 306: /* other_para_list */ + case 326: /* select_list */ + case 328: /* partition_by_clause_opt */ + case 330: /* group_by_clause_opt */ + case 332: /* select_sublist */ + case 336: /* group_by_list */ + case 338: /* order_by_clause_opt */ + case 342: /* sort_specification_list */ { - nodesDestroyList((yypminor->yy376)); + nodesDestroyList((yypminor->yy652)); } break; - case 238: /* alter_db_option */ - case 259: /* alter_table_option */ + case 247: /* alter_db_option */ + case 268: /* alter_table_option */ { } break; - case 250: /* type_name */ + case 259: /* type_name */ { } break; - case 299: /* compare_op */ - case 300: /* in_op */ + case 309: /* compare_op */ + case 310: /* in_op */ { } break; - case 312: /* join_type */ + case 322: /* join_type */ { } break; - case 325: /* fill_mode */ + case 335: /* fill_mode */ { } break; - case 334: /* ordering_specification_opt */ + case 344: /* ordering_specification_opt */ { } break; - case 335: /* null_ordering_opt */ + case 345: /* null_ordering_opt */ { } @@ -2224,443 +2462,447 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ signed char nrhs; /* Negative of the number of RHS symbols in the rule */ } yyRuleInfo[] = { - { 222, -6 }, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ - { 222, -4 }, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ - { 223, 0 }, /* (2) account_options ::= */ - { 223, -3 }, /* (3) account_options ::= account_options PPS literal */ - { 223, -3 }, /* (4) account_options ::= account_options TSERIES literal */ - { 223, -3 }, /* (5) account_options ::= account_options STORAGE literal */ - { 223, -3 }, /* (6) account_options ::= account_options STREAMS literal */ - { 223, -3 }, /* (7) account_options ::= account_options QTIME literal */ - { 223, -3 }, /* (8) account_options ::= account_options DBS literal */ - { 223, -3 }, /* (9) account_options ::= account_options USERS literal */ - { 223, -3 }, /* (10) account_options ::= account_options CONNS literal */ - { 223, -3 }, /* (11) account_options ::= account_options STATE literal */ - { 224, -1 }, /* (12) alter_account_options ::= alter_account_option */ - { 224, -2 }, /* (13) alter_account_options ::= alter_account_options alter_account_option */ - { 226, -2 }, /* (14) alter_account_option ::= PASS literal */ - { 226, -2 }, /* (15) alter_account_option ::= PPS literal */ - { 226, -2 }, /* (16) alter_account_option ::= TSERIES literal */ - { 226, -2 }, /* (17) alter_account_option ::= STORAGE literal */ - { 226, -2 }, /* (18) alter_account_option ::= STREAMS literal */ - { 226, -2 }, /* (19) alter_account_option ::= QTIME literal */ - { 226, -2 }, /* (20) alter_account_option ::= DBS literal */ - { 226, -2 }, /* (21) alter_account_option ::= USERS literal */ - { 226, -2 }, /* (22) alter_account_option ::= CONNS literal */ - { 226, -2 }, /* (23) alter_account_option ::= STATE literal */ - { 222, -5 }, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING */ - { 222, -5 }, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ - { 222, -5 }, /* (26) cmd ::= ALTER USER user_name PRIVILEGE NK_STRING */ - { 222, -3 }, /* (27) cmd ::= DROP USER user_name */ - { 222, -3 }, /* (28) cmd ::= CREATE DNODE dnode_endpoint */ - { 222, -5 }, /* (29) cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER */ - { 222, -3 }, /* (30) cmd ::= DROP DNODE NK_INTEGER */ - { 222, -3 }, /* (31) cmd ::= DROP DNODE dnode_endpoint */ - { 222, -4 }, /* (32) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ - { 222, -5 }, /* (33) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ - { 222, -4 }, /* (34) cmd ::= ALTER ALL DNODES NK_STRING */ - { 222, -5 }, /* (35) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ - { 228, -1 }, /* (36) dnode_endpoint ::= NK_STRING */ - { 229, -1 }, /* (37) dnode_host_name ::= NK_ID */ - { 229, -1 }, /* (38) dnode_host_name ::= NK_IPTOKEN */ - { 222, -3 }, /* (39) cmd ::= ALTER LOCAL NK_STRING */ - { 222, -4 }, /* (40) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ - { 222, -5 }, /* (41) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ - { 222, -5 }, /* (42) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ - { 222, -5 }, /* (43) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ - { 222, -5 }, /* (44) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ - { 222, -5 }, /* (45) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ - { 222, -5 }, /* (46) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ - { 222, -5 }, /* (47) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ - { 222, -5 }, /* (48) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ - { 222, -5 }, /* (49) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ - { 222, -4 }, /* (50) cmd ::= DROP DATABASE exists_opt db_name */ - { 222, -2 }, /* (51) cmd ::= USE db_name */ - { 222, -4 }, /* (52) cmd ::= ALTER DATABASE db_name alter_db_options */ - { 230, -3 }, /* (53) not_exists_opt ::= IF NOT EXISTS */ - { 230, 0 }, /* (54) not_exists_opt ::= */ - { 233, -2 }, /* (55) exists_opt ::= IF EXISTS */ - { 233, 0 }, /* (56) exists_opt ::= */ - { 232, 0 }, /* (57) db_options ::= */ - { 232, -3 }, /* (58) db_options ::= db_options BLOCKS NK_INTEGER */ - { 232, -3 }, /* (59) db_options ::= db_options CACHE NK_INTEGER */ - { 232, -3 }, /* (60) db_options ::= db_options CACHELAST NK_INTEGER */ - { 232, -3 }, /* (61) db_options ::= db_options COMP NK_INTEGER */ - { 232, -3 }, /* (62) db_options ::= db_options DAYS NK_INTEGER */ - { 232, -3 }, /* (63) db_options ::= db_options DAYS NK_VARIABLE */ - { 232, -3 }, /* (64) db_options ::= db_options FSYNC NK_INTEGER */ - { 232, -3 }, /* (65) db_options ::= db_options MAXROWS NK_INTEGER */ - { 232, -3 }, /* (66) db_options ::= db_options MINROWS NK_INTEGER */ - { 232, -3 }, /* (67) db_options ::= db_options KEEP integer_list */ - { 232, -3 }, /* (68) db_options ::= db_options KEEP variable_list */ - { 232, -3 }, /* (69) db_options ::= db_options PRECISION NK_STRING */ - { 232, -3 }, /* (70) db_options ::= db_options QUORUM NK_INTEGER */ - { 232, -3 }, /* (71) db_options ::= db_options REPLICA NK_INTEGER */ - { 232, -3 }, /* (72) db_options ::= db_options TTL NK_INTEGER */ - { 232, -3 }, /* (73) db_options ::= db_options WAL NK_INTEGER */ - { 232, -3 }, /* (74) db_options ::= db_options VGROUPS NK_INTEGER */ - { 232, -3 }, /* (75) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ - { 232, -3 }, /* (76) db_options ::= db_options STREAM_MODE NK_INTEGER */ - { 232, -3 }, /* (77) db_options ::= db_options RETENTIONS retention_list */ - { 232, -3 }, /* (78) db_options ::= db_options STRICT NK_INTEGER */ - { 234, -1 }, /* (79) alter_db_options ::= alter_db_option */ - { 234, -2 }, /* (80) alter_db_options ::= alter_db_options alter_db_option */ - { 238, -2 }, /* (81) alter_db_option ::= BLOCKS NK_INTEGER */ - { 238, -2 }, /* (82) alter_db_option ::= FSYNC NK_INTEGER */ - { 238, -2 }, /* (83) alter_db_option ::= KEEP integer_list */ - { 238, -2 }, /* (84) alter_db_option ::= KEEP variable_list */ - { 238, -2 }, /* (85) alter_db_option ::= WAL NK_INTEGER */ - { 238, -2 }, /* (86) alter_db_option ::= QUORUM NK_INTEGER */ - { 238, -2 }, /* (87) alter_db_option ::= CACHELAST NK_INTEGER */ - { 238, -2 }, /* (88) alter_db_option ::= REPLICA NK_INTEGER */ - { 238, -2 }, /* (89) alter_db_option ::= STRICT NK_INTEGER */ - { 235, -1 }, /* (90) integer_list ::= NK_INTEGER */ - { 235, -3 }, /* (91) integer_list ::= integer_list NK_COMMA NK_INTEGER */ - { 236, -1 }, /* (92) variable_list ::= NK_VARIABLE */ - { 236, -3 }, /* (93) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ - { 237, -1 }, /* (94) retention_list ::= retention */ - { 237, -3 }, /* (95) retention_list ::= retention_list NK_COMMA retention */ - { 239, -3 }, /* (96) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ - { 222, -9 }, /* (97) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - { 222, -3 }, /* (98) cmd ::= CREATE TABLE multi_create_clause */ - { 222, -9 }, /* (99) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ - { 222, -3 }, /* (100) cmd ::= DROP TABLE multi_drop_clause */ - { 222, -4 }, /* (101) cmd ::= DROP STABLE exists_opt full_table_name */ - { 222, -3 }, /* (102) cmd ::= ALTER TABLE alter_table_clause */ - { 222, -3 }, /* (103) cmd ::= ALTER STABLE alter_table_clause */ - { 247, -2 }, /* (104) alter_table_clause ::= full_table_name alter_table_options */ - { 247, -5 }, /* (105) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ - { 247, -4 }, /* (106) alter_table_clause ::= full_table_name DROP COLUMN column_name */ - { 247, -5 }, /* (107) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ - { 247, -5 }, /* (108) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ - { 247, -5 }, /* (109) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ - { 247, -4 }, /* (110) alter_table_clause ::= full_table_name DROP TAG column_name */ - { 247, -5 }, /* (111) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ - { 247, -5 }, /* (112) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ - { 247, -6 }, /* (113) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal */ - { 244, -1 }, /* (114) multi_create_clause ::= create_subtable_clause */ - { 244, -2 }, /* (115) multi_create_clause ::= multi_create_clause create_subtable_clause */ - { 251, -9 }, /* (116) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP */ - { 246, -1 }, /* (117) multi_drop_clause ::= drop_table_clause */ - { 246, -2 }, /* (118) multi_drop_clause ::= multi_drop_clause drop_table_clause */ - { 254, -2 }, /* (119) drop_table_clause ::= exists_opt full_table_name */ - { 252, 0 }, /* (120) specific_tags_opt ::= */ - { 252, -3 }, /* (121) specific_tags_opt ::= NK_LP col_name_list NK_RP */ - { 240, -1 }, /* (122) full_table_name ::= table_name */ - { 240, -3 }, /* (123) full_table_name ::= db_name NK_DOT table_name */ - { 241, -1 }, /* (124) column_def_list ::= column_def */ - { 241, -3 }, /* (125) column_def_list ::= column_def_list NK_COMMA column_def */ - { 257, -2 }, /* (126) column_def ::= column_name type_name */ - { 257, -4 }, /* (127) column_def ::= column_name type_name COMMENT NK_STRING */ - { 250, -1 }, /* (128) type_name ::= BOOL */ - { 250, -1 }, /* (129) type_name ::= TINYINT */ - { 250, -1 }, /* (130) type_name ::= SMALLINT */ - { 250, -1 }, /* (131) type_name ::= INT */ - { 250, -1 }, /* (132) type_name ::= INTEGER */ - { 250, -1 }, /* (133) type_name ::= BIGINT */ - { 250, -1 }, /* (134) type_name ::= FLOAT */ - { 250, -1 }, /* (135) type_name ::= DOUBLE */ - { 250, -4 }, /* (136) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ - { 250, -1 }, /* (137) type_name ::= TIMESTAMP */ - { 250, -4 }, /* (138) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ - { 250, -2 }, /* (139) type_name ::= TINYINT UNSIGNED */ - { 250, -2 }, /* (140) type_name ::= SMALLINT UNSIGNED */ - { 250, -2 }, /* (141) type_name ::= INT UNSIGNED */ - { 250, -2 }, /* (142) type_name ::= BIGINT UNSIGNED */ - { 250, -1 }, /* (143) type_name ::= JSON */ - { 250, -4 }, /* (144) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ - { 250, -1 }, /* (145) type_name ::= MEDIUMBLOB */ - { 250, -1 }, /* (146) type_name ::= BLOB */ - { 250, -4 }, /* (147) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ - { 250, -1 }, /* (148) type_name ::= DECIMAL */ - { 250, -4 }, /* (149) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ - { 250, -6 }, /* (150) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ - { 242, 0 }, /* (151) tags_def_opt ::= */ - { 242, -1 }, /* (152) tags_def_opt ::= tags_def */ - { 245, -4 }, /* (153) tags_def ::= TAGS NK_LP column_def_list NK_RP */ - { 243, 0 }, /* (154) table_options ::= */ - { 243, -3 }, /* (155) table_options ::= table_options COMMENT NK_STRING */ - { 243, -3 }, /* (156) table_options ::= table_options KEEP integer_list */ - { 243, -3 }, /* (157) table_options ::= table_options KEEP variable_list */ - { 243, -3 }, /* (158) table_options ::= table_options TTL NK_INTEGER */ - { 243, -5 }, /* (159) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ - { 243, -5 }, /* (160) table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP */ - { 243, -3 }, /* (161) table_options ::= table_options FILE_FACTOR NK_FLOAT */ - { 243, -3 }, /* (162) table_options ::= table_options DELAY NK_INTEGER */ - { 248, -1 }, /* (163) alter_table_options ::= alter_table_option */ - { 248, -2 }, /* (164) alter_table_options ::= alter_table_options alter_table_option */ - { 259, -2 }, /* (165) alter_table_option ::= COMMENT NK_STRING */ - { 259, -2 }, /* (166) alter_table_option ::= KEEP integer_list */ - { 259, -2 }, /* (167) alter_table_option ::= KEEP variable_list */ - { 259, -2 }, /* (168) alter_table_option ::= TTL NK_INTEGER */ - { 255, -1 }, /* (169) col_name_list ::= col_name */ - { 255, -3 }, /* (170) col_name_list ::= col_name_list NK_COMMA col_name */ - { 260, -1 }, /* (171) col_name ::= column_name */ - { 222, -2 }, /* (172) cmd ::= SHOW DNODES */ - { 222, -2 }, /* (173) cmd ::= SHOW USERS */ - { 222, -2 }, /* (174) cmd ::= SHOW DATABASES */ - { 222, -4 }, /* (175) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ - { 222, -4 }, /* (176) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ - { 222, -3 }, /* (177) cmd ::= SHOW db_name_cond_opt VGROUPS */ - { 222, -2 }, /* (178) cmd ::= SHOW MNODES */ - { 222, -2 }, /* (179) cmd ::= SHOW MODULES */ - { 222, -2 }, /* (180) cmd ::= SHOW QNODES */ - { 222, -2 }, /* (181) cmd ::= SHOW FUNCTIONS */ - { 222, -5 }, /* (182) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ - { 222, -2 }, /* (183) cmd ::= SHOW STREAMS */ - { 222, -2 }, /* (184) cmd ::= SHOW ACCOUNTS */ - { 222, -2 }, /* (185) cmd ::= SHOW APPS */ - { 222, -2 }, /* (186) cmd ::= SHOW CONNECTIONS */ - { 222, -2 }, /* (187) cmd ::= SHOW LICENCE */ - { 222, -2 }, /* (188) cmd ::= SHOW GRANTS */ - { 222, -4 }, /* (189) cmd ::= SHOW CREATE DATABASE db_name */ - { 222, -4 }, /* (190) cmd ::= SHOW CREATE TABLE full_table_name */ - { 222, -4 }, /* (191) cmd ::= SHOW CREATE STABLE full_table_name */ - { 222, -2 }, /* (192) cmd ::= SHOW QUERIES */ - { 222, -2 }, /* (193) cmd ::= SHOW SCORES */ - { 222, -2 }, /* (194) cmd ::= SHOW TOPICS */ - { 222, -2 }, /* (195) cmd ::= SHOW VARIABLES */ - { 222, -2 }, /* (196) cmd ::= SHOW BNODES */ - { 222, -2 }, /* (197) cmd ::= SHOW SNODES */ - { 222, -2 }, /* (198) cmd ::= SHOW CLUSTER */ - { 261, 0 }, /* (199) db_name_cond_opt ::= */ - { 261, -2 }, /* (200) db_name_cond_opt ::= db_name NK_DOT */ - { 262, 0 }, /* (201) like_pattern_opt ::= */ - { 262, -2 }, /* (202) like_pattern_opt ::= LIKE NK_STRING */ - { 263, -1 }, /* (203) table_name_cond ::= table_name */ - { 264, 0 }, /* (204) from_db_opt ::= */ - { 264, -2 }, /* (205) from_db_opt ::= FROM db_name */ - { 258, -1 }, /* (206) func_name_list ::= func_name */ - { 258, -3 }, /* (207) func_name_list ::= func_name_list NK_COMMA func_name */ - { 265, -1 }, /* (208) func_name ::= function_name */ - { 222, -8 }, /* (209) cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options */ - { 222, -10 }, /* (210) cmd ::= CREATE FULLTEXT INDEX not_exists_opt index_name ON table_name NK_LP col_name_list NK_RP */ - { 222, -6 }, /* (211) cmd ::= DROP INDEX exists_opt index_name ON table_name */ - { 268, 0 }, /* (212) index_options ::= */ - { 268, -9 }, /* (213) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ - { 268, -11 }, /* (214) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ - { 269, -1 }, /* (215) func_list ::= func */ - { 269, -3 }, /* (216) func_list ::= func_list NK_COMMA func */ - { 272, -4 }, /* (217) func ::= function_name NK_LP expression_list NK_RP */ - { 222, -6 }, /* (218) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ - { 222, -6 }, /* (219) cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name */ - { 222, -4 }, /* (220) cmd ::= DROP TOPIC exists_opt topic_name */ - { 222, -2 }, /* (221) cmd ::= DESC full_table_name */ - { 222, -2 }, /* (222) cmd ::= DESCRIBE full_table_name */ - { 222, -3 }, /* (223) cmd ::= RESET QUERY CACHE */ - { 222, -4 }, /* (224) cmd ::= EXPLAIN analyze_opt explain_options query_expression */ - { 276, 0 }, /* (225) analyze_opt ::= */ - { 276, -1 }, /* (226) analyze_opt ::= ANALYZE */ - { 277, 0 }, /* (227) explain_options ::= */ - { 277, -3 }, /* (228) explain_options ::= explain_options VERBOSE NK_BOOL */ - { 277, -3 }, /* (229) explain_options ::= explain_options RATIO NK_FLOAT */ - { 222, -6 }, /* (230) cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP */ - { 222, -10 }, /* (231) cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ - { 222, -3 }, /* (232) cmd ::= DROP FUNCTION function_name */ - { 278, 0 }, /* (233) agg_func_opt ::= */ - { 278, -1 }, /* (234) agg_func_opt ::= AGGREGATE */ - { 279, 0 }, /* (235) bufsize_opt ::= */ - { 279, -2 }, /* (236) bufsize_opt ::= BUFSIZE NK_INTEGER */ - { 222, -8 }, /* (237) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression */ - { 222, -4 }, /* (238) cmd ::= DROP STREAM exists_opt stream_name */ - { 282, 0 }, /* (239) into_opt ::= */ - { 282, -2 }, /* (240) into_opt ::= INTO full_table_name */ - { 281, 0 }, /* (241) stream_options ::= */ - { 281, -3 }, /* (242) stream_options ::= stream_options TRIGGER AT_ONCE */ - { 281, -3 }, /* (243) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ - { 281, -3 }, /* (244) stream_options ::= stream_options WATERMARK duration_literal */ - { 222, -3 }, /* (245) cmd ::= KILL CONNECTION NK_INTEGER */ - { 222, -3 }, /* (246) cmd ::= KILL QUERY NK_INTEGER */ - { 222, -4 }, /* (247) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ - { 222, -4 }, /* (248) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ - { 222, -3 }, /* (249) cmd ::= SPLIT VGROUP NK_INTEGER */ - { 283, -2 }, /* (250) dnode_list ::= DNODE NK_INTEGER */ - { 283, -3 }, /* (251) dnode_list ::= dnode_list DNODE NK_INTEGER */ - { 222, -3 }, /* (252) cmd ::= SYNCDB db_name REPLICA */ - { 222, -1 }, /* (253) cmd ::= query_expression */ - { 225, -1 }, /* (254) literal ::= NK_INTEGER */ - { 225, -1 }, /* (255) literal ::= NK_FLOAT */ - { 225, -1 }, /* (256) literal ::= NK_STRING */ - { 225, -1 }, /* (257) literal ::= NK_BOOL */ - { 225, -2 }, /* (258) literal ::= TIMESTAMP NK_STRING */ - { 225, -1 }, /* (259) literal ::= duration_literal */ - { 225, -1 }, /* (260) literal ::= NULL */ - { 225, -1 }, /* (261) literal ::= NK_QUESTION */ - { 270, -1 }, /* (262) duration_literal ::= NK_VARIABLE */ - { 284, -1 }, /* (263) signed ::= NK_INTEGER */ - { 284, -2 }, /* (264) signed ::= NK_PLUS NK_INTEGER */ - { 284, -2 }, /* (265) signed ::= NK_MINUS NK_INTEGER */ - { 284, -1 }, /* (266) signed ::= NK_FLOAT */ - { 284, -2 }, /* (267) signed ::= NK_PLUS NK_FLOAT */ - { 284, -2 }, /* (268) signed ::= NK_MINUS NK_FLOAT */ - { 285, -1 }, /* (269) signed_literal ::= signed */ - { 285, -1 }, /* (270) signed_literal ::= NK_STRING */ - { 285, -1 }, /* (271) signed_literal ::= NK_BOOL */ - { 285, -2 }, /* (272) signed_literal ::= TIMESTAMP NK_STRING */ - { 285, -1 }, /* (273) signed_literal ::= duration_literal */ - { 285, -1 }, /* (274) signed_literal ::= NULL */ - { 253, -1 }, /* (275) literal_list ::= signed_literal */ - { 253, -3 }, /* (276) literal_list ::= literal_list NK_COMMA signed_literal */ - { 231, -1 }, /* (277) db_name ::= NK_ID */ - { 256, -1 }, /* (278) table_name ::= NK_ID */ - { 249, -1 }, /* (279) column_name ::= NK_ID */ - { 266, -1 }, /* (280) function_name ::= NK_ID */ - { 286, -1 }, /* (281) table_alias ::= NK_ID */ - { 287, -1 }, /* (282) column_alias ::= NK_ID */ - { 227, -1 }, /* (283) user_name ::= NK_ID */ - { 267, -1 }, /* (284) index_name ::= NK_ID */ - { 274, -1 }, /* (285) topic_name ::= NK_ID */ - { 280, -1 }, /* (286) stream_name ::= NK_ID */ - { 288, -1 }, /* (287) expression ::= literal */ - { 288, -1 }, /* (288) expression ::= pseudo_column */ - { 288, -1 }, /* (289) expression ::= column_reference */ - { 288, -1 }, /* (290) expression ::= function_expression */ - { 288, -1 }, /* (291) expression ::= subquery */ - { 288, -3 }, /* (292) expression ::= NK_LP expression NK_RP */ - { 288, -2 }, /* (293) expression ::= NK_PLUS expression */ - { 288, -2 }, /* (294) expression ::= NK_MINUS expression */ - { 288, -3 }, /* (295) expression ::= expression NK_PLUS expression */ - { 288, -3 }, /* (296) expression ::= expression NK_MINUS expression */ - { 288, -3 }, /* (297) expression ::= expression NK_STAR expression */ - { 288, -3 }, /* (298) expression ::= expression NK_SLASH expression */ - { 288, -3 }, /* (299) expression ::= expression NK_REM expression */ - { 288, -3 }, /* (300) expression ::= column_reference NK_ARROW NK_STRING */ - { 273, -1 }, /* (301) expression_list ::= expression */ - { 273, -3 }, /* (302) expression_list ::= expression_list NK_COMMA expression */ - { 290, -1 }, /* (303) column_reference ::= column_name */ - { 290, -3 }, /* (304) column_reference ::= table_name NK_DOT column_name */ - { 289, -1 }, /* (305) pseudo_column ::= ROWTS */ - { 289, -1 }, /* (306) pseudo_column ::= TBNAME */ - { 289, -1 }, /* (307) pseudo_column ::= QSTARTTS */ - { 289, -1 }, /* (308) pseudo_column ::= QENDTS */ - { 289, -1 }, /* (309) pseudo_column ::= WSTARTTS */ - { 289, -1 }, /* (310) pseudo_column ::= WENDTS */ - { 289, -1 }, /* (311) pseudo_column ::= WDURATION */ - { 291, -4 }, /* (312) function_expression ::= function_name NK_LP expression_list NK_RP */ - { 291, -4 }, /* (313) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ - { 291, -6 }, /* (314) function_expression ::= CAST NK_LP expression AS type_name NK_RP */ - { 291, -3 }, /* (315) function_expression ::= noarg_func NK_LP NK_RP */ - { 295, -1 }, /* (316) noarg_func ::= NOW */ - { 295, -1 }, /* (317) noarg_func ::= TODAY */ - { 295, -1 }, /* (318) noarg_func ::= TIMEZONE */ - { 293, -1 }, /* (319) star_func ::= COUNT */ - { 293, -1 }, /* (320) star_func ::= FIRST */ - { 293, -1 }, /* (321) star_func ::= LAST */ - { 293, -1 }, /* (322) star_func ::= LAST_ROW */ - { 294, -1 }, /* (323) star_func_para_list ::= NK_STAR */ - { 294, -1 }, /* (324) star_func_para_list ::= other_para_list */ - { 296, -1 }, /* (325) other_para_list ::= star_func_para */ - { 296, -3 }, /* (326) other_para_list ::= other_para_list NK_COMMA star_func_para */ - { 297, -1 }, /* (327) star_func_para ::= expression */ - { 297, -3 }, /* (328) star_func_para ::= table_name NK_DOT NK_STAR */ - { 298, -3 }, /* (329) predicate ::= expression compare_op expression */ - { 298, -5 }, /* (330) predicate ::= expression BETWEEN expression AND expression */ - { 298, -6 }, /* (331) predicate ::= expression NOT BETWEEN expression AND expression */ - { 298, -3 }, /* (332) predicate ::= expression IS NULL */ - { 298, -4 }, /* (333) predicate ::= expression IS NOT NULL */ - { 298, -3 }, /* (334) predicate ::= expression in_op in_predicate_value */ - { 299, -1 }, /* (335) compare_op ::= NK_LT */ - { 299, -1 }, /* (336) compare_op ::= NK_GT */ - { 299, -1 }, /* (337) compare_op ::= NK_LE */ - { 299, -1 }, /* (338) compare_op ::= NK_GE */ - { 299, -1 }, /* (339) compare_op ::= NK_NE */ - { 299, -1 }, /* (340) compare_op ::= NK_EQ */ - { 299, -1 }, /* (341) compare_op ::= LIKE */ - { 299, -2 }, /* (342) compare_op ::= NOT LIKE */ - { 299, -1 }, /* (343) compare_op ::= MATCH */ - { 299, -1 }, /* (344) compare_op ::= NMATCH */ - { 299, -1 }, /* (345) compare_op ::= CONTAINS */ - { 300, -1 }, /* (346) in_op ::= IN */ - { 300, -2 }, /* (347) in_op ::= NOT IN */ - { 301, -3 }, /* (348) in_predicate_value ::= NK_LP expression_list NK_RP */ - { 302, -1 }, /* (349) boolean_value_expression ::= boolean_primary */ - { 302, -2 }, /* (350) boolean_value_expression ::= NOT boolean_primary */ - { 302, -3 }, /* (351) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ - { 302, -3 }, /* (352) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ - { 303, -1 }, /* (353) boolean_primary ::= predicate */ - { 303, -3 }, /* (354) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - { 304, -1 }, /* (355) common_expression ::= expression */ - { 304, -1 }, /* (356) common_expression ::= boolean_value_expression */ - { 305, -2 }, /* (357) from_clause ::= FROM table_reference_list */ - { 306, -1 }, /* (358) table_reference_list ::= table_reference */ - { 306, -3 }, /* (359) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - { 307, -1 }, /* (360) table_reference ::= table_primary */ - { 307, -1 }, /* (361) table_reference ::= joined_table */ - { 308, -2 }, /* (362) table_primary ::= table_name alias_opt */ - { 308, -4 }, /* (363) table_primary ::= db_name NK_DOT table_name alias_opt */ - { 308, -2 }, /* (364) table_primary ::= subquery alias_opt */ - { 308, -1 }, /* (365) table_primary ::= parenthesized_joined_table */ - { 310, 0 }, /* (366) alias_opt ::= */ - { 310, -1 }, /* (367) alias_opt ::= table_alias */ - { 310, -2 }, /* (368) alias_opt ::= AS table_alias */ - { 311, -3 }, /* (369) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - { 311, -3 }, /* (370) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - { 309, -6 }, /* (371) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - { 312, 0 }, /* (372) join_type ::= */ - { 312, -1 }, /* (373) join_type ::= INNER */ - { 314, -9 }, /* (374) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - { 315, 0 }, /* (375) set_quantifier_opt ::= */ - { 315, -1 }, /* (376) set_quantifier_opt ::= DISTINCT */ - { 315, -1 }, /* (377) set_quantifier_opt ::= ALL */ - { 316, -1 }, /* (378) select_list ::= NK_STAR */ - { 316, -1 }, /* (379) select_list ::= select_sublist */ - { 322, -1 }, /* (380) select_sublist ::= select_item */ - { 322, -3 }, /* (381) select_sublist ::= select_sublist NK_COMMA select_item */ - { 323, -1 }, /* (382) select_item ::= common_expression */ - { 323, -2 }, /* (383) select_item ::= common_expression column_alias */ - { 323, -3 }, /* (384) select_item ::= common_expression AS column_alias */ - { 323, -3 }, /* (385) select_item ::= table_name NK_DOT NK_STAR */ - { 317, 0 }, /* (386) where_clause_opt ::= */ - { 317, -2 }, /* (387) where_clause_opt ::= WHERE search_condition */ - { 318, 0 }, /* (388) partition_by_clause_opt ::= */ - { 318, -3 }, /* (389) partition_by_clause_opt ::= PARTITION BY expression_list */ - { 319, 0 }, /* (390) twindow_clause_opt ::= */ - { 319, -6 }, /* (391) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ - { 319, -4 }, /* (392) twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ - { 319, -6 }, /* (393) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - { 319, -8 }, /* (394) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - { 271, 0 }, /* (395) sliding_opt ::= */ - { 271, -4 }, /* (396) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - { 324, 0 }, /* (397) fill_opt ::= */ - { 324, -4 }, /* (398) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - { 324, -6 }, /* (399) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ - { 325, -1 }, /* (400) fill_mode ::= NONE */ - { 325, -1 }, /* (401) fill_mode ::= PREV */ - { 325, -1 }, /* (402) fill_mode ::= NULL */ - { 325, -1 }, /* (403) fill_mode ::= LINEAR */ - { 325, -1 }, /* (404) fill_mode ::= NEXT */ - { 320, 0 }, /* (405) group_by_clause_opt ::= */ - { 320, -3 }, /* (406) group_by_clause_opt ::= GROUP BY group_by_list */ - { 326, -1 }, /* (407) group_by_list ::= expression */ - { 326, -3 }, /* (408) group_by_list ::= group_by_list NK_COMMA expression */ - { 321, 0 }, /* (409) having_clause_opt ::= */ - { 321, -2 }, /* (410) having_clause_opt ::= HAVING search_condition */ - { 275, -4 }, /* (411) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 327, -1 }, /* (412) query_expression_body ::= query_primary */ - { 327, -4 }, /* (413) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ - { 327, -3 }, /* (414) query_expression_body ::= query_expression_body UNION query_expression_body */ - { 331, -1 }, /* (415) query_primary ::= query_specification */ - { 328, 0 }, /* (416) order_by_clause_opt ::= */ - { 328, -3 }, /* (417) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 329, 0 }, /* (418) slimit_clause_opt ::= */ - { 329, -2 }, /* (419) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - { 329, -4 }, /* (420) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 329, -4 }, /* (421) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 330, 0 }, /* (422) limit_clause_opt ::= */ - { 330, -2 }, /* (423) limit_clause_opt ::= LIMIT NK_INTEGER */ - { 330, -4 }, /* (424) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 330, -4 }, /* (425) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 292, -3 }, /* (426) subquery ::= NK_LP query_expression NK_RP */ - { 313, -1 }, /* (427) search_condition ::= common_expression */ - { 332, -1 }, /* (428) sort_specification_list ::= sort_specification */ - { 332, -3 }, /* (429) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 333, -3 }, /* (430) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ - { 334, 0 }, /* (431) ordering_specification_opt ::= */ - { 334, -1 }, /* (432) ordering_specification_opt ::= ASC */ - { 334, -1 }, /* (433) ordering_specification_opt ::= DESC */ - { 335, 0 }, /* (434) null_ordering_opt ::= */ - { 335, -2 }, /* (435) null_ordering_opt ::= NULLS FIRST */ - { 335, -2 }, /* (436) null_ordering_opt ::= NULLS LAST */ + { 231, -6 }, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ + { 231, -4 }, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ + { 232, 0 }, /* (2) account_options ::= */ + { 232, -3 }, /* (3) account_options ::= account_options PPS literal */ + { 232, -3 }, /* (4) account_options ::= account_options TSERIES literal */ + { 232, -3 }, /* (5) account_options ::= account_options STORAGE literal */ + { 232, -3 }, /* (6) account_options ::= account_options STREAMS literal */ + { 232, -3 }, /* (7) account_options ::= account_options QTIME literal */ + { 232, -3 }, /* (8) account_options ::= account_options DBS literal */ + { 232, -3 }, /* (9) account_options ::= account_options USERS literal */ + { 232, -3 }, /* (10) account_options ::= account_options CONNS literal */ + { 232, -3 }, /* (11) account_options ::= account_options STATE literal */ + { 233, -1 }, /* (12) alter_account_options ::= alter_account_option */ + { 233, -2 }, /* (13) alter_account_options ::= alter_account_options alter_account_option */ + { 235, -2 }, /* (14) alter_account_option ::= PASS literal */ + { 235, -2 }, /* (15) alter_account_option ::= PPS literal */ + { 235, -2 }, /* (16) alter_account_option ::= TSERIES literal */ + { 235, -2 }, /* (17) alter_account_option ::= STORAGE literal */ + { 235, -2 }, /* (18) alter_account_option ::= STREAMS literal */ + { 235, -2 }, /* (19) alter_account_option ::= QTIME literal */ + { 235, -2 }, /* (20) alter_account_option ::= DBS literal */ + { 235, -2 }, /* (21) alter_account_option ::= USERS literal */ + { 235, -2 }, /* (22) alter_account_option ::= CONNS literal */ + { 235, -2 }, /* (23) alter_account_option ::= STATE literal */ + { 231, -5 }, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING */ + { 231, -5 }, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ + { 231, -5 }, /* (26) cmd ::= ALTER USER user_name PRIVILEGE NK_STRING */ + { 231, -3 }, /* (27) cmd ::= DROP USER user_name */ + { 231, -3 }, /* (28) cmd ::= CREATE DNODE dnode_endpoint */ + { 231, -5 }, /* (29) cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER */ + { 231, -3 }, /* (30) cmd ::= DROP DNODE NK_INTEGER */ + { 231, -3 }, /* (31) cmd ::= DROP DNODE dnode_endpoint */ + { 231, -4 }, /* (32) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ + { 231, -5 }, /* (33) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ + { 231, -4 }, /* (34) cmd ::= ALTER ALL DNODES NK_STRING */ + { 231, -5 }, /* (35) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ + { 237, -1 }, /* (36) dnode_endpoint ::= NK_STRING */ + { 238, -1 }, /* (37) dnode_host_name ::= NK_ID */ + { 238, -1 }, /* (38) dnode_host_name ::= NK_IPTOKEN */ + { 231, -3 }, /* (39) cmd ::= ALTER LOCAL NK_STRING */ + { 231, -4 }, /* (40) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ + { 231, -5 }, /* (41) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ + { 231, -5 }, /* (42) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ + { 231, -5 }, /* (43) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ + { 231, -5 }, /* (44) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ + { 231, -5 }, /* (45) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ + { 231, -5 }, /* (46) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ + { 231, -5 }, /* (47) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ + { 231, -5 }, /* (48) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ + { 231, -5 }, /* (49) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ + { 231, -4 }, /* (50) cmd ::= DROP DATABASE exists_opt db_name */ + { 231, -2 }, /* (51) cmd ::= USE db_name */ + { 231, -4 }, /* (52) cmd ::= ALTER DATABASE db_name alter_db_options */ + { 239, -3 }, /* (53) not_exists_opt ::= IF NOT EXISTS */ + { 239, 0 }, /* (54) not_exists_opt ::= */ + { 242, -2 }, /* (55) exists_opt ::= IF EXISTS */ + { 242, 0 }, /* (56) exists_opt ::= */ + { 241, 0 }, /* (57) db_options ::= */ + { 241, -3 }, /* (58) db_options ::= db_options BLOCKS NK_INTEGER */ + { 241, -3 }, /* (59) db_options ::= db_options CACHE NK_INTEGER */ + { 241, -3 }, /* (60) db_options ::= db_options CACHELAST NK_INTEGER */ + { 241, -3 }, /* (61) db_options ::= db_options COMP NK_INTEGER */ + { 241, -3 }, /* (62) db_options ::= db_options DAYS NK_INTEGER */ + { 241, -3 }, /* (63) db_options ::= db_options DAYS NK_VARIABLE */ + { 241, -3 }, /* (64) db_options ::= db_options FSYNC NK_INTEGER */ + { 241, -3 }, /* (65) db_options ::= db_options MAXROWS NK_INTEGER */ + { 241, -3 }, /* (66) db_options ::= db_options MINROWS NK_INTEGER */ + { 241, -3 }, /* (67) db_options ::= db_options KEEP integer_list */ + { 241, -3 }, /* (68) db_options ::= db_options KEEP variable_list */ + { 241, -3 }, /* (69) db_options ::= db_options PRECISION NK_STRING */ + { 241, -3 }, /* (70) db_options ::= db_options QUORUM NK_INTEGER */ + { 241, -3 }, /* (71) db_options ::= db_options REPLICA NK_INTEGER */ + { 241, -3 }, /* (72) db_options ::= db_options TTL NK_INTEGER */ + { 241, -3 }, /* (73) db_options ::= db_options WAL NK_INTEGER */ + { 241, -3 }, /* (74) db_options ::= db_options VGROUPS NK_INTEGER */ + { 241, -3 }, /* (75) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ + { 241, -3 }, /* (76) db_options ::= db_options STREAM_MODE NK_INTEGER */ + { 241, -3 }, /* (77) db_options ::= db_options RETENTIONS retention_list */ + { 241, -3 }, /* (78) db_options ::= db_options STRICT NK_INTEGER */ + { 243, -1 }, /* (79) alter_db_options ::= alter_db_option */ + { 243, -2 }, /* (80) alter_db_options ::= alter_db_options alter_db_option */ + { 247, -2 }, /* (81) alter_db_option ::= BLOCKS NK_INTEGER */ + { 247, -2 }, /* (82) alter_db_option ::= FSYNC NK_INTEGER */ + { 247, -2 }, /* (83) alter_db_option ::= KEEP integer_list */ + { 247, -2 }, /* (84) alter_db_option ::= KEEP variable_list */ + { 247, -2 }, /* (85) alter_db_option ::= WAL NK_INTEGER */ + { 247, -2 }, /* (86) alter_db_option ::= QUORUM NK_INTEGER */ + { 247, -2 }, /* (87) alter_db_option ::= CACHELAST NK_INTEGER */ + { 247, -2 }, /* (88) alter_db_option ::= REPLICA NK_INTEGER */ + { 247, -2 }, /* (89) alter_db_option ::= STRICT NK_INTEGER */ + { 244, -1 }, /* (90) integer_list ::= NK_INTEGER */ + { 244, -3 }, /* (91) integer_list ::= integer_list NK_COMMA NK_INTEGER */ + { 245, -1 }, /* (92) variable_list ::= NK_VARIABLE */ + { 245, -3 }, /* (93) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ + { 246, -1 }, /* (94) retention_list ::= retention */ + { 246, -3 }, /* (95) retention_list ::= retention_list NK_COMMA retention */ + { 248, -3 }, /* (96) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ + { 231, -9 }, /* (97) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + { 231, -3 }, /* (98) cmd ::= CREATE TABLE multi_create_clause */ + { 231, -9 }, /* (99) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ + { 231, -3 }, /* (100) cmd ::= DROP TABLE multi_drop_clause */ + { 231, -4 }, /* (101) cmd ::= DROP STABLE exists_opt full_table_name */ + { 231, -3 }, /* (102) cmd ::= ALTER TABLE alter_table_clause */ + { 231, -3 }, /* (103) cmd ::= ALTER STABLE alter_table_clause */ + { 256, -2 }, /* (104) alter_table_clause ::= full_table_name alter_table_options */ + { 256, -5 }, /* (105) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ + { 256, -4 }, /* (106) alter_table_clause ::= full_table_name DROP COLUMN column_name */ + { 256, -5 }, /* (107) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ + { 256, -5 }, /* (108) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ + { 256, -5 }, /* (109) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ + { 256, -4 }, /* (110) alter_table_clause ::= full_table_name DROP TAG column_name */ + { 256, -5 }, /* (111) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ + { 256, -5 }, /* (112) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ + { 256, -6 }, /* (113) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal */ + { 253, -1 }, /* (114) multi_create_clause ::= create_subtable_clause */ + { 253, -2 }, /* (115) multi_create_clause ::= multi_create_clause create_subtable_clause */ + { 260, -9 }, /* (116) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP */ + { 255, -1 }, /* (117) multi_drop_clause ::= drop_table_clause */ + { 255, -2 }, /* (118) multi_drop_clause ::= multi_drop_clause drop_table_clause */ + { 263, -2 }, /* (119) drop_table_clause ::= exists_opt full_table_name */ + { 261, 0 }, /* (120) specific_tags_opt ::= */ + { 261, -3 }, /* (121) specific_tags_opt ::= NK_LP col_name_list NK_RP */ + { 249, -1 }, /* (122) full_table_name ::= table_name */ + { 249, -3 }, /* (123) full_table_name ::= db_name NK_DOT table_name */ + { 250, -1 }, /* (124) column_def_list ::= column_def */ + { 250, -3 }, /* (125) column_def_list ::= column_def_list NK_COMMA column_def */ + { 266, -2 }, /* (126) column_def ::= column_name type_name */ + { 266, -4 }, /* (127) column_def ::= column_name type_name COMMENT NK_STRING */ + { 259, -1 }, /* (128) type_name ::= BOOL */ + { 259, -1 }, /* (129) type_name ::= TINYINT */ + { 259, -1 }, /* (130) type_name ::= SMALLINT */ + { 259, -1 }, /* (131) type_name ::= INT */ + { 259, -1 }, /* (132) type_name ::= INTEGER */ + { 259, -1 }, /* (133) type_name ::= BIGINT */ + { 259, -1 }, /* (134) type_name ::= FLOAT */ + { 259, -1 }, /* (135) type_name ::= DOUBLE */ + { 259, -4 }, /* (136) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ + { 259, -1 }, /* (137) type_name ::= TIMESTAMP */ + { 259, -4 }, /* (138) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ + { 259, -2 }, /* (139) type_name ::= TINYINT UNSIGNED */ + { 259, -2 }, /* (140) type_name ::= SMALLINT UNSIGNED */ + { 259, -2 }, /* (141) type_name ::= INT UNSIGNED */ + { 259, -2 }, /* (142) type_name ::= BIGINT UNSIGNED */ + { 259, -1 }, /* (143) type_name ::= JSON */ + { 259, -4 }, /* (144) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ + { 259, -1 }, /* (145) type_name ::= MEDIUMBLOB */ + { 259, -1 }, /* (146) type_name ::= BLOB */ + { 259, -4 }, /* (147) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ + { 259, -1 }, /* (148) type_name ::= DECIMAL */ + { 259, -4 }, /* (149) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ + { 259, -6 }, /* (150) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ + { 251, 0 }, /* (151) tags_def_opt ::= */ + { 251, -1 }, /* (152) tags_def_opt ::= tags_def */ + { 254, -4 }, /* (153) tags_def ::= TAGS NK_LP column_def_list NK_RP */ + { 252, 0 }, /* (154) table_options ::= */ + { 252, -3 }, /* (155) table_options ::= table_options COMMENT NK_STRING */ + { 252, -3 }, /* (156) table_options ::= table_options KEEP integer_list */ + { 252, -3 }, /* (157) table_options ::= table_options KEEP variable_list */ + { 252, -3 }, /* (158) table_options ::= table_options TTL NK_INTEGER */ + { 252, -5 }, /* (159) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ + { 252, -5 }, /* (160) table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP */ + { 252, -3 }, /* (161) table_options ::= table_options FILE_FACTOR NK_FLOAT */ + { 252, -3 }, /* (162) table_options ::= table_options DELAY NK_INTEGER */ + { 257, -1 }, /* (163) alter_table_options ::= alter_table_option */ + { 257, -2 }, /* (164) alter_table_options ::= alter_table_options alter_table_option */ + { 268, -2 }, /* (165) alter_table_option ::= COMMENT NK_STRING */ + { 268, -2 }, /* (166) alter_table_option ::= KEEP integer_list */ + { 268, -2 }, /* (167) alter_table_option ::= KEEP variable_list */ + { 268, -2 }, /* (168) alter_table_option ::= TTL NK_INTEGER */ + { 264, -1 }, /* (169) col_name_list ::= col_name */ + { 264, -3 }, /* (170) col_name_list ::= col_name_list NK_COMMA col_name */ + { 269, -1 }, /* (171) col_name ::= column_name */ + { 231, -2 }, /* (172) cmd ::= SHOW DNODES */ + { 231, -2 }, /* (173) cmd ::= SHOW USERS */ + { 231, -2 }, /* (174) cmd ::= SHOW DATABASES */ + { 231, -4 }, /* (175) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ + { 231, -4 }, /* (176) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ + { 231, -3 }, /* (177) cmd ::= SHOW db_name_cond_opt VGROUPS */ + { 231, -2 }, /* (178) cmd ::= SHOW MNODES */ + { 231, -2 }, /* (179) cmd ::= SHOW MODULES */ + { 231, -2 }, /* (180) cmd ::= SHOW QNODES */ + { 231, -2 }, /* (181) cmd ::= SHOW FUNCTIONS */ + { 231, -5 }, /* (182) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ + { 231, -2 }, /* (183) cmd ::= SHOW STREAMS */ + { 231, -2 }, /* (184) cmd ::= SHOW ACCOUNTS */ + { 231, -2 }, /* (185) cmd ::= SHOW APPS */ + { 231, -2 }, /* (186) cmd ::= SHOW CONNECTIONS */ + { 231, -2 }, /* (187) cmd ::= SHOW LICENCE */ + { 231, -2 }, /* (188) cmd ::= SHOW GRANTS */ + { 231, -4 }, /* (189) cmd ::= SHOW CREATE DATABASE db_name */ + { 231, -4 }, /* (190) cmd ::= SHOW CREATE TABLE full_table_name */ + { 231, -4 }, /* (191) cmd ::= SHOW CREATE STABLE full_table_name */ + { 231, -2 }, /* (192) cmd ::= SHOW QUERIES */ + { 231, -2 }, /* (193) cmd ::= SHOW SCORES */ + { 231, -2 }, /* (194) cmd ::= SHOW TOPICS */ + { 231, -2 }, /* (195) cmd ::= SHOW VARIABLES */ + { 231, -2 }, /* (196) cmd ::= SHOW BNODES */ + { 231, -2 }, /* (197) cmd ::= SHOW SNODES */ + { 231, -2 }, /* (198) cmd ::= SHOW CLUSTER */ + { 270, 0 }, /* (199) db_name_cond_opt ::= */ + { 270, -2 }, /* (200) db_name_cond_opt ::= db_name NK_DOT */ + { 271, 0 }, /* (201) like_pattern_opt ::= */ + { 271, -2 }, /* (202) like_pattern_opt ::= LIKE NK_STRING */ + { 272, -1 }, /* (203) table_name_cond ::= table_name */ + { 273, 0 }, /* (204) from_db_opt ::= */ + { 273, -2 }, /* (205) from_db_opt ::= FROM db_name */ + { 267, -1 }, /* (206) func_name_list ::= func_name */ + { 267, -3 }, /* (207) func_name_list ::= func_name_list NK_COMMA func_name */ + { 274, -1 }, /* (208) func_name ::= function_name */ + { 231, -8 }, /* (209) cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options */ + { 231, -10 }, /* (210) cmd ::= CREATE FULLTEXT INDEX not_exists_opt index_name ON table_name NK_LP col_name_list NK_RP */ + { 231, -6 }, /* (211) cmd ::= DROP INDEX exists_opt index_name ON table_name */ + { 277, 0 }, /* (212) index_options ::= */ + { 277, -9 }, /* (213) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ + { 277, -11 }, /* (214) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ + { 278, -1 }, /* (215) func_list ::= func */ + { 278, -3 }, /* (216) func_list ::= func_list NK_COMMA func */ + { 281, -4 }, /* (217) func ::= function_name NK_LP expression_list NK_RP */ + { 231, -7 }, /* (218) cmd ::= CREATE TOPIC not_exists_opt topic_name topic_options AS query_expression */ + { 231, -7 }, /* (219) cmd ::= CREATE TOPIC not_exists_opt topic_name topic_options AS db_name */ + { 231, -4 }, /* (220) cmd ::= DROP TOPIC exists_opt topic_name */ + { 284, 0 }, /* (221) topic_options ::= */ + { 284, -3 }, /* (222) topic_options ::= topic_options WITH TABLE */ + { 284, -3 }, /* (223) topic_options ::= topic_options WITH SCHEMA */ + { 284, -3 }, /* (224) topic_options ::= topic_options WITH TAG */ + { 231, -2 }, /* (225) cmd ::= DESC full_table_name */ + { 231, -2 }, /* (226) cmd ::= DESCRIBE full_table_name */ + { 231, -3 }, /* (227) cmd ::= RESET QUERY CACHE */ + { 231, -4 }, /* (228) cmd ::= EXPLAIN analyze_opt explain_options query_expression */ + { 286, 0 }, /* (229) analyze_opt ::= */ + { 286, -1 }, /* (230) analyze_opt ::= ANALYZE */ + { 287, 0 }, /* (231) explain_options ::= */ + { 287, -3 }, /* (232) explain_options ::= explain_options VERBOSE NK_BOOL */ + { 287, -3 }, /* (233) explain_options ::= explain_options RATIO NK_FLOAT */ + { 231, -6 }, /* (234) cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP */ + { 231, -10 }, /* (235) cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ + { 231, -3 }, /* (236) cmd ::= DROP FUNCTION function_name */ + { 288, 0 }, /* (237) agg_func_opt ::= */ + { 288, -1 }, /* (238) agg_func_opt ::= AGGREGATE */ + { 289, 0 }, /* (239) bufsize_opt ::= */ + { 289, -2 }, /* (240) bufsize_opt ::= BUFSIZE NK_INTEGER */ + { 231, -8 }, /* (241) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression */ + { 231, -4 }, /* (242) cmd ::= DROP STREAM exists_opt stream_name */ + { 292, 0 }, /* (243) into_opt ::= */ + { 292, -2 }, /* (244) into_opt ::= INTO full_table_name */ + { 291, 0 }, /* (245) stream_options ::= */ + { 291, -3 }, /* (246) stream_options ::= stream_options TRIGGER AT_ONCE */ + { 291, -3 }, /* (247) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ + { 291, -3 }, /* (248) stream_options ::= stream_options WATERMARK duration_literal */ + { 231, -3 }, /* (249) cmd ::= KILL CONNECTION NK_INTEGER */ + { 231, -3 }, /* (250) cmd ::= KILL QUERY NK_INTEGER */ + { 231, -4 }, /* (251) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + { 231, -4 }, /* (252) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ + { 231, -3 }, /* (253) cmd ::= SPLIT VGROUP NK_INTEGER */ + { 293, -2 }, /* (254) dnode_list ::= DNODE NK_INTEGER */ + { 293, -3 }, /* (255) dnode_list ::= dnode_list DNODE NK_INTEGER */ + { 231, -3 }, /* (256) cmd ::= SYNCDB db_name REPLICA */ + { 231, -1 }, /* (257) cmd ::= query_expression */ + { 234, -1 }, /* (258) literal ::= NK_INTEGER */ + { 234, -1 }, /* (259) literal ::= NK_FLOAT */ + { 234, -1 }, /* (260) literal ::= NK_STRING */ + { 234, -1 }, /* (261) literal ::= NK_BOOL */ + { 234, -2 }, /* (262) literal ::= TIMESTAMP NK_STRING */ + { 234, -1 }, /* (263) literal ::= duration_literal */ + { 234, -1 }, /* (264) literal ::= NULL */ + { 234, -1 }, /* (265) literal ::= NK_QUESTION */ + { 279, -1 }, /* (266) duration_literal ::= NK_VARIABLE */ + { 294, -1 }, /* (267) signed ::= NK_INTEGER */ + { 294, -2 }, /* (268) signed ::= NK_PLUS NK_INTEGER */ + { 294, -2 }, /* (269) signed ::= NK_MINUS NK_INTEGER */ + { 294, -1 }, /* (270) signed ::= NK_FLOAT */ + { 294, -2 }, /* (271) signed ::= NK_PLUS NK_FLOAT */ + { 294, -2 }, /* (272) signed ::= NK_MINUS NK_FLOAT */ + { 295, -1 }, /* (273) signed_literal ::= signed */ + { 295, -1 }, /* (274) signed_literal ::= NK_STRING */ + { 295, -1 }, /* (275) signed_literal ::= NK_BOOL */ + { 295, -2 }, /* (276) signed_literal ::= TIMESTAMP NK_STRING */ + { 295, -1 }, /* (277) signed_literal ::= duration_literal */ + { 295, -1 }, /* (278) signed_literal ::= NULL */ + { 262, -1 }, /* (279) literal_list ::= signed_literal */ + { 262, -3 }, /* (280) literal_list ::= literal_list NK_COMMA signed_literal */ + { 240, -1 }, /* (281) db_name ::= NK_ID */ + { 265, -1 }, /* (282) table_name ::= NK_ID */ + { 258, -1 }, /* (283) column_name ::= NK_ID */ + { 275, -1 }, /* (284) function_name ::= NK_ID */ + { 296, -1 }, /* (285) table_alias ::= NK_ID */ + { 297, -1 }, /* (286) column_alias ::= NK_ID */ + { 236, -1 }, /* (287) user_name ::= NK_ID */ + { 276, -1 }, /* (288) index_name ::= NK_ID */ + { 283, -1 }, /* (289) topic_name ::= NK_ID */ + { 290, -1 }, /* (290) stream_name ::= NK_ID */ + { 298, -1 }, /* (291) expression ::= literal */ + { 298, -1 }, /* (292) expression ::= pseudo_column */ + { 298, -1 }, /* (293) expression ::= column_reference */ + { 298, -1 }, /* (294) expression ::= function_expression */ + { 298, -1 }, /* (295) expression ::= subquery */ + { 298, -3 }, /* (296) expression ::= NK_LP expression NK_RP */ + { 298, -2 }, /* (297) expression ::= NK_PLUS expression */ + { 298, -2 }, /* (298) expression ::= NK_MINUS expression */ + { 298, -3 }, /* (299) expression ::= expression NK_PLUS expression */ + { 298, -3 }, /* (300) expression ::= expression NK_MINUS expression */ + { 298, -3 }, /* (301) expression ::= expression NK_STAR expression */ + { 298, -3 }, /* (302) expression ::= expression NK_SLASH expression */ + { 298, -3 }, /* (303) expression ::= expression NK_REM expression */ + { 298, -3 }, /* (304) expression ::= column_reference NK_ARROW NK_STRING */ + { 282, -1 }, /* (305) expression_list ::= expression */ + { 282, -3 }, /* (306) expression_list ::= expression_list NK_COMMA expression */ + { 300, -1 }, /* (307) column_reference ::= column_name */ + { 300, -3 }, /* (308) column_reference ::= table_name NK_DOT column_name */ + { 299, -1 }, /* (309) pseudo_column ::= ROWTS */ + { 299, -1 }, /* (310) pseudo_column ::= TBNAME */ + { 299, -1 }, /* (311) pseudo_column ::= QSTARTTS */ + { 299, -1 }, /* (312) pseudo_column ::= QENDTS */ + { 299, -1 }, /* (313) pseudo_column ::= WSTARTTS */ + { 299, -1 }, /* (314) pseudo_column ::= WENDTS */ + { 299, -1 }, /* (315) pseudo_column ::= WDURATION */ + { 301, -4 }, /* (316) function_expression ::= function_name NK_LP expression_list NK_RP */ + { 301, -4 }, /* (317) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ + { 301, -6 }, /* (318) function_expression ::= CAST NK_LP expression AS type_name NK_RP */ + { 301, -3 }, /* (319) function_expression ::= noarg_func NK_LP NK_RP */ + { 305, -1 }, /* (320) noarg_func ::= NOW */ + { 305, -1 }, /* (321) noarg_func ::= TODAY */ + { 305, -1 }, /* (322) noarg_func ::= TIMEZONE */ + { 303, -1 }, /* (323) star_func ::= COUNT */ + { 303, -1 }, /* (324) star_func ::= FIRST */ + { 303, -1 }, /* (325) star_func ::= LAST */ + { 303, -1 }, /* (326) star_func ::= LAST_ROW */ + { 304, -1 }, /* (327) star_func_para_list ::= NK_STAR */ + { 304, -1 }, /* (328) star_func_para_list ::= other_para_list */ + { 306, -1 }, /* (329) other_para_list ::= star_func_para */ + { 306, -3 }, /* (330) other_para_list ::= other_para_list NK_COMMA star_func_para */ + { 307, -1 }, /* (331) star_func_para ::= expression */ + { 307, -3 }, /* (332) star_func_para ::= table_name NK_DOT NK_STAR */ + { 308, -3 }, /* (333) predicate ::= expression compare_op expression */ + { 308, -5 }, /* (334) predicate ::= expression BETWEEN expression AND expression */ + { 308, -6 }, /* (335) predicate ::= expression NOT BETWEEN expression AND expression */ + { 308, -3 }, /* (336) predicate ::= expression IS NULL */ + { 308, -4 }, /* (337) predicate ::= expression IS NOT NULL */ + { 308, -3 }, /* (338) predicate ::= expression in_op in_predicate_value */ + { 309, -1 }, /* (339) compare_op ::= NK_LT */ + { 309, -1 }, /* (340) compare_op ::= NK_GT */ + { 309, -1 }, /* (341) compare_op ::= NK_LE */ + { 309, -1 }, /* (342) compare_op ::= NK_GE */ + { 309, -1 }, /* (343) compare_op ::= NK_NE */ + { 309, -1 }, /* (344) compare_op ::= NK_EQ */ + { 309, -1 }, /* (345) compare_op ::= LIKE */ + { 309, -2 }, /* (346) compare_op ::= NOT LIKE */ + { 309, -1 }, /* (347) compare_op ::= MATCH */ + { 309, -1 }, /* (348) compare_op ::= NMATCH */ + { 309, -1 }, /* (349) compare_op ::= CONTAINS */ + { 310, -1 }, /* (350) in_op ::= IN */ + { 310, -2 }, /* (351) in_op ::= NOT IN */ + { 311, -3 }, /* (352) in_predicate_value ::= NK_LP expression_list NK_RP */ + { 312, -1 }, /* (353) boolean_value_expression ::= boolean_primary */ + { 312, -2 }, /* (354) boolean_value_expression ::= NOT boolean_primary */ + { 312, -3 }, /* (355) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + { 312, -3 }, /* (356) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + { 313, -1 }, /* (357) boolean_primary ::= predicate */ + { 313, -3 }, /* (358) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + { 314, -1 }, /* (359) common_expression ::= expression */ + { 314, -1 }, /* (360) common_expression ::= boolean_value_expression */ + { 315, -2 }, /* (361) from_clause ::= FROM table_reference_list */ + { 316, -1 }, /* (362) table_reference_list ::= table_reference */ + { 316, -3 }, /* (363) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 317, -1 }, /* (364) table_reference ::= table_primary */ + { 317, -1 }, /* (365) table_reference ::= joined_table */ + { 318, -2 }, /* (366) table_primary ::= table_name alias_opt */ + { 318, -4 }, /* (367) table_primary ::= db_name NK_DOT table_name alias_opt */ + { 318, -2 }, /* (368) table_primary ::= subquery alias_opt */ + { 318, -1 }, /* (369) table_primary ::= parenthesized_joined_table */ + { 320, 0 }, /* (370) alias_opt ::= */ + { 320, -1 }, /* (371) alias_opt ::= table_alias */ + { 320, -2 }, /* (372) alias_opt ::= AS table_alias */ + { 321, -3 }, /* (373) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 321, -3 }, /* (374) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 319, -6 }, /* (375) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 322, 0 }, /* (376) join_type ::= */ + { 322, -1 }, /* (377) join_type ::= INNER */ + { 324, -9 }, /* (378) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 325, 0 }, /* (379) set_quantifier_opt ::= */ + { 325, -1 }, /* (380) set_quantifier_opt ::= DISTINCT */ + { 325, -1 }, /* (381) set_quantifier_opt ::= ALL */ + { 326, -1 }, /* (382) select_list ::= NK_STAR */ + { 326, -1 }, /* (383) select_list ::= select_sublist */ + { 332, -1 }, /* (384) select_sublist ::= select_item */ + { 332, -3 }, /* (385) select_sublist ::= select_sublist NK_COMMA select_item */ + { 333, -1 }, /* (386) select_item ::= common_expression */ + { 333, -2 }, /* (387) select_item ::= common_expression column_alias */ + { 333, -3 }, /* (388) select_item ::= common_expression AS column_alias */ + { 333, -3 }, /* (389) select_item ::= table_name NK_DOT NK_STAR */ + { 327, 0 }, /* (390) where_clause_opt ::= */ + { 327, -2 }, /* (391) where_clause_opt ::= WHERE search_condition */ + { 328, 0 }, /* (392) partition_by_clause_opt ::= */ + { 328, -3 }, /* (393) partition_by_clause_opt ::= PARTITION BY expression_list */ + { 329, 0 }, /* (394) twindow_clause_opt ::= */ + { 329, -6 }, /* (395) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + { 329, -4 }, /* (396) twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ + { 329, -6 }, /* (397) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 329, -8 }, /* (398) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 280, 0 }, /* (399) sliding_opt ::= */ + { 280, -4 }, /* (400) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 334, 0 }, /* (401) fill_opt ::= */ + { 334, -4 }, /* (402) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 334, -6 }, /* (403) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + { 335, -1 }, /* (404) fill_mode ::= NONE */ + { 335, -1 }, /* (405) fill_mode ::= PREV */ + { 335, -1 }, /* (406) fill_mode ::= NULL */ + { 335, -1 }, /* (407) fill_mode ::= LINEAR */ + { 335, -1 }, /* (408) fill_mode ::= NEXT */ + { 330, 0 }, /* (409) group_by_clause_opt ::= */ + { 330, -3 }, /* (410) group_by_clause_opt ::= GROUP BY group_by_list */ + { 336, -1 }, /* (411) group_by_list ::= expression */ + { 336, -3 }, /* (412) group_by_list ::= group_by_list NK_COMMA expression */ + { 331, 0 }, /* (413) having_clause_opt ::= */ + { 331, -2 }, /* (414) having_clause_opt ::= HAVING search_condition */ + { 285, -4 }, /* (415) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 337, -1 }, /* (416) query_expression_body ::= query_primary */ + { 337, -4 }, /* (417) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ + { 337, -3 }, /* (418) query_expression_body ::= query_expression_body UNION query_expression_body */ + { 341, -1 }, /* (419) query_primary ::= query_specification */ + { 338, 0 }, /* (420) order_by_clause_opt ::= */ + { 338, -3 }, /* (421) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 339, 0 }, /* (422) slimit_clause_opt ::= */ + { 339, -2 }, /* (423) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 339, -4 }, /* (424) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 339, -4 }, /* (425) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 340, 0 }, /* (426) limit_clause_opt ::= */ + { 340, -2 }, /* (427) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 340, -4 }, /* (428) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 340, -4 }, /* (429) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 302, -3 }, /* (430) subquery ::= NK_LP query_expression NK_RP */ + { 323, -1 }, /* (431) search_condition ::= common_expression */ + { 342, -1 }, /* (432) sort_specification_list ::= sort_specification */ + { 342, -3 }, /* (433) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 343, -3 }, /* (434) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + { 344, 0 }, /* (435) ordering_specification_opt ::= */ + { 344, -1 }, /* (436) ordering_specification_opt ::= ASC */ + { 344, -1 }, /* (437) ordering_specification_opt ::= DESC */ + { 345, 0 }, /* (438) null_ordering_opt ::= */ + { 345, -2 }, /* (439) null_ordering_opt ::= NULLS FIRST */ + { 345, -2 }, /* (440) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -2749,11 +2991,11 @@ static YYACTIONTYPE yy_reduce( YYMINORTYPE yylhsminor; case 0: /* cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,223,&yymsp[0].minor); + yy_destructor(yypParser,232,&yymsp[0].minor); break; case 1: /* cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,224,&yymsp[0].minor); + yy_destructor(yypParser,233,&yymsp[0].minor); break; case 2: /* account_options ::= */ { } @@ -2767,20 +3009,20 @@ static YYACTIONTYPE yy_reduce( case 9: /* account_options ::= account_options USERS literal */ yytestcase(yyruleno==9); case 10: /* account_options ::= account_options CONNS literal */ yytestcase(yyruleno==10); case 11: /* account_options ::= account_options STATE literal */ yytestcase(yyruleno==11); -{ yy_destructor(yypParser,223,&yymsp[-2].minor); +{ yy_destructor(yypParser,232,&yymsp[-2].minor); { } - yy_destructor(yypParser,225,&yymsp[0].minor); + yy_destructor(yypParser,234,&yymsp[0].minor); } break; case 12: /* alter_account_options ::= alter_account_option */ -{ yy_destructor(yypParser,226,&yymsp[0].minor); +{ yy_destructor(yypParser,235,&yymsp[0].minor); { } } break; case 13: /* alter_account_options ::= alter_account_options alter_account_option */ -{ yy_destructor(yypParser,224,&yymsp[-1].minor); +{ yy_destructor(yypParser,233,&yymsp[-1].minor); { } - yy_destructor(yypParser,226,&yymsp[0].minor); + yy_destructor(yypParser,235,&yymsp[0].minor); } break; case 14: /* alter_account_option ::= PASS literal */ @@ -2794,31 +3036,31 @@ static YYACTIONTYPE yy_reduce( case 22: /* alter_account_option ::= CONNS literal */ yytestcase(yyruleno==22); case 23: /* alter_account_option ::= STATE literal */ yytestcase(yyruleno==23); { } - yy_destructor(yypParser,225,&yymsp[0].minor); + yy_destructor(yypParser,234,&yymsp[0].minor); break; case 24: /* cmd ::= CREATE USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-2].minor.yy449, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-2].minor.yy517, &yymsp[0].minor.yy0); } break; case 25: /* cmd ::= ALTER USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy449, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy517, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } break; case 26: /* cmd ::= ALTER USER user_name PRIVILEGE NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy449, TSDB_ALTER_USER_PRIVILEGES, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy517, TSDB_ALTER_USER_PRIVILEGES, &yymsp[0].minor.yy0); } break; case 27: /* cmd ::= DROP USER user_name */ -{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy449); } +{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy517); } break; case 28: /* cmd ::= CREATE DNODE dnode_endpoint */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy449, NULL); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy517, NULL); } break; case 29: /* cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy449, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy517, &yymsp[0].minor.yy0); } break; case 30: /* cmd ::= DROP DNODE NK_INTEGER */ { pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy0); } break; case 31: /* cmd ::= DROP DNODE dnode_endpoint */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy449); } +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy517); } break; case 32: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, NULL); } @@ -2835,25 +3077,25 @@ static YYACTIONTYPE yy_reduce( case 36: /* dnode_endpoint ::= NK_STRING */ case 37: /* dnode_host_name ::= NK_ID */ yytestcase(yyruleno==37); case 38: /* dnode_host_name ::= NK_IPTOKEN */ yytestcase(yyruleno==38); - case 277: /* db_name ::= NK_ID */ yytestcase(yyruleno==277); - case 278: /* table_name ::= NK_ID */ yytestcase(yyruleno==278); - case 279: /* column_name ::= NK_ID */ yytestcase(yyruleno==279); - case 280: /* function_name ::= NK_ID */ yytestcase(yyruleno==280); - case 281: /* table_alias ::= NK_ID */ yytestcase(yyruleno==281); - case 282: /* column_alias ::= NK_ID */ yytestcase(yyruleno==282); - case 283: /* user_name ::= NK_ID */ yytestcase(yyruleno==283); - case 284: /* index_name ::= NK_ID */ yytestcase(yyruleno==284); - case 285: /* topic_name ::= NK_ID */ yytestcase(yyruleno==285); - case 286: /* stream_name ::= NK_ID */ yytestcase(yyruleno==286); - case 316: /* noarg_func ::= NOW */ yytestcase(yyruleno==316); - case 317: /* noarg_func ::= TODAY */ yytestcase(yyruleno==317); - case 318: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==318); - case 319: /* star_func ::= COUNT */ yytestcase(yyruleno==319); - case 320: /* star_func ::= FIRST */ yytestcase(yyruleno==320); - case 321: /* star_func ::= LAST */ yytestcase(yyruleno==321); - case 322: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==322); -{ yylhsminor.yy449 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy449 = yylhsminor.yy449; + case 281: /* db_name ::= NK_ID */ yytestcase(yyruleno==281); + case 282: /* table_name ::= NK_ID */ yytestcase(yyruleno==282); + case 283: /* column_name ::= NK_ID */ yytestcase(yyruleno==283); + case 284: /* function_name ::= NK_ID */ yytestcase(yyruleno==284); + case 285: /* table_alias ::= NK_ID */ yytestcase(yyruleno==285); + case 286: /* column_alias ::= NK_ID */ yytestcase(yyruleno==286); + case 287: /* user_name ::= NK_ID */ yytestcase(yyruleno==287); + case 288: /* index_name ::= NK_ID */ yytestcase(yyruleno==288); + case 289: /* topic_name ::= NK_ID */ yytestcase(yyruleno==289); + case 290: /* stream_name ::= NK_ID */ yytestcase(yyruleno==290); + case 320: /* noarg_func ::= NOW */ yytestcase(yyruleno==320); + case 321: /* noarg_func ::= TODAY */ yytestcase(yyruleno==321); + case 322: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==322); + case 323: /* star_func ::= COUNT */ yytestcase(yyruleno==323); + case 324: /* star_func ::= FIRST */ yytestcase(yyruleno==324); + case 325: /* star_func ::= LAST */ yytestcase(yyruleno==325); + case 326: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==326); +{ yylhsminor.yy517 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy517 = yylhsminor.yy517; break; case 39: /* cmd ::= ALTER LOCAL NK_STRING */ { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[0].minor.yy0, NULL); } @@ -2886,163 +3128,163 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_MNODE_STMT, &yymsp[0].minor.yy0); } break; case 49: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ -{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy89, &yymsp[-1].minor.yy449, yymsp[0].minor.yy392); } +{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy673, &yymsp[-1].minor.yy517, yymsp[0].minor.yy456); } break; case 50: /* cmd ::= DROP DATABASE exists_opt db_name */ -{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy89, &yymsp[0].minor.yy449); } +{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy673, &yymsp[0].minor.yy517); } break; case 51: /* cmd ::= USE db_name */ -{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy449); } +{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy517); } break; case 52: /* cmd ::= ALTER DATABASE db_name alter_db_options */ -{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy449, yymsp[0].minor.yy392); } +{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy517, yymsp[0].minor.yy456); } break; case 53: /* not_exists_opt ::= IF NOT EXISTS */ -{ yymsp[-2].minor.yy89 = true; } +{ yymsp[-2].minor.yy673 = true; } break; case 54: /* not_exists_opt ::= */ case 56: /* exists_opt ::= */ yytestcase(yyruleno==56); - case 225: /* analyze_opt ::= */ yytestcase(yyruleno==225); - case 233: /* agg_func_opt ::= */ yytestcase(yyruleno==233); - case 375: /* set_quantifier_opt ::= */ yytestcase(yyruleno==375); -{ yymsp[1].minor.yy89 = false; } + case 229: /* analyze_opt ::= */ yytestcase(yyruleno==229); + case 237: /* agg_func_opt ::= */ yytestcase(yyruleno==237); + case 379: /* set_quantifier_opt ::= */ yytestcase(yyruleno==379); +{ yymsp[1].minor.yy673 = false; } break; case 55: /* exists_opt ::= IF EXISTS */ -{ yymsp[-1].minor.yy89 = true; } +{ yymsp[-1].minor.yy673 = true; } break; case 57: /* db_options ::= */ -{ yymsp[1].minor.yy392 = createDatabaseOptions(pCxt); } +{ yymsp[1].minor.yy456 = createDatabaseOptions(pCxt); } break; case 58: /* db_options ::= db_options BLOCKS NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pNumOfBlocks = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pNumOfBlocks = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 59: /* db_options ::= db_options CACHE NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pCacheBlockSize = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pCacheBlockSize = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 60: /* db_options ::= db_options CACHELAST NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pCachelast = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pCachelast = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 61: /* db_options ::= db_options COMP NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pCompressionLevel = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pCompressionLevel = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 62: /* db_options ::= db_options DAYS NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pDaysPerFile = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pDaysPerFile = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 63: /* db_options ::= db_options DAYS NK_VARIABLE */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pDaysPerFile = (SValueNode*)createDurationValueNode(pCxt, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pDaysPerFile = (SValueNode*)createDurationValueNode(pCxt, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 64: /* db_options ::= db_options FSYNC NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pFsyncPeriod = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pFsyncPeriod = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 65: /* db_options ::= db_options MAXROWS NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pMaxRowsPerBlock = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pMaxRowsPerBlock = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 66: /* db_options ::= db_options MINROWS NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pMinRowsPerBlock = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pMinRowsPerBlock = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 67: /* db_options ::= db_options KEEP integer_list */ case 68: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==68); -{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pKeep = yymsp[0].minor.yy376; yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pKeep = yymsp[0].minor.yy652; yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 69: /* db_options ::= db_options PRECISION NK_STRING */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pPrecision = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pPrecision = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 70: /* db_options ::= db_options QUORUM NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pQuorum = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pQuorum = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 71: /* db_options ::= db_options REPLICA NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pReplica = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pReplica = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 72: /* db_options ::= db_options TTL NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pTtl = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pTtl = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 73: /* db_options ::= db_options WAL NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pWalLevel = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pWalLevel = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 74: /* db_options ::= db_options VGROUPS NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pNumOfVgroups = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pNumOfVgroups = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 75: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pSingleStable = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pSingleStable = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 76: /* db_options ::= db_options STREAM_MODE NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pStreamMode = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pStreamMode = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 77: /* db_options ::= db_options RETENTIONS retention_list */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pRetentions = yymsp[0].minor.yy376; yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pRetentions = yymsp[0].minor.yy652; yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 78: /* db_options ::= db_options STRICT NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pStrict = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pStrict = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 79: /* alter_db_options ::= alter_db_option */ -{ yylhsminor.yy392 = createDatabaseOptions(pCxt); yylhsminor.yy392 = setDatabaseAlterOption(pCxt, yylhsminor.yy392, &yymsp[0].minor.yy221); } - yymsp[0].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createDatabaseOptions(pCxt); yylhsminor.yy456 = setDatabaseAlterOption(pCxt, yylhsminor.yy456, &yymsp[0].minor.yy145); } + yymsp[0].minor.yy456 = yylhsminor.yy456; break; case 80: /* alter_db_options ::= alter_db_options alter_db_option */ -{ yylhsminor.yy392 = setDatabaseAlterOption(pCxt, yymsp[-1].minor.yy392, &yymsp[0].minor.yy221); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = setDatabaseAlterOption(pCxt, yymsp[-1].minor.yy456, &yymsp[0].minor.yy145); } + yymsp[-1].minor.yy456 = yylhsminor.yy456; break; case 81: /* alter_db_option ::= BLOCKS NK_INTEGER */ -{ yymsp[-1].minor.yy221.type = DB_OPTION_BLOCKS; yymsp[-1].minor.yy221.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy145.type = DB_OPTION_BLOCKS; yymsp[-1].minor.yy145.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } break; case 82: /* alter_db_option ::= FSYNC NK_INTEGER */ -{ yymsp[-1].minor.yy221.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy221.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy145.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy145.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } break; case 83: /* alter_db_option ::= KEEP integer_list */ case 84: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==84); -{ yymsp[-1].minor.yy221.type = DB_OPTION_KEEP; yymsp[-1].minor.yy221.pList = yymsp[0].minor.yy376; } +{ yymsp[-1].minor.yy145.type = DB_OPTION_KEEP; yymsp[-1].minor.yy145.pList = yymsp[0].minor.yy652; } break; case 85: /* alter_db_option ::= WAL NK_INTEGER */ -{ yymsp[-1].minor.yy221.type = DB_OPTION_WAL; yymsp[-1].minor.yy221.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy145.type = DB_OPTION_WAL; yymsp[-1].minor.yy145.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } break; case 86: /* alter_db_option ::= QUORUM NK_INTEGER */ -{ yymsp[-1].minor.yy221.type = DB_OPTION_QUORUM; yymsp[-1].minor.yy221.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy145.type = DB_OPTION_QUORUM; yymsp[-1].minor.yy145.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } break; case 87: /* alter_db_option ::= CACHELAST NK_INTEGER */ -{ yymsp[-1].minor.yy221.type = DB_OPTION_CACHELAST; yymsp[-1].minor.yy221.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy145.type = DB_OPTION_CACHELAST; yymsp[-1].minor.yy145.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } break; case 88: /* alter_db_option ::= REPLICA NK_INTEGER */ -{ yymsp[-1].minor.yy221.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy221.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy145.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy145.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } break; case 89: /* alter_db_option ::= STRICT NK_INTEGER */ -{ yymsp[-1].minor.yy221.type = DB_OPTION_STRICT; yymsp[-1].minor.yy221.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy145.type = DB_OPTION_STRICT; yymsp[-1].minor.yy145.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } break; case 90: /* integer_list ::= NK_INTEGER */ -{ yylhsminor.yy376 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy376 = yylhsminor.yy376; +{ yylhsminor.yy652 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy652 = yylhsminor.yy652; break; case 91: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ - case 251: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==251); -{ yylhsminor.yy376 = addNodeToList(pCxt, yymsp[-2].minor.yy376, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy376 = yylhsminor.yy376; + case 255: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==255); +{ yylhsminor.yy652 = addNodeToList(pCxt, yymsp[-2].minor.yy652, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy652 = yylhsminor.yy652; break; case 92: /* variable_list ::= NK_VARIABLE */ -{ yylhsminor.yy376 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy376 = yylhsminor.yy376; +{ yylhsminor.yy652 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy652 = yylhsminor.yy652; break; case 93: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ -{ yylhsminor.yy376 = addNodeToList(pCxt, yymsp[-2].minor.yy376, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy376 = yylhsminor.yy376; +{ yylhsminor.yy652 = addNodeToList(pCxt, yymsp[-2].minor.yy652, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy652 = yylhsminor.yy652; break; case 94: /* retention_list ::= retention */ case 114: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==114); @@ -3051,255 +3293,255 @@ static YYACTIONTYPE yy_reduce( case 169: /* col_name_list ::= col_name */ yytestcase(yyruleno==169); case 206: /* func_name_list ::= func_name */ yytestcase(yyruleno==206); case 215: /* func_list ::= func */ yytestcase(yyruleno==215); - case 275: /* literal_list ::= signed_literal */ yytestcase(yyruleno==275); - case 325: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==325); - case 380: /* select_sublist ::= select_item */ yytestcase(yyruleno==380); - case 428: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==428); -{ yylhsminor.yy376 = createNodeList(pCxt, yymsp[0].minor.yy392); } - yymsp[0].minor.yy376 = yylhsminor.yy376; + case 279: /* literal_list ::= signed_literal */ yytestcase(yyruleno==279); + case 329: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==329); + case 384: /* select_sublist ::= select_item */ yytestcase(yyruleno==384); + case 432: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==432); +{ yylhsminor.yy652 = createNodeList(pCxt, yymsp[0].minor.yy456); } + yymsp[0].minor.yy652 = yylhsminor.yy652; break; case 95: /* retention_list ::= retention_list NK_COMMA retention */ case 125: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==125); case 170: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==170); case 207: /* func_name_list ::= func_name_list NK_COMMA func_name */ yytestcase(yyruleno==207); case 216: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==216); - case 276: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==276); - case 326: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==326); - case 381: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==381); - case 429: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==429); -{ yylhsminor.yy376 = addNodeToList(pCxt, yymsp[-2].minor.yy376, yymsp[0].minor.yy392); } - yymsp[-2].minor.yy376 = yylhsminor.yy376; + case 280: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==280); + case 330: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==330); + case 385: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==385); + case 433: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==433); +{ yylhsminor.yy652 = addNodeToList(pCxt, yymsp[-2].minor.yy652, yymsp[0].minor.yy456); } + yymsp[-2].minor.yy652 = yylhsminor.yy652; break; case 96: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ -{ yylhsminor.yy392 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 97: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ case 99: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==99); -{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy89, yymsp[-5].minor.yy392, yymsp[-3].minor.yy376, yymsp[-1].minor.yy376, yymsp[0].minor.yy392); } +{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy673, yymsp[-5].minor.yy456, yymsp[-3].minor.yy652, yymsp[-1].minor.yy652, yymsp[0].minor.yy456); } break; case 98: /* cmd ::= CREATE TABLE multi_create_clause */ -{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy376); } +{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy652); } break; case 100: /* cmd ::= DROP TABLE multi_drop_clause */ -{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy376); } +{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy652); } break; case 101: /* cmd ::= DROP STABLE exists_opt full_table_name */ -{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy89, yymsp[0].minor.yy392); } +{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy673, yymsp[0].minor.yy456); } break; case 102: /* cmd ::= ALTER TABLE alter_table_clause */ case 103: /* cmd ::= ALTER STABLE alter_table_clause */ yytestcase(yyruleno==103); - case 253: /* cmd ::= query_expression */ yytestcase(yyruleno==253); -{ pCxt->pRootNode = yymsp[0].minor.yy392; } + case 257: /* cmd ::= query_expression */ yytestcase(yyruleno==257); +{ pCxt->pRootNode = yymsp[0].minor.yy456; } break; case 104: /* alter_table_clause ::= full_table_name alter_table_options */ -{ yylhsminor.yy392 = createAlterTableOption(pCxt, yymsp[-1].minor.yy392, yymsp[0].minor.yy392); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createAlterTableOption(pCxt, yymsp[-1].minor.yy456, yymsp[0].minor.yy456); } + yymsp[-1].minor.yy456 = yylhsminor.yy456; break; case 105: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ -{ yylhsminor.yy392 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy392, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy449, yymsp[0].minor.yy112); } - yymsp[-4].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy456, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy517, yymsp[0].minor.yy380); } + yymsp[-4].minor.yy456 = yylhsminor.yy456; break; case 106: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ -{ yylhsminor.yy392 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy392, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy449); } - yymsp[-3].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy456, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy517); } + yymsp[-3].minor.yy456 = yylhsminor.yy456; break; case 107: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ -{ yylhsminor.yy392 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy392, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy449, yymsp[0].minor.yy112); } - yymsp[-4].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy456, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy517, yymsp[0].minor.yy380); } + yymsp[-4].minor.yy456 = yylhsminor.yy456; break; case 108: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ -{ yylhsminor.yy392 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy392, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy449, &yymsp[0].minor.yy449); } - yymsp[-4].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy456, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy517, &yymsp[0].minor.yy517); } + yymsp[-4].minor.yy456 = yylhsminor.yy456; break; case 109: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ -{ yylhsminor.yy392 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy392, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy449, yymsp[0].minor.yy112); } - yymsp[-4].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy456, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy517, yymsp[0].minor.yy380); } + yymsp[-4].minor.yy456 = yylhsminor.yy456; break; case 110: /* alter_table_clause ::= full_table_name DROP TAG column_name */ -{ yylhsminor.yy392 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy392, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy449); } - yymsp[-3].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy456, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy517); } + yymsp[-3].minor.yy456 = yylhsminor.yy456; break; case 111: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ -{ yylhsminor.yy392 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy392, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy449, yymsp[0].minor.yy112); } - yymsp[-4].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy456, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy517, yymsp[0].minor.yy380); } + yymsp[-4].minor.yy456 = yylhsminor.yy456; break; case 112: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ -{ yylhsminor.yy392 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy392, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy449, &yymsp[0].minor.yy449); } - yymsp[-4].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy456, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy517, &yymsp[0].minor.yy517); } + yymsp[-4].minor.yy456 = yylhsminor.yy456; break; case 113: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal */ -{ yylhsminor.yy392 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy392, &yymsp[-2].minor.yy449, yymsp[0].minor.yy392); } - yymsp[-5].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy456, &yymsp[-2].minor.yy517, yymsp[0].minor.yy456); } + yymsp[-5].minor.yy456 = yylhsminor.yy456; break; case 115: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ case 118: /* multi_drop_clause ::= multi_drop_clause drop_table_clause */ yytestcase(yyruleno==118); -{ yylhsminor.yy376 = addNodeToList(pCxt, yymsp[-1].minor.yy376, yymsp[0].minor.yy392); } - yymsp[-1].minor.yy376 = yylhsminor.yy376; +{ yylhsminor.yy652 = addNodeToList(pCxt, yymsp[-1].minor.yy652, yymsp[0].minor.yy456); } + yymsp[-1].minor.yy652 = yylhsminor.yy652; break; case 116: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP */ -{ yylhsminor.yy392 = createCreateSubTableClause(pCxt, yymsp[-8].minor.yy89, yymsp[-7].minor.yy392, yymsp[-5].minor.yy392, yymsp[-4].minor.yy376, yymsp[-1].minor.yy376); } - yymsp[-8].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createCreateSubTableClause(pCxt, yymsp[-8].minor.yy673, yymsp[-7].minor.yy456, yymsp[-5].minor.yy456, yymsp[-4].minor.yy652, yymsp[-1].minor.yy652); } + yymsp[-8].minor.yy456 = yylhsminor.yy456; break; case 119: /* drop_table_clause ::= exists_opt full_table_name */ -{ yylhsminor.yy392 = createDropTableClause(pCxt, yymsp[-1].minor.yy89, yymsp[0].minor.yy392); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createDropTableClause(pCxt, yymsp[-1].minor.yy673, yymsp[0].minor.yy456); } + yymsp[-1].minor.yy456 = yylhsminor.yy456; break; case 120: /* specific_tags_opt ::= */ case 151: /* tags_def_opt ::= */ yytestcase(yyruleno==151); - case 388: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==388); - case 405: /* group_by_clause_opt ::= */ yytestcase(yyruleno==405); - case 416: /* order_by_clause_opt ::= */ yytestcase(yyruleno==416); -{ yymsp[1].minor.yy376 = NULL; } + case 392: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==392); + case 409: /* group_by_clause_opt ::= */ yytestcase(yyruleno==409); + case 420: /* order_by_clause_opt ::= */ yytestcase(yyruleno==420); +{ yymsp[1].minor.yy652 = NULL; } break; case 121: /* specific_tags_opt ::= NK_LP col_name_list NK_RP */ -{ yymsp[-2].minor.yy376 = yymsp[-1].minor.yy376; } +{ yymsp[-2].minor.yy652 = yymsp[-1].minor.yy652; } break; case 122: /* full_table_name ::= table_name */ -{ yylhsminor.yy392 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy449, NULL); } - yymsp[0].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy517, NULL); } + yymsp[0].minor.yy456 = yylhsminor.yy456; break; case 123: /* full_table_name ::= db_name NK_DOT table_name */ -{ yylhsminor.yy392 = createRealTableNode(pCxt, &yymsp[-2].minor.yy449, &yymsp[0].minor.yy449, NULL); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createRealTableNode(pCxt, &yymsp[-2].minor.yy517, &yymsp[0].minor.yy517, NULL); } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 126: /* column_def ::= column_name type_name */ -{ yylhsminor.yy392 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy449, yymsp[0].minor.yy112, NULL); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy517, yymsp[0].minor.yy380, NULL); } + yymsp[-1].minor.yy456 = yylhsminor.yy456; break; case 127: /* column_def ::= column_name type_name COMMENT NK_STRING */ -{ yylhsminor.yy392 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy449, yymsp[-2].minor.yy112, &yymsp[0].minor.yy0); } - yymsp[-3].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy517, yymsp[-2].minor.yy380, &yymsp[0].minor.yy0); } + yymsp[-3].minor.yy456 = yylhsminor.yy456; break; case 128: /* type_name ::= BOOL */ -{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_BOOL); } +{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_BOOL); } break; case 129: /* type_name ::= TINYINT */ -{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_TINYINT); } +{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_TINYINT); } break; case 130: /* type_name ::= SMALLINT */ -{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_SMALLINT); } +{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_SMALLINT); } break; case 131: /* type_name ::= INT */ case 132: /* type_name ::= INTEGER */ yytestcase(yyruleno==132); -{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_INT); } +{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_INT); } break; case 133: /* type_name ::= BIGINT */ -{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_BIGINT); } +{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_BIGINT); } break; case 134: /* type_name ::= FLOAT */ -{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_FLOAT); } +{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_FLOAT); } break; case 135: /* type_name ::= DOUBLE */ -{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_DOUBLE); } +{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_DOUBLE); } break; case 136: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy112 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy380 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } break; case 137: /* type_name ::= TIMESTAMP */ -{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } +{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } break; case 138: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy112 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy380 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } break; case 139: /* type_name ::= TINYINT UNSIGNED */ -{ yymsp[-1].minor.yy112 = createDataType(TSDB_DATA_TYPE_UTINYINT); } +{ yymsp[-1].minor.yy380 = createDataType(TSDB_DATA_TYPE_UTINYINT); } break; case 140: /* type_name ::= SMALLINT UNSIGNED */ -{ yymsp[-1].minor.yy112 = createDataType(TSDB_DATA_TYPE_USMALLINT); } +{ yymsp[-1].minor.yy380 = createDataType(TSDB_DATA_TYPE_USMALLINT); } break; case 141: /* type_name ::= INT UNSIGNED */ -{ yymsp[-1].minor.yy112 = createDataType(TSDB_DATA_TYPE_UINT); } +{ yymsp[-1].minor.yy380 = createDataType(TSDB_DATA_TYPE_UINT); } break; case 142: /* type_name ::= BIGINT UNSIGNED */ -{ yymsp[-1].minor.yy112 = createDataType(TSDB_DATA_TYPE_UBIGINT); } +{ yymsp[-1].minor.yy380 = createDataType(TSDB_DATA_TYPE_UBIGINT); } break; case 143: /* type_name ::= JSON */ -{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_JSON); } +{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_JSON); } break; case 144: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy112 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy380 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } break; case 145: /* type_name ::= MEDIUMBLOB */ -{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } +{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } break; case 146: /* type_name ::= BLOB */ -{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_BLOB); } +{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_BLOB); } break; case 147: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy112 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy380 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } break; case 148: /* type_name ::= DECIMAL */ -{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 149: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy112 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +{ yymsp[-3].minor.yy380 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 150: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ -{ yymsp[-5].minor.yy112 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +{ yymsp[-5].minor.yy380 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 152: /* tags_def_opt ::= tags_def */ - case 324: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==324); - case 379: /* select_list ::= select_sublist */ yytestcase(yyruleno==379); -{ yylhsminor.yy376 = yymsp[0].minor.yy376; } - yymsp[0].minor.yy376 = yylhsminor.yy376; + case 328: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==328); + case 383: /* select_list ::= select_sublist */ yytestcase(yyruleno==383); +{ yylhsminor.yy652 = yymsp[0].minor.yy652; } + yymsp[0].minor.yy652 = yylhsminor.yy652; break; case 153: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ -{ yymsp[-3].minor.yy376 = yymsp[-1].minor.yy376; } +{ yymsp[-3].minor.yy652 = yymsp[-1].minor.yy652; } break; case 154: /* table_options ::= */ -{ yymsp[1].minor.yy392 = createTableOptions(pCxt); } +{ yymsp[1].minor.yy456 = createTableOptions(pCxt); } break; case 155: /* table_options ::= table_options COMMENT NK_STRING */ -{ ((STableOptions*)yymsp[-2].minor.yy392)->pComments = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((STableOptions*)yymsp[-2].minor.yy456)->pComments = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 156: /* table_options ::= table_options KEEP integer_list */ case 157: /* table_options ::= table_options KEEP variable_list */ yytestcase(yyruleno==157); -{ ((STableOptions*)yymsp[-2].minor.yy392)->pKeep = yymsp[0].minor.yy376; yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((STableOptions*)yymsp[-2].minor.yy456)->pKeep = yymsp[0].minor.yy652; yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 158: /* table_options ::= table_options TTL NK_INTEGER */ -{ ((STableOptions*)yymsp[-2].minor.yy392)->pTtl = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((STableOptions*)yymsp[-2].minor.yy456)->pTtl = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 159: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ -{ ((STableOptions*)yymsp[-4].minor.yy392)->pSma = yymsp[-1].minor.yy376; yylhsminor.yy392 = yymsp[-4].minor.yy392; } - yymsp[-4].minor.yy392 = yylhsminor.yy392; +{ ((STableOptions*)yymsp[-4].minor.yy456)->pSma = yymsp[-1].minor.yy652; yylhsminor.yy456 = yymsp[-4].minor.yy456; } + yymsp[-4].minor.yy456 = yylhsminor.yy456; break; case 160: /* table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP */ -{ ((STableOptions*)yymsp[-4].minor.yy392)->pFuncs = yymsp[-1].minor.yy376; yylhsminor.yy392 = yymsp[-4].minor.yy392; } - yymsp[-4].minor.yy392 = yylhsminor.yy392; +{ ((STableOptions*)yymsp[-4].minor.yy456)->pFuncs = yymsp[-1].minor.yy652; yylhsminor.yy456 = yymsp[-4].minor.yy456; } + yymsp[-4].minor.yy456 = yylhsminor.yy456; break; case 161: /* table_options ::= table_options FILE_FACTOR NK_FLOAT */ -{ ((STableOptions*)yymsp[-2].minor.yy392)->pFilesFactor = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((STableOptions*)yymsp[-2].minor.yy456)->pFilesFactor = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 162: /* table_options ::= table_options DELAY NK_INTEGER */ -{ ((STableOptions*)yymsp[-2].minor.yy392)->pDelay = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; +{ ((STableOptions*)yymsp[-2].minor.yy456)->pDelay = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; case 163: /* alter_table_options ::= alter_table_option */ -{ yylhsminor.yy392 = createTableOptions(pCxt); yylhsminor.yy392 = setTableAlterOption(pCxt, yylhsminor.yy392, &yymsp[0].minor.yy221); } - yymsp[0].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createTableOptions(pCxt); yylhsminor.yy456 = setTableAlterOption(pCxt, yylhsminor.yy456, &yymsp[0].minor.yy145); } + yymsp[0].minor.yy456 = yylhsminor.yy456; break; case 164: /* alter_table_options ::= alter_table_options alter_table_option */ -{ yylhsminor.yy392 = setTableAlterOption(pCxt, yymsp[-1].minor.yy392, &yymsp[0].minor.yy221); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = setTableAlterOption(pCxt, yymsp[-1].minor.yy456, &yymsp[0].minor.yy145); } + yymsp[-1].minor.yy456 = yylhsminor.yy456; break; case 165: /* alter_table_option ::= COMMENT NK_STRING */ -{ yymsp[-1].minor.yy221.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy221.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy145.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy145.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } break; case 166: /* alter_table_option ::= KEEP integer_list */ case 167: /* alter_table_option ::= KEEP variable_list */ yytestcase(yyruleno==167); -{ yymsp[-1].minor.yy221.type = TABLE_OPTION_KEEP; yymsp[-1].minor.yy221.pList = yymsp[0].minor.yy376; } +{ yymsp[-1].minor.yy145.type = TABLE_OPTION_KEEP; yymsp[-1].minor.yy145.pList = yymsp[0].minor.yy652; } break; case 168: /* alter_table_option ::= TTL NK_INTEGER */ -{ yymsp[-1].minor.yy221.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy221.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy145.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy145.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } break; case 171: /* col_name ::= column_name */ -{ yylhsminor.yy392 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy449); } - yymsp[0].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy517); } + yymsp[0].minor.yy456 = yylhsminor.yy456; break; case 172: /* cmd ::= SHOW DNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT, NULL, NULL); } @@ -3311,13 +3553,13 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT, NULL, NULL); } break; case 175: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy392, yymsp[0].minor.yy392); } +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy456, yymsp[0].minor.yy456); } break; case 176: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy392, yymsp[0].minor.yy392); } +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy456, yymsp[0].minor.yy456); } break; case 177: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy392, NULL); } +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy456, NULL); } break; case 178: /* cmd ::= SHOW MNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT, NULL, NULL); } @@ -3332,7 +3574,7 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT, NULL, NULL); } break; case 182: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[-1].minor.yy392, yymsp[0].minor.yy392); } +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[-1].minor.yy456, yymsp[0].minor.yy456); } break; case 183: /* cmd ::= SHOW STREAMS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT, NULL, NULL); } @@ -3351,13 +3593,13 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCE_STMT, NULL, NULL); } break; case 189: /* cmd ::= SHOW CREATE DATABASE db_name */ -{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy449); } +{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy517); } break; case 190: /* cmd ::= SHOW CREATE TABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy392); } +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy456); } break; case 191: /* cmd ::= SHOW CREATE STABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy392); } +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy456); } break; case 192: /* cmd ::= SHOW QUERIES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT, NULL, NULL); } @@ -3382,643 +3624,658 @@ static YYACTIONTYPE yy_reduce( break; case 199: /* db_name_cond_opt ::= */ case 204: /* from_db_opt ::= */ yytestcase(yyruleno==204); -{ yymsp[1].minor.yy392 = createDefaultDatabaseCondValue(pCxt); } +{ yymsp[1].minor.yy456 = createDefaultDatabaseCondValue(pCxt); } break; case 200: /* db_name_cond_opt ::= db_name NK_DOT */ -{ yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy449); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy517); } + yymsp[-1].minor.yy456 = yylhsminor.yy456; break; case 201: /* like_pattern_opt ::= */ case 212: /* index_options ::= */ yytestcase(yyruleno==212); - case 239: /* into_opt ::= */ yytestcase(yyruleno==239); - case 386: /* where_clause_opt ::= */ yytestcase(yyruleno==386); - case 390: /* twindow_clause_opt ::= */ yytestcase(yyruleno==390); - case 395: /* sliding_opt ::= */ yytestcase(yyruleno==395); - case 397: /* fill_opt ::= */ yytestcase(yyruleno==397); - case 409: /* having_clause_opt ::= */ yytestcase(yyruleno==409); - case 418: /* slimit_clause_opt ::= */ yytestcase(yyruleno==418); - case 422: /* limit_clause_opt ::= */ yytestcase(yyruleno==422); -{ yymsp[1].minor.yy392 = NULL; } + case 243: /* into_opt ::= */ yytestcase(yyruleno==243); + case 390: /* where_clause_opt ::= */ yytestcase(yyruleno==390); + case 394: /* twindow_clause_opt ::= */ yytestcase(yyruleno==394); + case 399: /* sliding_opt ::= */ yytestcase(yyruleno==399); + case 401: /* fill_opt ::= */ yytestcase(yyruleno==401); + case 413: /* having_clause_opt ::= */ yytestcase(yyruleno==413); + case 422: /* slimit_clause_opt ::= */ yytestcase(yyruleno==422); + case 426: /* limit_clause_opt ::= */ yytestcase(yyruleno==426); +{ yymsp[1].minor.yy456 = NULL; } break; case 202: /* like_pattern_opt ::= LIKE NK_STRING */ -{ yymsp[-1].minor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } break; case 203: /* table_name_cond ::= table_name */ -{ yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy449); } - yymsp[0].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy517); } + yymsp[0].minor.yy456 = yylhsminor.yy456; break; case 205: /* from_db_opt ::= FROM db_name */ -{ yymsp[-1].minor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy449); } +{ yymsp[-1].minor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy517); } break; case 208: /* func_name ::= function_name */ -{ yylhsminor.yy392 = createFunctionNode(pCxt, &yymsp[0].minor.yy449, NULL); } - yymsp[0].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createFunctionNode(pCxt, &yymsp[0].minor.yy517, NULL); } + yymsp[0].minor.yy456 = yylhsminor.yy456; break; case 209: /* cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy89, &yymsp[-3].minor.yy449, &yymsp[-1].minor.yy449, NULL, yymsp[0].minor.yy392); } +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy673, &yymsp[-3].minor.yy517, &yymsp[-1].minor.yy517, NULL, yymsp[0].minor.yy456); } break; case 210: /* cmd ::= CREATE FULLTEXT INDEX not_exists_opt index_name ON table_name NK_LP col_name_list NK_RP */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, yymsp[-6].minor.yy89, &yymsp[-5].minor.yy449, &yymsp[-3].minor.yy449, yymsp[-1].minor.yy376, NULL); } +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, yymsp[-6].minor.yy673, &yymsp[-5].minor.yy517, &yymsp[-3].minor.yy517, yymsp[-1].minor.yy652, NULL); } break; case 211: /* cmd ::= DROP INDEX exists_opt index_name ON table_name */ -{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-3].minor.yy89, &yymsp[-2].minor.yy449, &yymsp[0].minor.yy449); } +{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-3].minor.yy673, &yymsp[-2].minor.yy517, &yymsp[0].minor.yy517); } break; case 213: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ -{ yymsp[-8].minor.yy392 = createIndexOption(pCxt, yymsp[-6].minor.yy376, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), NULL, yymsp[0].minor.yy392); } +{ yymsp[-8].minor.yy456 = createIndexOption(pCxt, yymsp[-6].minor.yy652, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), NULL, yymsp[0].minor.yy456); } break; case 214: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ -{ yymsp[-10].minor.yy392 = createIndexOption(pCxt, yymsp[-8].minor.yy376, releaseRawExprNode(pCxt, yymsp[-4].minor.yy392), releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), yymsp[0].minor.yy392); } +{ yymsp[-10].minor.yy456 = createIndexOption(pCxt, yymsp[-8].minor.yy652, releaseRawExprNode(pCxt, yymsp[-4].minor.yy456), releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), yymsp[0].minor.yy456); } break; case 217: /* func ::= function_name NK_LP expression_list NK_RP */ -{ yylhsminor.yy392 = createFunctionNode(pCxt, &yymsp[-3].minor.yy449, yymsp[-1].minor.yy376); } - yymsp[-3].minor.yy392 = yylhsminor.yy392; +{ yylhsminor.yy456 = createFunctionNode(pCxt, &yymsp[-3].minor.yy517, yymsp[-1].minor.yy652); } + yymsp[-3].minor.yy456 = yylhsminor.yy456; break; - case 218: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ -{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-3].minor.yy89, &yymsp[-2].minor.yy449, yymsp[0].minor.yy392, NULL); } + case 218: /* cmd ::= CREATE TOPIC not_exists_opt topic_name topic_options AS query_expression */ +{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-4].minor.yy673, &yymsp[-3].minor.yy517, yymsp[0].minor.yy456, NULL, yymsp[-2].minor.yy456); } break; - case 219: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name */ -{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-3].minor.yy89, &yymsp[-2].minor.yy449, NULL, &yymsp[0].minor.yy449); } + case 219: /* cmd ::= CREATE TOPIC not_exists_opt topic_name topic_options AS db_name */ +{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-4].minor.yy673, &yymsp[-3].minor.yy517, NULL, &yymsp[0].minor.yy517, yymsp[-2].minor.yy456); } break; case 220: /* cmd ::= DROP TOPIC exists_opt topic_name */ -{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy89, &yymsp[0].minor.yy449); } +{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy673, &yymsp[0].minor.yy517); } break; - case 221: /* cmd ::= DESC full_table_name */ - case 222: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==222); -{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy392); } + case 221: /* topic_options ::= */ +{ yymsp[1].minor.yy456 = createTopicOptions(pCxt); } break; - case 223: /* cmd ::= RESET QUERY CACHE */ + case 222: /* topic_options ::= topic_options WITH TABLE */ +{ ((STopicOptions*)yymsp[-2].minor.yy456)->withTable = true; yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; + break; + case 223: /* topic_options ::= topic_options WITH SCHEMA */ +{ ((STopicOptions*)yymsp[-2].minor.yy456)->withSchema = true; yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; + break; + case 224: /* topic_options ::= topic_options WITH TAG */ +{ ((STopicOptions*)yymsp[-2].minor.yy456)->withTag = true; yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; + break; + case 225: /* cmd ::= DESC full_table_name */ + case 226: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==226); +{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy456); } + break; + case 227: /* cmd ::= RESET QUERY CACHE */ { pCxt->pRootNode = createResetQueryCacheStmt(pCxt); } break; - case 224: /* cmd ::= EXPLAIN analyze_opt explain_options query_expression */ -{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy89, yymsp[-1].minor.yy392, yymsp[0].minor.yy392); } + case 228: /* cmd ::= EXPLAIN analyze_opt explain_options query_expression */ +{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy673, yymsp[-1].minor.yy456, yymsp[0].minor.yy456); } break; - case 226: /* analyze_opt ::= ANALYZE */ - case 234: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==234); - case 376: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==376); -{ yymsp[0].minor.yy89 = true; } + case 230: /* analyze_opt ::= ANALYZE */ + case 238: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==238); + case 380: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==380); +{ yymsp[0].minor.yy673 = true; } break; - case 227: /* explain_options ::= */ -{ yymsp[1].minor.yy392 = createDefaultExplainOptions(pCxt); } + case 231: /* explain_options ::= */ +{ yymsp[1].minor.yy456 = createDefaultExplainOptions(pCxt); } break; - case 228: /* explain_options ::= explain_options VERBOSE NK_BOOL */ -{ yylhsminor.yy392 = setExplainVerbose(pCxt, yymsp[-2].minor.yy392, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 232: /* explain_options ::= explain_options VERBOSE NK_BOOL */ +{ yylhsminor.yy456 = setExplainVerbose(pCxt, yymsp[-2].minor.yy456, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 229: /* explain_options ::= explain_options RATIO NK_FLOAT */ -{ yylhsminor.yy392 = setExplainRatio(pCxt, yymsp[-2].minor.yy392, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 233: /* explain_options ::= explain_options RATIO NK_FLOAT */ +{ yylhsminor.yy456 = setExplainRatio(pCxt, yymsp[-2].minor.yy456, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 230: /* cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP */ -{ pCxt->pRootNode = createCompactStmt(pCxt, yymsp[-1].minor.yy376); } + case 234: /* cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP */ +{ pCxt->pRootNode = createCompactStmt(pCxt, yymsp[-1].minor.yy652); } break; - case 231: /* cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ -{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy89, yymsp[-8].minor.yy89, &yymsp[-5].minor.yy449, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy112, yymsp[0].minor.yy4); } + case 235: /* cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ +{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy673, yymsp[-8].minor.yy673, &yymsp[-5].minor.yy517, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy380, yymsp[0].minor.yy376); } break; - case 232: /* cmd ::= DROP FUNCTION function_name */ -{ pCxt->pRootNode = createDropFunctionStmt(pCxt, &yymsp[0].minor.yy449); } + case 236: /* cmd ::= DROP FUNCTION function_name */ +{ pCxt->pRootNode = createDropFunctionStmt(pCxt, &yymsp[0].minor.yy517); } break; - case 235: /* bufsize_opt ::= */ -{ yymsp[1].minor.yy4 = 0; } + case 239: /* bufsize_opt ::= */ +{ yymsp[1].minor.yy376 = 0; } break; - case 236: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ -{ yymsp[-1].minor.yy4 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + case 240: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ +{ yymsp[-1].minor.yy376 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 237: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression */ -{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-5].minor.yy89, &yymsp[-4].minor.yy449, yymsp[-2].minor.yy392, yymsp[-3].minor.yy392, yymsp[0].minor.yy392); } + case 241: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression */ +{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-5].minor.yy673, &yymsp[-4].minor.yy517, yymsp[-2].minor.yy456, yymsp[-3].minor.yy456, yymsp[0].minor.yy456); } break; - case 238: /* cmd ::= DROP STREAM exists_opt stream_name */ -{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy89, &yymsp[0].minor.yy449); } + case 242: /* cmd ::= DROP STREAM exists_opt stream_name */ +{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy673, &yymsp[0].minor.yy517); } break; - case 240: /* into_opt ::= INTO full_table_name */ - case 357: /* from_clause ::= FROM table_reference_list */ yytestcase(yyruleno==357); - case 387: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==387); - case 410: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==410); -{ yymsp[-1].minor.yy392 = yymsp[0].minor.yy392; } + case 244: /* into_opt ::= INTO full_table_name */ + case 361: /* from_clause ::= FROM table_reference_list */ yytestcase(yyruleno==361); + case 391: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==391); + case 414: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==414); +{ yymsp[-1].minor.yy456 = yymsp[0].minor.yy456; } break; - case 241: /* stream_options ::= */ -{ yymsp[1].minor.yy392 = createStreamOptions(pCxt); } + case 245: /* stream_options ::= */ +{ yymsp[1].minor.yy456 = createStreamOptions(pCxt); } break; - case 242: /* stream_options ::= stream_options TRIGGER AT_ONCE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy392)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 246: /* stream_options ::= stream_options TRIGGER AT_ONCE */ +{ ((SStreamOptions*)yymsp[-2].minor.yy456)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 243: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy392)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 247: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ +{ ((SStreamOptions*)yymsp[-2].minor.yy456)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 244: /* stream_options ::= stream_options WATERMARK duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy392)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy392); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 248: /* stream_options ::= stream_options WATERMARK duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy456)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy456); yylhsminor.yy456 = yymsp[-2].minor.yy456; } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 245: /* cmd ::= KILL CONNECTION NK_INTEGER */ + case 249: /* cmd ::= KILL CONNECTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &yymsp[0].minor.yy0); } break; - case 246: /* cmd ::= KILL QUERY NK_INTEGER */ + case 250: /* cmd ::= KILL QUERY NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_QUERY_STMT, &yymsp[0].minor.yy0); } break; - case 247: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + case 251: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 248: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ -{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy376); } + case 252: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ +{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy652); } break; - case 249: /* cmd ::= SPLIT VGROUP NK_INTEGER */ + case 253: /* cmd ::= SPLIT VGROUP NK_INTEGER */ { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 250: /* dnode_list ::= DNODE NK_INTEGER */ -{ yymsp[-1].minor.yy376 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + case 254: /* dnode_list ::= DNODE NK_INTEGER */ +{ yymsp[-1].minor.yy652 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } break; - case 252: /* cmd ::= SYNCDB db_name REPLICA */ -{ pCxt->pRootNode = createSyncdbStmt(pCxt, &yymsp[-1].minor.yy449); } + case 256: /* cmd ::= SYNCDB db_name REPLICA */ +{ pCxt->pRootNode = createSyncdbStmt(pCxt, &yymsp[-1].minor.yy517); } break; - case 254: /* literal ::= NK_INTEGER */ -{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 258: /* literal ::= NK_INTEGER */ +{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy456 = yylhsminor.yy456; break; - case 255: /* literal ::= NK_FLOAT */ -{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 259: /* literal ::= NK_FLOAT */ +{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy456 = yylhsminor.yy456; break; - case 256: /* literal ::= NK_STRING */ -{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 260: /* literal ::= NK_STRING */ +{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy456 = yylhsminor.yy456; break; - case 257: /* literal ::= NK_BOOL */ -{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 261: /* literal ::= NK_BOOL */ +{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy456 = yylhsminor.yy456; break; - case 258: /* literal ::= TIMESTAMP NK_STRING */ -{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; + case 262: /* literal ::= TIMESTAMP NK_STRING */ +{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy456 = yylhsminor.yy456; break; - case 259: /* literal ::= duration_literal */ - case 269: /* signed_literal ::= signed */ yytestcase(yyruleno==269); - case 287: /* expression ::= literal */ yytestcase(yyruleno==287); - case 288: /* expression ::= pseudo_column */ yytestcase(yyruleno==288); - case 289: /* expression ::= column_reference */ yytestcase(yyruleno==289); - case 290: /* expression ::= function_expression */ yytestcase(yyruleno==290); - case 291: /* expression ::= subquery */ yytestcase(yyruleno==291); - case 349: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==349); - case 353: /* boolean_primary ::= predicate */ yytestcase(yyruleno==353); - case 355: /* common_expression ::= expression */ yytestcase(yyruleno==355); - case 356: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==356); - case 358: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==358); - case 360: /* table_reference ::= table_primary */ yytestcase(yyruleno==360); - case 361: /* table_reference ::= joined_table */ yytestcase(yyruleno==361); - case 365: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==365); - case 412: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==412); - case 415: /* query_primary ::= query_specification */ yytestcase(yyruleno==415); -{ yylhsminor.yy392 = yymsp[0].minor.yy392; } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 263: /* literal ::= duration_literal */ + case 273: /* signed_literal ::= signed */ yytestcase(yyruleno==273); + case 291: /* expression ::= literal */ yytestcase(yyruleno==291); + case 292: /* expression ::= pseudo_column */ yytestcase(yyruleno==292); + case 293: /* expression ::= column_reference */ yytestcase(yyruleno==293); + case 294: /* expression ::= function_expression */ yytestcase(yyruleno==294); + case 295: /* expression ::= subquery */ yytestcase(yyruleno==295); + case 353: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==353); + case 357: /* boolean_primary ::= predicate */ yytestcase(yyruleno==357); + case 359: /* common_expression ::= expression */ yytestcase(yyruleno==359); + case 360: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==360); + case 362: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==362); + case 364: /* table_reference ::= table_primary */ yytestcase(yyruleno==364); + case 365: /* table_reference ::= joined_table */ yytestcase(yyruleno==365); + case 369: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==369); + case 416: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==416); + case 419: /* query_primary ::= query_specification */ yytestcase(yyruleno==419); +{ yylhsminor.yy456 = yymsp[0].minor.yy456; } + yymsp[0].minor.yy456 = yylhsminor.yy456; break; - case 260: /* literal ::= NULL */ -{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 264: /* literal ::= NULL */ +{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy456 = yylhsminor.yy456; break; - case 261: /* literal ::= NK_QUESTION */ -{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 265: /* literal ::= NK_QUESTION */ +{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy456 = yylhsminor.yy456; break; - case 262: /* duration_literal ::= NK_VARIABLE */ -{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 266: /* duration_literal ::= NK_VARIABLE */ +{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy456 = yylhsminor.yy456; break; - case 263: /* signed ::= NK_INTEGER */ -{ yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 267: /* signed ::= NK_INTEGER */ +{ yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy456 = yylhsminor.yy456; break; - case 264: /* signed ::= NK_PLUS NK_INTEGER */ -{ yymsp[-1].minor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + case 268: /* signed ::= NK_PLUS NK_INTEGER */ +{ yymsp[-1].minor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } break; - case 265: /* signed ::= NK_MINUS NK_INTEGER */ + case 269: /* signed ::= NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); + yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; + yymsp[-1].minor.yy456 = yylhsminor.yy456; break; - case 266: /* signed ::= NK_FLOAT */ -{ yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 270: /* signed ::= NK_FLOAT */ +{ yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy456 = yylhsminor.yy456; break; - case 267: /* signed ::= NK_PLUS NK_FLOAT */ -{ yymsp[-1].minor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + case 271: /* signed ::= NK_PLUS NK_FLOAT */ +{ yymsp[-1].minor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } break; - case 268: /* signed ::= NK_MINUS NK_FLOAT */ + case 272: /* signed ::= NK_MINUS NK_FLOAT */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); + yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; + yymsp[-1].minor.yy456 = yylhsminor.yy456; break; - case 270: /* signed_literal ::= NK_STRING */ -{ yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 274: /* signed_literal ::= NK_STRING */ +{ yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy456 = yylhsminor.yy456; break; - case 271: /* signed_literal ::= NK_BOOL */ -{ yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 275: /* signed_literal ::= NK_BOOL */ +{ yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy456 = yylhsminor.yy456; break; - case 272: /* signed_literal ::= TIMESTAMP NK_STRING */ -{ yymsp[-1].minor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } + case 276: /* signed_literal ::= TIMESTAMP NK_STRING */ +{ yymsp[-1].minor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } break; - case 273: /* signed_literal ::= duration_literal */ - case 327: /* star_func_para ::= expression */ yytestcase(yyruleno==327); - case 382: /* select_item ::= common_expression */ yytestcase(yyruleno==382); - case 427: /* search_condition ::= common_expression */ yytestcase(yyruleno==427); -{ yylhsminor.yy392 = releaseRawExprNode(pCxt, yymsp[0].minor.yy392); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 277: /* signed_literal ::= duration_literal */ + case 331: /* star_func_para ::= expression */ yytestcase(yyruleno==331); + case 386: /* select_item ::= common_expression */ yytestcase(yyruleno==386); + case 431: /* search_condition ::= common_expression */ yytestcase(yyruleno==431); +{ yylhsminor.yy456 = releaseRawExprNode(pCxt, yymsp[0].minor.yy456); } + yymsp[0].minor.yy456 = yylhsminor.yy456; break; - case 274: /* signed_literal ::= NULL */ -{ yymsp[0].minor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, NULL); } + case 278: /* signed_literal ::= NULL */ +{ yymsp[0].minor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, NULL); } break; - case 292: /* expression ::= NK_LP expression NK_RP */ - case 354: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==354); -{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy392)); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 296: /* expression ::= NK_LP expression NK_RP */ + case 358: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==358); +{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy456)); } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 293: /* expression ::= NK_PLUS expression */ + case 297: /* expression ::= NK_PLUS expression */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy392)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); + yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy456)); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; + yymsp[-1].minor.yy456 = yylhsminor.yy456; break; - case 294: /* expression ::= NK_MINUS expression */ + case 298: /* expression ::= NK_MINUS expression */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy392), NULL)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); + yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy456), NULL)); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; + yymsp[-1].minor.yy456 = yylhsminor.yy456; break; - case 295: /* expression ::= expression NK_PLUS expression */ + case 299: /* expression ::= expression NK_PLUS expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); + yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 296: /* expression ::= expression NK_MINUS expression */ + case 300: /* expression ::= expression NK_MINUS expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); + yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 297: /* expression ::= expression NK_STAR expression */ + case 301: /* expression ::= expression NK_STAR expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); + yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 298: /* expression ::= expression NK_SLASH expression */ + case 302: /* expression ::= expression NK_SLASH expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); + yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 299: /* expression ::= expression NK_REM expression */ + case 303: /* expression ::= expression NK_REM expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); + yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 300: /* expression ::= column_reference NK_ARROW NK_STRING */ + case 304: /* expression ::= column_reference NK_ARROW NK_STRING */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); + yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 301: /* expression_list ::= expression */ -{ yylhsminor.yy376 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy392)); } - yymsp[0].minor.yy376 = yylhsminor.yy376; + case 305: /* expression_list ::= expression */ +{ yylhsminor.yy652 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy456)); } + yymsp[0].minor.yy652 = yylhsminor.yy652; break; - case 302: /* expression_list ::= expression_list NK_COMMA expression */ -{ yylhsminor.yy376 = addNodeToList(pCxt, yymsp[-2].minor.yy376, releaseRawExprNode(pCxt, yymsp[0].minor.yy392)); } - yymsp[-2].minor.yy376 = yylhsminor.yy376; + case 306: /* expression_list ::= expression_list NK_COMMA expression */ +{ yylhsminor.yy652 = addNodeToList(pCxt, yymsp[-2].minor.yy652, releaseRawExprNode(pCxt, yymsp[0].minor.yy456)); } + yymsp[-2].minor.yy652 = yylhsminor.yy652; break; - case 303: /* column_reference ::= column_name */ -{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy449, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy449)); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 307: /* column_reference ::= column_name */ +{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy517, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy517)); } + yymsp[0].minor.yy456 = yylhsminor.yy456; break; - case 304: /* column_reference ::= table_name NK_DOT column_name */ -{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy449, &yymsp[0].minor.yy449, createColumnNode(pCxt, &yymsp[-2].minor.yy449, &yymsp[0].minor.yy449)); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 308: /* column_reference ::= table_name NK_DOT column_name */ +{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy517, &yymsp[0].minor.yy517, createColumnNode(pCxt, &yymsp[-2].minor.yy517, &yymsp[0].minor.yy517)); } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 305: /* pseudo_column ::= ROWTS */ - case 306: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==306); - case 307: /* pseudo_column ::= QSTARTTS */ yytestcase(yyruleno==307); - case 308: /* pseudo_column ::= QENDTS */ yytestcase(yyruleno==308); - case 309: /* pseudo_column ::= WSTARTTS */ yytestcase(yyruleno==309); - case 310: /* pseudo_column ::= WENDTS */ yytestcase(yyruleno==310); - case 311: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==311); -{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 309: /* pseudo_column ::= ROWTS */ + case 310: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==310); + case 311: /* pseudo_column ::= QSTARTTS */ yytestcase(yyruleno==311); + case 312: /* pseudo_column ::= QENDTS */ yytestcase(yyruleno==312); + case 313: /* pseudo_column ::= WSTARTTS */ yytestcase(yyruleno==313); + case 314: /* pseudo_column ::= WENDTS */ yytestcase(yyruleno==314); + case 315: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==315); +{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } + yymsp[0].minor.yy456 = yylhsminor.yy456; break; - case 312: /* function_expression ::= function_name NK_LP expression_list NK_RP */ - case 313: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==313); -{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy449, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy449, yymsp[-1].minor.yy376)); } - yymsp[-3].minor.yy392 = yylhsminor.yy392; + case 316: /* function_expression ::= function_name NK_LP expression_list NK_RP */ + case 317: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==317); +{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy517, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy517, yymsp[-1].minor.yy652)); } + yymsp[-3].minor.yy456 = yylhsminor.yy456; break; - case 314: /* function_expression ::= CAST NK_LP expression AS type_name NK_RP */ -{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy392), yymsp[-1].minor.yy112)); } - yymsp[-5].minor.yy392 = yylhsminor.yy392; + case 318: /* function_expression ::= CAST NK_LP expression AS type_name NK_RP */ +{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy456), yymsp[-1].minor.yy380)); } + yymsp[-5].minor.yy456 = yylhsminor.yy456; break; - case 315: /* function_expression ::= noarg_func NK_LP NK_RP */ -{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy449, &yymsp[0].minor.yy0, createFunctionNodeNoArg(pCxt, &yymsp[-2].minor.yy449)); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 319: /* function_expression ::= noarg_func NK_LP NK_RP */ +{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy517, &yymsp[0].minor.yy0, createFunctionNodeNoArg(pCxt, &yymsp[-2].minor.yy517)); } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 323: /* star_func_para_list ::= NK_STAR */ -{ yylhsminor.yy376 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy376 = yylhsminor.yy376; + case 327: /* star_func_para_list ::= NK_STAR */ +{ yylhsminor.yy652 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy652 = yylhsminor.yy652; break; - case 328: /* star_func_para ::= table_name NK_DOT NK_STAR */ - case 385: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==385); -{ yylhsminor.yy392 = createColumnNode(pCxt, &yymsp[-2].minor.yy449, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 332: /* star_func_para ::= table_name NK_DOT NK_STAR */ + case 389: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==389); +{ yylhsminor.yy456 = createColumnNode(pCxt, &yymsp[-2].minor.yy517, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 329: /* predicate ::= expression compare_op expression */ - case 334: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==334); + case 333: /* predicate ::= expression compare_op expression */ + case 338: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==338); { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy380, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); + yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy348, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 330: /* predicate ::= expression BETWEEN expression AND expression */ + case 334: /* predicate ::= expression BETWEEN expression AND expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy392); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy392), releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy456); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); + yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy456), releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); } - yymsp[-4].minor.yy392 = yylhsminor.yy392; + yymsp[-4].minor.yy456 = yylhsminor.yy456; break; - case 331: /* predicate ::= expression NOT BETWEEN expression AND expression */ + case 335: /* predicate ::= expression NOT BETWEEN expression AND expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy392); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy392), releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy456); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); + yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy456), releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); } - yymsp[-5].minor.yy392 = yylhsminor.yy392; + yymsp[-5].minor.yy456 = yylhsminor.yy456; break; - case 332: /* predicate ::= expression IS NULL */ + case 336: /* predicate ::= expression IS NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); + yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), NULL)); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 333: /* predicate ::= expression IS NOT NULL */ + case 337: /* predicate ::= expression IS NOT NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy392), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy456); + yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy456), NULL)); } - yymsp[-3].minor.yy392 = yylhsminor.yy392; + yymsp[-3].minor.yy456 = yylhsminor.yy456; break; - case 335: /* compare_op ::= NK_LT */ -{ yymsp[0].minor.yy380 = OP_TYPE_LOWER_THAN; } + case 339: /* compare_op ::= NK_LT */ +{ yymsp[0].minor.yy348 = OP_TYPE_LOWER_THAN; } break; - case 336: /* compare_op ::= NK_GT */ -{ yymsp[0].minor.yy380 = OP_TYPE_GREATER_THAN; } + case 340: /* compare_op ::= NK_GT */ +{ yymsp[0].minor.yy348 = OP_TYPE_GREATER_THAN; } break; - case 337: /* compare_op ::= NK_LE */ -{ yymsp[0].minor.yy380 = OP_TYPE_LOWER_EQUAL; } + case 341: /* compare_op ::= NK_LE */ +{ yymsp[0].minor.yy348 = OP_TYPE_LOWER_EQUAL; } break; - case 338: /* compare_op ::= NK_GE */ -{ yymsp[0].minor.yy380 = OP_TYPE_GREATER_EQUAL; } + case 342: /* compare_op ::= NK_GE */ +{ yymsp[0].minor.yy348 = OP_TYPE_GREATER_EQUAL; } break; - case 339: /* compare_op ::= NK_NE */ -{ yymsp[0].minor.yy380 = OP_TYPE_NOT_EQUAL; } + case 343: /* compare_op ::= NK_NE */ +{ yymsp[0].minor.yy348 = OP_TYPE_NOT_EQUAL; } break; - case 340: /* compare_op ::= NK_EQ */ -{ yymsp[0].minor.yy380 = OP_TYPE_EQUAL; } + case 344: /* compare_op ::= NK_EQ */ +{ yymsp[0].minor.yy348 = OP_TYPE_EQUAL; } break; - case 341: /* compare_op ::= LIKE */ -{ yymsp[0].minor.yy380 = OP_TYPE_LIKE; } + case 345: /* compare_op ::= LIKE */ +{ yymsp[0].minor.yy348 = OP_TYPE_LIKE; } break; - case 342: /* compare_op ::= NOT LIKE */ -{ yymsp[-1].minor.yy380 = OP_TYPE_NOT_LIKE; } + case 346: /* compare_op ::= NOT LIKE */ +{ yymsp[-1].minor.yy348 = OP_TYPE_NOT_LIKE; } break; - case 343: /* compare_op ::= MATCH */ -{ yymsp[0].minor.yy380 = OP_TYPE_MATCH; } + case 347: /* compare_op ::= MATCH */ +{ yymsp[0].minor.yy348 = OP_TYPE_MATCH; } break; - case 344: /* compare_op ::= NMATCH */ -{ yymsp[0].minor.yy380 = OP_TYPE_NMATCH; } + case 348: /* compare_op ::= NMATCH */ +{ yymsp[0].minor.yy348 = OP_TYPE_NMATCH; } break; - case 345: /* compare_op ::= CONTAINS */ -{ yymsp[0].minor.yy380 = OP_TYPE_JSON_CONTAINS; } + case 349: /* compare_op ::= CONTAINS */ +{ yymsp[0].minor.yy348 = OP_TYPE_JSON_CONTAINS; } break; - case 346: /* in_op ::= IN */ -{ yymsp[0].minor.yy380 = OP_TYPE_IN; } + case 350: /* in_op ::= IN */ +{ yymsp[0].minor.yy348 = OP_TYPE_IN; } break; - case 347: /* in_op ::= NOT IN */ -{ yymsp[-1].minor.yy380 = OP_TYPE_NOT_IN; } + case 351: /* in_op ::= NOT IN */ +{ yymsp[-1].minor.yy348 = OP_TYPE_NOT_IN; } break; - case 348: /* in_predicate_value ::= NK_LP expression_list NK_RP */ -{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy376)); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 352: /* in_predicate_value ::= NK_LP expression_list NK_RP */ +{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy652)); } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 350: /* boolean_value_expression ::= NOT boolean_primary */ + case 354: /* boolean_value_expression ::= NOT boolean_primary */ { - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy392), NULL)); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); + yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy456), NULL)); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; + yymsp[-1].minor.yy456 = yylhsminor.yy456; break; - case 351: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + case 355: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); + yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 352: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + case 356: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); + yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 359: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -{ yylhsminor.yy392 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy392, yymsp[0].minor.yy392, NULL); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 363: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ yylhsminor.yy456 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy456, yymsp[0].minor.yy456, NULL); } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 362: /* table_primary ::= table_name alias_opt */ -{ yylhsminor.yy392 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy449, &yymsp[0].minor.yy449); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; + case 366: /* table_primary ::= table_name alias_opt */ +{ yylhsminor.yy456 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy517, &yymsp[0].minor.yy517); } + yymsp[-1].minor.yy456 = yylhsminor.yy456; break; - case 363: /* table_primary ::= db_name NK_DOT table_name alias_opt */ -{ yylhsminor.yy392 = createRealTableNode(pCxt, &yymsp[-3].minor.yy449, &yymsp[-1].minor.yy449, &yymsp[0].minor.yy449); } - yymsp[-3].minor.yy392 = yylhsminor.yy392; + case 367: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ yylhsminor.yy456 = createRealTableNode(pCxt, &yymsp[-3].minor.yy517, &yymsp[-1].minor.yy517, &yymsp[0].minor.yy517); } + yymsp[-3].minor.yy456 = yylhsminor.yy456; break; - case 364: /* table_primary ::= subquery alias_opt */ -{ yylhsminor.yy392 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy392), &yymsp[0].minor.yy449); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; + case 368: /* table_primary ::= subquery alias_opt */ +{ yylhsminor.yy456 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy456), &yymsp[0].minor.yy517); } + yymsp[-1].minor.yy456 = yylhsminor.yy456; break; - case 366: /* alias_opt ::= */ -{ yymsp[1].minor.yy449 = nil_token; } + case 370: /* alias_opt ::= */ +{ yymsp[1].minor.yy517 = nil_token; } break; - case 367: /* alias_opt ::= table_alias */ -{ yylhsminor.yy449 = yymsp[0].minor.yy449; } - yymsp[0].minor.yy449 = yylhsminor.yy449; + case 371: /* alias_opt ::= table_alias */ +{ yylhsminor.yy517 = yymsp[0].minor.yy517; } + yymsp[0].minor.yy517 = yylhsminor.yy517; break; - case 368: /* alias_opt ::= AS table_alias */ -{ yymsp[-1].minor.yy449 = yymsp[0].minor.yy449; } + case 372: /* alias_opt ::= AS table_alias */ +{ yymsp[-1].minor.yy517 = yymsp[0].minor.yy517; } break; - case 369: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - case 370: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==370); -{ yymsp[-2].minor.yy392 = yymsp[-1].minor.yy392; } + case 373: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + case 374: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==374); +{ yymsp[-2].minor.yy456 = yymsp[-1].minor.yy456; } break; - case 371: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -{ yylhsminor.yy392 = createJoinTableNode(pCxt, yymsp[-4].minor.yy372, yymsp[-5].minor.yy392, yymsp[-2].minor.yy392, yymsp[0].minor.yy392); } - yymsp[-5].minor.yy392 = yylhsminor.yy392; + case 375: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ yylhsminor.yy456 = createJoinTableNode(pCxt, yymsp[-4].minor.yy684, yymsp[-5].minor.yy456, yymsp[-2].minor.yy456, yymsp[0].minor.yy456); } + yymsp[-5].minor.yy456 = yylhsminor.yy456; break; - case 372: /* join_type ::= */ -{ yymsp[1].minor.yy372 = JOIN_TYPE_INNER; } + case 376: /* join_type ::= */ +{ yymsp[1].minor.yy684 = JOIN_TYPE_INNER; } break; - case 373: /* join_type ::= INNER */ -{ yymsp[0].minor.yy372 = JOIN_TYPE_INNER; } + case 377: /* join_type ::= INNER */ +{ yymsp[0].minor.yy684 = JOIN_TYPE_INNER; } break; - case 374: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 378: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { - yymsp[-8].minor.yy392 = createSelectStmt(pCxt, yymsp[-7].minor.yy89, yymsp[-6].minor.yy376, yymsp[-5].minor.yy392); - yymsp[-8].minor.yy392 = addWhereClause(pCxt, yymsp[-8].minor.yy392, yymsp[-4].minor.yy392); - yymsp[-8].minor.yy392 = addPartitionByClause(pCxt, yymsp[-8].minor.yy392, yymsp[-3].minor.yy376); - yymsp[-8].minor.yy392 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy392, yymsp[-2].minor.yy392); - yymsp[-8].minor.yy392 = addGroupByClause(pCxt, yymsp[-8].minor.yy392, yymsp[-1].minor.yy376); - yymsp[-8].minor.yy392 = addHavingClause(pCxt, yymsp[-8].minor.yy392, yymsp[0].minor.yy392); + yymsp[-8].minor.yy456 = createSelectStmt(pCxt, yymsp[-7].minor.yy673, yymsp[-6].minor.yy652, yymsp[-5].minor.yy456); + yymsp[-8].minor.yy456 = addWhereClause(pCxt, yymsp[-8].minor.yy456, yymsp[-4].minor.yy456); + yymsp[-8].minor.yy456 = addPartitionByClause(pCxt, yymsp[-8].minor.yy456, yymsp[-3].minor.yy652); + yymsp[-8].minor.yy456 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy456, yymsp[-2].minor.yy456); + yymsp[-8].minor.yy456 = addGroupByClause(pCxt, yymsp[-8].minor.yy456, yymsp[-1].minor.yy652); + yymsp[-8].minor.yy456 = addHavingClause(pCxt, yymsp[-8].minor.yy456, yymsp[0].minor.yy456); } break; - case 377: /* set_quantifier_opt ::= ALL */ -{ yymsp[0].minor.yy89 = false; } + case 381: /* set_quantifier_opt ::= ALL */ +{ yymsp[0].minor.yy673 = false; } break; - case 378: /* select_list ::= NK_STAR */ -{ yymsp[0].minor.yy376 = NULL; } + case 382: /* select_list ::= NK_STAR */ +{ yymsp[0].minor.yy652 = NULL; } break; - case 383: /* select_item ::= common_expression column_alias */ -{ yylhsminor.yy392 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy392), &yymsp[0].minor.yy449); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; + case 387: /* select_item ::= common_expression column_alias */ +{ yylhsminor.yy456 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy456), &yymsp[0].minor.yy517); } + yymsp[-1].minor.yy456 = yylhsminor.yy456; break; - case 384: /* select_item ::= common_expression AS column_alias */ -{ yylhsminor.yy392 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), &yymsp[0].minor.yy449); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 388: /* select_item ::= common_expression AS column_alias */ +{ yylhsminor.yy456 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), &yymsp[0].minor.yy517); } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 389: /* partition_by_clause_opt ::= PARTITION BY expression_list */ - case 406: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==406); - case 417: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==417); -{ yymsp[-2].minor.yy376 = yymsp[0].minor.yy376; } + case 393: /* partition_by_clause_opt ::= PARTITION BY expression_list */ + case 410: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==410); + case 421: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==421); +{ yymsp[-2].minor.yy652 = yymsp[0].minor.yy652; } break; - case 391: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ -{ yymsp[-5].minor.yy392 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy392), releaseRawExprNode(pCxt, yymsp[-1].minor.yy392)); } + case 395: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ +{ yymsp[-5].minor.yy456 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy456), releaseRawExprNode(pCxt, yymsp[-1].minor.yy456)); } break; - case 392: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ -{ yymsp[-3].minor.yy392 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy392)); } + case 396: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ +{ yymsp[-3].minor.yy456 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy456)); } break; - case 393: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-5].minor.yy392 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy392), NULL, yymsp[-1].minor.yy392, yymsp[0].minor.yy392); } + case 397: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-5].minor.yy456 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy456), NULL, yymsp[-1].minor.yy456, yymsp[0].minor.yy456); } break; - case 394: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-7].minor.yy392 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy392), releaseRawExprNode(pCxt, yymsp[-3].minor.yy392), yymsp[-1].minor.yy392, yymsp[0].minor.yy392); } + case 398: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-7].minor.yy456 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy456), releaseRawExprNode(pCxt, yymsp[-3].minor.yy456), yymsp[-1].minor.yy456, yymsp[0].minor.yy456); } break; - case 396: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ -{ yymsp[-3].minor.yy392 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy392); } + case 400: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ +{ yymsp[-3].minor.yy456 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy456); } break; - case 398: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -{ yymsp[-3].minor.yy392 = createFillNode(pCxt, yymsp[-1].minor.yy102, NULL); } + case 402: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ yymsp[-3].minor.yy456 = createFillNode(pCxt, yymsp[-1].minor.yy534, NULL); } break; - case 399: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ -{ yymsp[-5].minor.yy392 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy376)); } + case 403: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ +{ yymsp[-5].minor.yy456 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy652)); } break; - case 400: /* fill_mode ::= NONE */ -{ yymsp[0].minor.yy102 = FILL_MODE_NONE; } + case 404: /* fill_mode ::= NONE */ +{ yymsp[0].minor.yy534 = FILL_MODE_NONE; } break; - case 401: /* fill_mode ::= PREV */ -{ yymsp[0].minor.yy102 = FILL_MODE_PREV; } + case 405: /* fill_mode ::= PREV */ +{ yymsp[0].minor.yy534 = FILL_MODE_PREV; } break; - case 402: /* fill_mode ::= NULL */ -{ yymsp[0].minor.yy102 = FILL_MODE_NULL; } + case 406: /* fill_mode ::= NULL */ +{ yymsp[0].minor.yy534 = FILL_MODE_NULL; } break; - case 403: /* fill_mode ::= LINEAR */ -{ yymsp[0].minor.yy102 = FILL_MODE_LINEAR; } + case 407: /* fill_mode ::= LINEAR */ +{ yymsp[0].minor.yy534 = FILL_MODE_LINEAR; } break; - case 404: /* fill_mode ::= NEXT */ -{ yymsp[0].minor.yy102 = FILL_MODE_NEXT; } + case 408: /* fill_mode ::= NEXT */ +{ yymsp[0].minor.yy534 = FILL_MODE_NEXT; } break; - case 407: /* group_by_list ::= expression */ -{ yylhsminor.yy376 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); } - yymsp[0].minor.yy376 = yylhsminor.yy376; + case 411: /* group_by_list ::= expression */ +{ yylhsminor.yy652 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); } + yymsp[0].minor.yy652 = yylhsminor.yy652; break; - case 408: /* group_by_list ::= group_by_list NK_COMMA expression */ -{ yylhsminor.yy376 = addNodeToList(pCxt, yymsp[-2].minor.yy376, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); } - yymsp[-2].minor.yy376 = yylhsminor.yy376; + case 412: /* group_by_list ::= group_by_list NK_COMMA expression */ +{ yylhsminor.yy652 = addNodeToList(pCxt, yymsp[-2].minor.yy652, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); } + yymsp[-2].minor.yy652 = yylhsminor.yy652; break; - case 411: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 415: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ { - yylhsminor.yy392 = addOrderByClause(pCxt, yymsp[-3].minor.yy392, yymsp[-2].minor.yy376); - yylhsminor.yy392 = addSlimitClause(pCxt, yylhsminor.yy392, yymsp[-1].minor.yy392); - yylhsminor.yy392 = addLimitClause(pCxt, yylhsminor.yy392, yymsp[0].minor.yy392); + yylhsminor.yy456 = addOrderByClause(pCxt, yymsp[-3].minor.yy456, yymsp[-2].minor.yy652); + yylhsminor.yy456 = addSlimitClause(pCxt, yylhsminor.yy456, yymsp[-1].minor.yy456); + yylhsminor.yy456 = addLimitClause(pCxt, yylhsminor.yy456, yymsp[0].minor.yy456); } - yymsp[-3].minor.yy392 = yylhsminor.yy392; + yymsp[-3].minor.yy456 = yylhsminor.yy456; break; - case 413: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ -{ yylhsminor.yy392 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy392, yymsp[0].minor.yy392); } - yymsp[-3].minor.yy392 = yylhsminor.yy392; + case 417: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ +{ yylhsminor.yy456 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy456, yymsp[0].minor.yy456); } + yymsp[-3].minor.yy456 = yylhsminor.yy456; break; - case 414: /* query_expression_body ::= query_expression_body UNION query_expression_body */ -{ yylhsminor.yy392 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy392, yymsp[0].minor.yy392); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 418: /* query_expression_body ::= query_expression_body UNION query_expression_body */ +{ yylhsminor.yy456 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy456, yymsp[0].minor.yy456); } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 419: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 423: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==423); -{ yymsp[-1].minor.yy392 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + case 423: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 427: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==427); +{ yymsp[-1].minor.yy456 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 420: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 424: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==424); -{ yymsp[-3].minor.yy392 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 424: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 428: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==428); +{ yymsp[-3].minor.yy456 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 421: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 425: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==425); -{ yymsp[-3].minor.yy392 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 425: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 429: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==429); +{ yymsp[-3].minor.yy456 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 426: /* subquery ::= NK_LP query_expression NK_RP */ -{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy392); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 430: /* subquery ::= NK_LP query_expression NK_RP */ +{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy456); } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 430: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ -{ yylhsminor.yy392 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), yymsp[-1].minor.yy386, yymsp[0].minor.yy361); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 434: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ +{ yylhsminor.yy456 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), yymsp[-1].minor.yy250, yymsp[0].minor.yy645); } + yymsp[-2].minor.yy456 = yylhsminor.yy456; break; - case 431: /* ordering_specification_opt ::= */ -{ yymsp[1].minor.yy386 = ORDER_ASC; } + case 435: /* ordering_specification_opt ::= */ +{ yymsp[1].minor.yy250 = ORDER_ASC; } break; - case 432: /* ordering_specification_opt ::= ASC */ -{ yymsp[0].minor.yy386 = ORDER_ASC; } + case 436: /* ordering_specification_opt ::= ASC */ +{ yymsp[0].minor.yy250 = ORDER_ASC; } break; - case 433: /* ordering_specification_opt ::= DESC */ -{ yymsp[0].minor.yy386 = ORDER_DESC; } + case 437: /* ordering_specification_opt ::= DESC */ +{ yymsp[0].minor.yy250 = ORDER_DESC; } break; - case 434: /* null_ordering_opt ::= */ -{ yymsp[1].minor.yy361 = NULL_ORDER_DEFAULT; } + case 438: /* null_ordering_opt ::= */ +{ yymsp[1].minor.yy645 = NULL_ORDER_DEFAULT; } break; - case 435: /* null_ordering_opt ::= NULLS FIRST */ -{ yymsp[-1].minor.yy361 = NULL_ORDER_FIRST; } + case 439: /* null_ordering_opt ::= NULLS FIRST */ +{ yymsp[-1].minor.yy645 = NULL_ORDER_FIRST; } break; - case 436: /* null_ordering_opt ::= NULLS LAST */ -{ yymsp[-1].minor.yy361 = NULL_ORDER_LAST; } + case 440: /* null_ordering_opt ::= NULLS LAST */ +{ yymsp[-1].minor.yy645 = NULL_ORDER_LAST; } break; default: break; diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h index 144254b042..7f8bf14499 100644 --- a/source/libs/planner/inc/planInt.h +++ b/source/libs/planner/inc/planInt.h @@ -22,6 +22,10 @@ extern "C" { #include "planner.h" +#define QUERY_POLICY_VNODE 1 +#define QUERY_POLICY_HYBRID 2 +#define QUERY_POLICY_QNODE 3 + #define planFatal(param, ...) qFatal("PLAN: " param, __VA_ARGS__) #define planError(param, ...) qError("PLAN: " param, __VA_ARGS__) #define planWarn(param, ...) qWarn("PLAN: " param, __VA_ARGS__) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index b13c0a8888..2c7ad27927 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -639,8 +639,8 @@ static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSel } if (NULL != pSelect->pLimit) { - pProject->limit = ((SLimitNode*)pSelect->pLimit)->limit; - pProject->offset = ((SLimitNode*)pSelect->pLimit)->offset; + pProject->limit = pSelect->pLimit->limit; + pProject->offset = pSelect->pLimit->offset; } else { pProject->limit = -1; pProject->offset = -1; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 745029730f..d5e12608ac 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -16,6 +16,8 @@ #include "planInt.h" #include "functionMgt.h" +#include "tglobal.h" +#include "catalog.h" typedef struct SSlotIdInfo { int16_t slotId; @@ -1233,7 +1235,13 @@ int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryP return TSDB_CODE_OUT_OF_MEMORY; } - int32_t code = doCreatePhysiPlan(&cxt, pLogicPlan, pPlan); + int32_t code = TSDB_CODE_SUCCESS; + if (tsQueryPolicy > QUERY_POLICY_VNODE) { + code = catalogGetQnodeList(pCxt->pCatalog, pCxt->pTransporter, &pCxt->mgmtEpSet, pExecNodeList); + } + if (TSDB_CODE_SUCCESS == code) { + code = doCreatePhysiPlan(&cxt, pLogicPlan, pPlan); + } if (TSDB_CODE_SUCCESS == code) { setExplainInfo(pCxt, *pPlan); } From 395ba3dc5fe1891991df8d4ad26a3e537be7fb5e Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 22 Apr 2022 18:38:38 +0800 Subject: [PATCH 72/83] fix: some syntax parsing problems --- tests/script/tsim/db/alter_option.sim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 2b230c3c45..57324367b5 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -183,10 +183,10 @@ sql_error alter database db days 0 sql_error alter database db days 14400 # set over than keep print ============== modify keep -sql alter database db keep 2000 +sql alter database db keep 3456000 sql show databases print keep $data7_db -if $data7_db != 2000,2000,2000 then +if $data7_db != 3456000,3456000,3456000 then return -1 endi From d53b3a4480cf18b322a6303c747768294a517c8a Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 22 Apr 2022 18:41:32 +0800 Subject: [PATCH 73/83] call syncCleanUp taosMemoryFree(pSyncEnv) --- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 3 +++ source/libs/sync/src/syncEnv.c | 1 + 2 files changed, 4 insertions(+) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 77171e8b4b..ffbc6eb3b9 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -273,6 +273,9 @@ static void vmCleanup(SMgmtWrapper *pWrapper) { // walCleanUp(); taosMemoryFree(pMgmt); pWrapper->pMgmt = NULL; + + syncCleanUp(); + dInfo("vnode-mgmt is cleaned up"); } diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index b4cbe493f2..0e04793a83 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -92,6 +92,7 @@ static SSyncEnv *doSyncEnvStart() { static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) { taosTmrCleanUp(pSyncEnv->pTimerManager); + taosMemoryFree(pSyncEnv); return 0; } From 60c9ec38ace2da6af22ddda858fe9bed827a49e7 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 22 Apr 2022 18:47:00 +0800 Subject: [PATCH 74/83] [test: add sleep for elect leader] --- tests/system-test/2-query/cast.py | 2 ++ tests/system-test/2-query/distinct.py | 2 ++ tests/system-test/2-query/varchar.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/tests/system-test/2-query/cast.py b/tests/system-test/2-query/cast.py index f09e7d1f63..86319ae25d 100644 --- a/tests/system-test/2-query/cast.py +++ b/tests/system-test/2-query/cast.py @@ -2,6 +2,7 @@ import taos import sys import datetime import inspect +import time from util.log import * from util.sql import * @@ -19,6 +20,7 @@ class TDTestCase: tdSql.prepare() tdLog.printNoPrefix("==========step1:create table") + time.sleep(2) tdSql.execute( '''create table stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) diff --git a/tests/system-test/2-query/distinct.py b/tests/system-test/2-query/distinct.py index a82f3a6f59..43d48a19cb 100644 --- a/tests/system-test/2-query/distinct.py +++ b/tests/system-test/2-query/distinct.py @@ -1,5 +1,6 @@ import taos import sys +import time from util.log import * from util.sql import * @@ -15,6 +16,7 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() + time.sleep(2) tdLog.printNoPrefix("==========step1:create table") tdSql.execute("create stable db.stb1 (ts timestamp, c1 int, c2 int) tags(t0 tinyint, t1 int, t2 int)") diff --git a/tests/system-test/2-query/varchar.py b/tests/system-test/2-query/varchar.py index 5cc6c8e399..f0e3d36ffd 100644 --- a/tests/system-test/2-query/varchar.py +++ b/tests/system-test/2-query/varchar.py @@ -1,5 +1,6 @@ import taos import sys +import time from util.log import * from util.sql import * @@ -17,6 +18,7 @@ class TDTestCase: tdSql.prepare() tdLog.printNoPrefix("==========step1:create table") + time.sleep(2) tdSql.execute( '''create table stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 varchar(16),c9 nchar(32), c10 timestamp) From 09efd7817663745a4c9e3ff25a195419b05fe529 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 22 Apr 2022 18:48:22 +0800 Subject: [PATCH 75/83] [test: add sleep for elect leader] --- tests/system-test/2-query/cast.py | 2 -- tests/system-test/2-query/distinct.py | 2 -- tests/system-test/2-query/varchar.py | 2 -- 3 files changed, 6 deletions(-) diff --git a/tests/system-test/2-query/cast.py b/tests/system-test/2-query/cast.py index 86319ae25d..f09e7d1f63 100644 --- a/tests/system-test/2-query/cast.py +++ b/tests/system-test/2-query/cast.py @@ -2,7 +2,6 @@ import taos import sys import datetime import inspect -import time from util.log import * from util.sql import * @@ -20,7 +19,6 @@ class TDTestCase: tdSql.prepare() tdLog.printNoPrefix("==========step1:create table") - time.sleep(2) tdSql.execute( '''create table stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) diff --git a/tests/system-test/2-query/distinct.py b/tests/system-test/2-query/distinct.py index 43d48a19cb..a82f3a6f59 100644 --- a/tests/system-test/2-query/distinct.py +++ b/tests/system-test/2-query/distinct.py @@ -1,6 +1,5 @@ import taos import sys -import time from util.log import * from util.sql import * @@ -16,7 +15,6 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() - time.sleep(2) tdLog.printNoPrefix("==========step1:create table") tdSql.execute("create stable db.stb1 (ts timestamp, c1 int, c2 int) tags(t0 tinyint, t1 int, t2 int)") diff --git a/tests/system-test/2-query/varchar.py b/tests/system-test/2-query/varchar.py index f0e3d36ffd..5cc6c8e399 100644 --- a/tests/system-test/2-query/varchar.py +++ b/tests/system-test/2-query/varchar.py @@ -1,6 +1,5 @@ import taos import sys -import time from util.log import * from util.sql import * @@ -18,7 +17,6 @@ class TDTestCase: tdSql.prepare() tdLog.printNoPrefix("==========step1:create table") - time.sleep(2) tdSql.execute( '''create table stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 varchar(16),c9 nchar(32), c10 timestamp) From b790102ffd32bb2924bf5efa1573c042ecb50946 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 22 Apr 2022 18:50:21 +0800 Subject: [PATCH 76/83] [test: add sleep for elect leader] --- tests/pytest/util/sql.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 9fe4ffa0d4..4a3ccff08a 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -60,6 +60,7 @@ class TDSql: self.cursor.execute(s) s = 'use db' self.cursor.execute(s) + time.sleep(2) def error(self, sql): expectErrNotOccured = True From 51fab8059520a98a96d16b65fbf1b31855766556 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 22 Apr 2022 19:10:28 +0800 Subject: [PATCH 77/83] [test: modify valgrind error check] --- tests/script/jenkins/basic.txt | 2 +- tests/script/sh/checkValgrind.sh | 3 ++- tests/script/tsim/valgrind/checkError.sim | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index a226e90094..51b188c8db 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -6,7 +6,7 @@ # ---- db ./test.sh -f tsim/db/create_all_options.sim -./test.sh -f tsim/db/alter_option.sim +#./test.sh -f tsim/db/alter_option.sim ./test.sh -f tsim/db/basic1.sim ./test.sh -f tsim/db/basic2.sim ./test.sh -f tsim/db/basic3.sim diff --git a/tests/script/sh/checkValgrind.sh b/tests/script/sh/checkValgrind.sh index 17d7ce1aff..e3afb10752 100755 --- a/tests/script/sh/checkValgrind.sh +++ b/tests/script/sh/checkValgrind.sh @@ -34,5 +34,6 @@ LOG_DIR=$TAOS_DIR/sim/$NODE_NAME/log #echo ---- $LOG_DIR -errors=`grep "ERROR SUMMARY:" ${LOG_DIR}/valgrind-taosd-*.log | cut -d ' ' -f 2,3,4,5 | tr -d "\n"` +#errors=`grep "ERROR SUMMARY:" ${LOG_DIR}/valgrind-taosd-*.log | cut -d ' ' -f 2,3,4,5 | tr -d "\n"` +errors=`cat ${LOG_DIR}/valgrind-taosd-*.log | grep "ERROR SUMMARY:" | awk '{print $4}' | awk '{sum+=$1}END{print sum}'` echo $errors diff --git a/tests/script/tsim/valgrind/checkError.sim b/tests/script/tsim/valgrind/checkError.sim index 80700dcbce..38f45d405c 100644 --- a/tests/script/tsim/valgrind/checkError.sim +++ b/tests/script/tsim/valgrind/checkError.sim @@ -71,7 +71,7 @@ print ====> start to check if there are ERRORS in vagrind log file for each dnod # -n : dnode[x] be check system_content sh/checkValgrind.sh -n dnode1 print cmd return result----> [ $system_content ] -if $system_content == @ERROR SUMMARY: 0 errors@ then +if $system_content == 0 then return 0 endi @@ -80,4 +80,4 @@ if $system_content == $null then return 0 endi -return -1 \ No newline at end of file +return -1 From 11cb0afa300693105d45ba9d9ea921c3b176b3cc Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 22 Apr 2022 19:28:27 +0800 Subject: [PATCH 78/83] feature/qnode --- include/common/tmsg.h | 8 +++++++- include/libs/qcom/query.h | 6 ------ source/common/src/tmsg.c | 27 +++++++++++++++++++------- source/dnode/mnode/impl/src/mndQnode.c | 14 +++++++------ source/libs/qcom/src/querymsg.c | 4 ++-- 5 files changed, 37 insertions(+), 22 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 5ea53b5510..d33244a3dd 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -639,13 +639,19 @@ int32_t tSerializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq); int32_t tDeserializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq); typedef struct { - SArray* epSetList; // SArray + SArray* addrsList; // SArray } SQnodeListRsp; int32_t tSerializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp); int32_t tDeserializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp); void tFreeSQnodeListRsp(SQnodeListRsp* pRsp); +typedef struct SQueryNodeAddr { + int32_t nodeId; // vgId or qnodeId + SEpSet epSet; +} SQueryNodeAddr; + + typedef struct { SArray* pArray; // Array of SUseDbRsp } SUseDbBatchRsp; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index bb550e75e8..e50276cc69 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -129,12 +129,6 @@ typedef struct SMsgSendInfo { SDataBuf msgInfo; } SMsgSendInfo; -typedef struct SQueryNodeAddr { - int32_t nodeId; // vgId or qnodeId - SEpSet epSet; -} SQueryNodeAddr; - - typedef struct SQueryNodeStat { int32_t tableNum; // vg table number, unit is TSDB_TABLE_NUM_UNIT } SQueryNodeStat; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 0ea196f1a1..cfb82b9f41 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -114,6 +114,19 @@ int32_t tDecodeSEpSet(SCoder *pDecoder, SEpSet *pEp) { return 0; } +int32_t tEncodeSQueryNodeAddr(SCoder *pEncoder, SQueryNodeAddr *pAddr) { + if (tEncodeI32(pEncoder, pAddr->nodeId) < 0) return -1; + if (tEncodeSEpSet(pEncoder, &pAddr->epSet) < 0) return -1; + return 0; +} + +int32_t tDecodeSQueryNodeAddr(SCoder *pDecoder, SQueryNodeAddr *pAddr) { + if (tDecodeI32(pDecoder, &pAddr->nodeId) < 0) return -1; + if (tDecodeSEpSet(pDecoder, &pAddr->epSet) < 0) return -1; + return 0; +} + + int32_t taosEncodeSEpSet(void **buf, const SEpSet *pEp) { int32_t tlen = 0; tlen += taosEncodeFixedI8(buf, pEp->inUse); @@ -2058,11 +2071,11 @@ int32_t tSerializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp) tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); if (tStartEncode(&encoder) < 0) return -1; - int32_t num = taosArrayGetSize(pRsp->epSetList); + int32_t num = taosArrayGetSize(pRsp->addrsList); if (tEncodeI32(&encoder, num) < 0) return -1; for (int32_t i = 0; i < num; ++i) { - SEpSet *epSet = taosArrayGet(pRsp->epSetList, i); - if (tEncodeSEpSet(&encoder, epSet) < 0) return -1; + SQueryNodeAddr *addr = taosArrayGet(pRsp->addrsList, i); + if (tEncodeSQueryNodeAddr(&encoder, addr) < 0) return -1; } tEndEncode(&encoder); @@ -2078,10 +2091,10 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp if (tStartDecode(&decoder) < 0) return -1; int32_t num = 0; if (tDecodeI32(&decoder, &num) < 0) return -1; - pRsp->epSetList = taosArrayInit(num, sizeof(SEpSet)); - if (NULL == pRsp->epSetList) return -1; + pRsp->addrsList = taosArrayInit(num, sizeof(SQueryNodeAddr)); + if (NULL == pRsp->addrsList) return -1; for (int32_t i = 0; i < num; ++i) { - if (tDecodeSEpSet(&decoder, TARRAY_GET_ELEM(pRsp->epSetList, i)) < 0) return -1; + if (tDecodeSQueryNodeAddr(&decoder, TARRAY_GET_ELEM(pRsp->addrsList, i)) < 0) return -1; } tEndDecode(&decoder); @@ -2089,7 +2102,7 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp return 0; } -void tFreeSQnodeListRsp(SQnodeListRsp *pRsp) { taosArrayDestroy(pRsp->epSetList); } +void tFreeSQnodeListRsp(SQnodeListRsp *pRsp) { taosArrayDestroy(pRsp->addrsList); } int32_t tSerializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq) { SCoder encoder = {0}; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 96dba9baf6..1c03ca30f4 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -444,8 +444,8 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) { goto _OVER; } - qlistRsp.epSetList = taosArrayInit(5, sizeof(SEpSet)); - if (NULL == qlistRsp.epSetList) { + qlistRsp.addrsList = taosArrayInit(5, sizeof(SQueryNodeAddr)); + if (NULL == qlistRsp.addrsList) { mError("failed to alloc epSet while process qnode list req"); terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; @@ -455,11 +455,13 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) { void *pIter = sdbFetch(pSdb, SDB_QNODE, NULL, (void **)&pObj); if (pIter == NULL) break; - SEpSet epSet = {.numOfEps = 1}; - tstrncpy(epSet.eps[0].fqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN); - epSet.eps[0].port = pObj->pDnode->port; + SQueryNodeAddr nodeAddr = {0}; + nodeAddr.nodeId = QNODE_HANDLE; + nodeAddr.epSet.numOfEps = 1; + tstrncpy(nodeAddr.epSet.eps[0].fqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN); + nodeAddr.epSet.eps[0].port = pObj->pDnode->port; - (void)taosArrayPush(qlistRsp.epSetList, &epSet); + (void)taosArrayPush(qlistRsp.addrsList, &nodeAddr); numOfRows++; sdbRelease(pSdb, pObj); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 9e37784cab..de805d2c77 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -359,10 +359,10 @@ PROCESS_QLIST_OVER: if (code != 0) { tFreeSQnodeListRsp(&out); - out.epSetList = NULL; + out.addrsList = NULL; } - *(SArray **)output = out.epSetList; + *(SArray **)output = out.addrsList; return code; } From 32e988766478b1caea3dab9883899860805a1c03 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 22 Apr 2022 19:47:00 +0800 Subject: [PATCH 79/83] feat(tmq): add push mode --- example/src/tmq.c | 2 - include/client/taos.h | 2 + include/os/osSemaphore.h | 45 +++--- include/util/tcache.h | 16 +- source/client/src/tmq.c | 12 +- source/dnode/mnode/impl/src/mndPerfSchema.c | 2 + source/dnode/mnode/impl/src/mndProfile.c | 65 ++++---- source/dnode/mnode/impl/src/mndShow.c | 2 +- source/dnode/mnode/impl/src/mndTopic.c | 4 +- source/dnode/vnode/src/inc/tq.h | 39 +++-- source/dnode/vnode/src/tq/tq.c | 167 +++++++++++++++++++- source/os/src/osSemaphore.c | 78 ++++++--- source/util/src/tcache.c | 144 +++++++++-------- source/util/test/cacheTest.cpp | 86 +++++----- 14 files changed, 441 insertions(+), 223 deletions(-) diff --git a/example/src/tmq.c b/example/src/tmq.c index 46f57799e7..2abf915fd8 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -101,8 +101,6 @@ int32_t create_topic() { } taos_free_result(pRes); - /*const char* sql = "select * from tu1";*/ - /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));*/ /*pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");*/ pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from ct1"); if (taos_errno(pRes) != 0) { diff --git a/include/client/taos.h b/include/client/taos.h index 3d139ce6d2..30e99e202c 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -245,6 +245,8 @@ typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *)); DLL_EXPORT tmq_list_t *tmq_list_new(); DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); DLL_EXPORT void tmq_list_destroy(tmq_list_t *); +DLL_EXPORT int32_t tmq_list_get_size(const tmq_list_t *); +DLL_EXPORT char **tmq_list_to_c_array(const tmq_list_t *); #if 0 DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen); diff --git a/include/os/osSemaphore.h b/include/os/osSemaphore.h index 594daf1bf3..21c88c9976 100644 --- a/include/os/osSemaphore.h +++ b/include/os/osSemaphore.h @@ -22,21 +22,28 @@ extern "C" { #include -#if defined (_TD_DARWIN_64) - typedef struct tsem_s *tsem_t; - int tsem_init(tsem_t *sem, int pshared, unsigned int value); - int tsem_wait(tsem_t *sem); - int tsem_post(tsem_t *sem); - int tsem_destroy(tsem_t *sem); +#if defined(_TD_DARWIN_64) + +typedef struct tsem_s *tsem_t; + +int tsem_init(tsem_t *sem, int pshared, unsigned int value); +int tsem_wait(tsem_t *sem); +int tsem_timewait(tsem_t *sim, int64_t nanosecs); +int tsem_post(tsem_t *sem); +int tsem_destroy(tsem_t *sem); + #else - #define tsem_t sem_t - #define tsem_init sem_init - int tsem_wait(tsem_t* sem); - #define tsem_post sem_post - #define tsem_destroy sem_destroy + +#define tsem_t sem_t +#define tsem_init sem_init +int tsem_wait(tsem_t *sem); +int tsem_timewait(tsem_t *sim, int64_t nanosecs); +#define tsem_post sem_post +#define tsem_destroy sem_destroy + #endif -#if defined (_TD_DARWIN_64) +#if defined(_TD_DARWIN_64) // #define TdThreadRwlock TdThreadMutex // #define taosThreadRwlockInit(lock, NULL) taosThreadMutexInit(lock, NULL) // #define taosThreadRwlockDestroy(lock) taosThreadMutexDestroy(lock) @@ -44,20 +51,20 @@ extern "C" { // #define taosThreadRwlockRdlock(lock) taosThreadMutexLock(lock) // #define taosThreadRwlockUnlock(lock) taosThreadMutexUnlock(lock) - #define TdThreadSpinlock TdThreadMutex - #define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL) - #define taosThreadSpinDestroy(lock) taosThreadMutexDestroy(lock) - #define taosThreadSpinLock(lock) taosThreadMutexLock(lock) - #define taosThreadSpinUnlock(lock) taosThreadMutexUnlock(lock) +#define TdThreadSpinlock TdThreadMutex +#define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL) +#define taosThreadSpinDestroy(lock) taosThreadMutexDestroy(lock) +#define taosThreadSpinLock(lock) taosThreadMutexLock(lock) +#define taosThreadSpinUnlock(lock) taosThreadMutexUnlock(lock) #endif bool taosCheckPthreadValid(TdThread thread); int64_t taosGetSelfPthreadId(); int64_t taosGetPthreadId(TdThread thread); -void taosResetPthread(TdThread* thread); +void taosResetPthread(TdThread *thread); bool taosComparePthread(TdThread first, TdThread second); int32_t taosGetPId(); -int32_t taosGetAppName(char* name, int32_t* len); +int32_t taosGetAppName(char *name, int32_t *len); #ifdef __cplusplus } diff --git a/include/util/tcache.h b/include/util/tcache.h index b5c1578380..d8ab018570 100644 --- a/include/util/tcache.h +++ b/include/util/tcache.h @@ -47,13 +47,13 @@ typedef struct STrashElem STrashElem; /** * initialize the cache object * @param keyType key type - * @param refreshTimeInSeconds refresh operation interval time, the maximum survival time when one element is expired + * @param refreshTimeInMs refresh operation interval time, the maximum survival time when one element is expired * and not referenced by other objects * @param extendLifespan auto extend lifespan, if accessed * @param fn free resource callback function * @return */ -SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn, +SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInMs, bool extendLifespan, __cache_free_fn_t fn, const char *cacheName); /** @@ -111,7 +111,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove); * @param pCacheObj * @return */ -size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj); +size_t taosCacheGetNumOfObj(const SCacheObj *pCacheObj); /** * move all data node into trash, clear node in trash can if it is not referenced by any clients @@ -145,11 +145,11 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1); */ void taosStopCacheRefreshWorker(); -SCacheIter* taosCacheCreateIter(const SCacheObj* pCacheObj); -bool taosCacheIterNext(SCacheIter* pIter); -void* taosCacheIterGetData(const SCacheIter* pIter, size_t* dataLen); -void* taosCacheIterGetKey(const SCacheIter* pIter, size_t* keyLen); -void taosCacheDestroyIter(SCacheIter* pIter); +SCacheIter *taosCacheCreateIter(const SCacheObj *pCacheObj); +bool taosCacheIterNext(SCacheIter *pIter); +void *taosCacheIterGetData(const SCacheIter *pIter, size_t *dataLen); +void *taosCacheIterGetKey(const SCacheIter *pIter, size_t *keyLen); +void taosCacheDestroyIter(SCacheIter *pIter); #ifdef __cplusplus } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 51e422f5c2..4dc0e6e5cf 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -260,6 +260,16 @@ void tmq_list_destroy(tmq_list_t* list) { taosArrayDestroy(container); } +int32_t tmq_list_get_size(const tmq_list_t* list) { + const SArray* container = &list->container; + return taosArrayGetSize(container); +} + +char** tmq_list_to_c_array(const tmq_list_t* list) { + const SArray* container = &list->container; + return container->pData; +} + static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) { return sprintf(dst, "%s:%d", topicName, vg); } @@ -387,7 +397,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->commit_cb = conf->commit_cb; pTmq->resetOffsetCfg = conf->resetOffset; - pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1); + pTmq->consumerId = tGenIdPI64(); pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); if (pTmq->clientTopics == NULL) { taosMemoryFree(pTmq); diff --git a/source/dnode/mnode/impl/src/mndPerfSchema.c b/source/dnode/mnode/impl/src/mndPerfSchema.c index fe96bcb709..737068a2dd 100644 --- a/source/dnode/mnode/impl/src/mndPerfSchema.c +++ b/source/dnode/mnode/impl/src/mndPerfSchema.c @@ -49,6 +49,7 @@ static const SPerfsTableSchema topicSchema[] = { static const SPerfsTableSchema consumerSchema[] = { {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "app_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "status", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, @@ -61,6 +62,7 @@ static const SPerfsTableSchema subscribeSchema[] = { {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, }; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 826a73afc6..fef57196f0 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -24,20 +24,20 @@ #include "version.h" typedef struct { - uint32_t id; - int8_t connType; - char user[TSDB_USER_LEN]; - char app[TSDB_APP_NAME_LEN]; // app name that invokes taosc - int64_t appStartTimeMs; // app start time - int32_t pid; // pid of app that invokes taosc - uint32_t ip; - uint16_t port; - int8_t killed; - int64_t loginTimeMs; - int64_t lastAccessTimeMs; - uint64_t killId; - int32_t numOfQueries; - SArray *pQueries; //SArray + uint32_t id; + int8_t connType; + char user[TSDB_USER_LEN]; + char app[TSDB_APP_NAME_LEN]; // app name that invokes taosc + int64_t appStartTimeMs; // app start time + int32_t pid; // pid of app that invokes taosc + uint32_t ip; + uint16_t port; + int8_t killed; + int64_t loginTimeMs; + int64_t lastAccessTimeMs; + uint64_t killId; + int32_t numOfQueries; + SArray *pQueries; // SArray } SConnObj; static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType, uint32_t ip, uint16_t port, @@ -58,7 +58,8 @@ static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter); int32_t mndInitProfile(SMnode *pMnode) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; - int32_t connCheckTime = tsShellActivityTimer * 2; + // in ms + int32_t connCheckTime = tsShellActivityTimer * 2 * 1000; pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, connCheckTime, true, (__cache_free_fn_t)mndFreeConn, "conn"); if (pMgmt->cache == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -71,9 +72,9 @@ int32_t mndInitProfile(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_KILL_QUERY, mndProcessKillQueryReq); mndSetMsgHandle(pMnode, TDMT_MND_KILL_CONN, mndProcessKillConnReq); -// mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndRetrieveConns); + // mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndRetrieveConns); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndCancelGetNextConn); -// mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndRetrieveQueries); + // mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndRetrieveQueries); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndCancelGetNextQuery); return 0; @@ -91,7 +92,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType int32_t pid, const char *app, int64_t startTime) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; - char connStr[255] = {0}; + char connStr[255] = {0}; int32_t len = snprintf(connStr, sizeof(connStr), "%s%d%d%d%s", user, ip, port, pid, app); int32_t connId = mndGenerateUid(connStr, len); if (startTime == 0) startTime = taosGetTimestampMs(); @@ -253,8 +254,8 @@ static int32_t mndSaveQueryList(SConnObj *pConn, SQueryHbReqBasic *pBasic) { pConn->pQueries = pBasic->queryDesc; pBasic->queryDesc = NULL; - - pConn->numOfQueries = pBasic->queryDesc ? taosArrayGetSize(pBasic->queryDesc) : 0; + + pConn->numOfQueries = pBasic->queryDesc ? taosArrayGetSize(pBasic->queryDesc) : 0; return TSDB_CODE_SUCCESS; } @@ -324,9 +325,10 @@ static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) { return NULL; } -static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHbReq *pHbReq, SClientHbBatchRsp *pBatchRsp) { +static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHbReq *pHbReq, + SClientHbBatchRsp *pBatchRsp) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; - SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL}; + SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL}; if (pHbReq->query) { SQueryHbReqBasic *pBasic = pHbReq->query; @@ -335,8 +337,9 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb rpcGetConnInfo(pMsg->handle, &connInfo); SConnObj *pConn = mndAcquireConn(pMnode, pBasic->connId); - if (pConn == NULL) { - pConn = mndCreateConn(pMnode, connInfo.user, CONN_TYPE__QUERY, connInfo.clientIp, connInfo.clientPort, pBasic->pid, pBasic->app, 0); + if (pConn == NULL) { + pConn = mndCreateConn(pMnode, connInfo.user, CONN_TYPE__QUERY, connInfo.clientIp, connInfo.clientPort, + pBasic->pid, pBasic->app, 0); if (pConn == NULL) { mError("user:%s, conn:%u is freed and failed to create new since %s", connInfo.user, pBasic->connId, terrstr()); return -1; @@ -345,7 +348,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } } else if (pConn->killed) { mError("user:%s, conn:%u is already killed", connInfo.user, pConn->id); - mndReleaseConn(pMnode, pConn); + mndReleaseConn(pMnode, pConn); terrno = TSDB_CODE_MND_INVALID_CONNECTION; return -1; } @@ -369,8 +372,8 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } rspBasic->connId = pConn->id; - rspBasic->totalDnodes = 1; //TODO - rspBasic->onlineDnodes = 1; //TODO + rspBasic->totalDnodes = 1; // TODO + rspBasic->onlineDnodes = 1; // TODO mndGetMnodeEpSet(pMnode, &rspBasic->epSet); mndReleaseConn(pMnode, pConn); @@ -379,7 +382,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb int32_t kvNum = taosHashGetSize(pHbReq->info); if (NULL == pHbReq->info || kvNum <= 0) { - taosArrayPush(pBatchRsp->rsps, &hbRsp); + taosArrayPush(pBatchRsp->rsps, &hbRsp); return TSDB_CODE_SUCCESS; } @@ -604,8 +607,8 @@ static int32_t mndRetrieveConns(SNodeMsg *pReq, SShowObj *pShow, char *data, int } static int32_t mndRetrieveQueries(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pNode; - int32_t numOfRows = 0; + SMnode *pMnode = pReq->pNode; + int32_t numOfRows = 0; #if 0 SConnObj *pConn = NULL; int32_t cols = 0; @@ -703,7 +706,7 @@ static int32_t mndRetrieveQueries(SNodeMsg *pReq, SShowObj *pShow, char *data, i } pShow->numOfRows += numOfRows; -#endif +#endif return numOfRows; } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 1a14c94640..804fa2e17f 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -28,7 +28,7 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq); int32_t mndInitShow(SMnode *pMnode) { SShowMgmt *pMgmt = &pMnode->showMgmt; - pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, 5, true, (__cache_free_fn_t)mndFreeShowObj, "show"); + pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, 5000, true, (__cache_free_fn_t)mndFreeShowObj, "show"); if (pMgmt->cache == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to alloc show cache since %s", terrstr()); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index f17b8a4d88..a4b98ba01a 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -534,8 +534,8 @@ static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB colDataAppend(pColInfo, numOfRows, (const char *)&pTopic->createTime, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - char *sql = taosMemoryCalloc(1, strlen(pTopic->sql) + 1 + VARSTR_HEADER_SIZE); - strcpy(&sql[VARSTR_HEADER_SIZE], pTopic->sql); + char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0}; + tstrncpy(&sql[VARSTR_HEADER_SIZE], pTopic->sql, TSDB_SHOW_SQL_LEN); varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE])); colDataAppend(pColInfo, numOfRows, (const char *)sql, false); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 6f929ce829..2250aab35c 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -18,8 +18,10 @@ #include "executor.h" #include "os.h" +#include "tcache.h" #include "thash.h" #include "tmsg.h" +#include "trpc.h" #include "ttimer.h" #include "wal.h" @@ -142,26 +144,37 @@ typedef struct { } STqMetaStore; typedef struct { - char subKey[TSDB_SUBSCRIBE_KEY_LEN]; - int64_t consumerId; - int32_t epoch; - int8_t subType; - int8_t withTbName; - int8_t withSchema; - int8_t withTag; - int8_t withTagSchema; - char* qmsg; + int64_t consumerId; + int32_t epoch; + int32_t skipLogNum; + int64_t reqOffset; + SRWLatch lock; + SRpcMsg* handle; +} STqPushHandle; + +typedef struct { + char subKey[TSDB_SUBSCRIBE_KEY_LEN]; + int64_t consumerId; + int32_t epoch; + int8_t subType; + int8_t withTbName; + int8_t withSchema; + int8_t withTag; + int8_t withTagSchema; + char* qmsg; + STqPushHandle pushHandle; // SRWLatch lock; SWalReadHandle* pWalReader; - // number should be identical to fetch thread num - STqReadHandle* pStreamReader[4]; - qTaskInfo_t task[4]; + // task number should be the same with fetch thread + STqReadHandle* pExecReader[5]; + qTaskInfo_t task[5]; } STqExec; struct STQ { char* path; // STqMetaStore* tqMeta; - SHashObj* execs; // subKey -> tqExec + SHashObj* pushMgr; // consumerId -> STqExec* + SHashObj* execs; // subKey -> STqExec SHashObj* pStreamTasks; SVnode* pVnode; SWal* pWal; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 30f75218b4..aa04ae3b46 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -41,6 +41,8 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { pTq->pStreamTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + pTq->pushMgr = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); + return pTq; } @@ -52,8 +54,139 @@ void tqClose(STQ* pTq) { // TODO } +int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { + if (msgType != TDMT_VND_SUBMIT) return 0; + void* pIter = NULL; + STqExec* pExec = NULL; + SSubmitReq* pReq = (SSubmitReq*)msg; + int32_t workerId = 4; + int64_t fetchOffset = ver; + + while (1) { + pIter = taosHashIterate(pTq->pushMgr, pIter); + if (pIter == NULL) break; + pExec = (STqExec**)pIter; + + taosWLockLatch(&pExec->pushHandle.lock); + + SRpcMsg* pMsg = atomic_load_ptr(&pExec->pushHandle.handle); + ASSERT(pMsg); + + SMqDataBlkRsp rsp = {0}; + rsp.reqOffset = pExec->pushHandle.reqOffset; + rsp.blockData = taosArrayInit(0, sizeof(int32_t)); + rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t)); + + if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { + qTaskInfo_t task = pExec->task[workerId]; + ASSERT(task); + qSetStreamInput(task, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); + while (1) { + SSDataBlock* pDataBlock = NULL; + uint64_t ts = 0; + if (qExecTask(task, &pDataBlock, &ts) < 0) { + ASSERT(0); + } + if (pDataBlock == NULL) break; + + ASSERT(pDataBlock->info.rows != 0); + ASSERT(pDataBlock->info.numOfCols != 0); + + int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pDataBlock); + void* buf = taosMemoryCalloc(1, dataStrLen); + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf; + pRetrieve->useconds = ts; + pRetrieve->precision = TSDB_DEFAULT_PRECISION; + pRetrieve->compressed = 0; + pRetrieve->completed = 1; + pRetrieve->numOfRows = htonl(pDataBlock->info.rows); + + // TODO enable compress + int32_t actualLen = 0; + blockCompressEncode(pDataBlock, pRetrieve->data, &actualLen, pDataBlock->info.numOfCols, false); + actualLen += sizeof(SRetrieveTableRsp); + ASSERT(actualLen <= dataStrLen); + taosArrayPush(rsp.blockDataLen, &actualLen); + taosArrayPush(rsp.blockData, &buf); + rsp.blockNum++; + } + } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { + STqReadHandle* pReader = pExec->pExecReader[workerId]; + tqReadHandleSetMsg(pReader, pReq, 0); + while (tqNextDataBlock(pReader)) { + SSDataBlock block = {0}; + if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.rows, + &block.info.numOfCols) < 0) { + ASSERT(0); + } + int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(&block); + void* buf = taosMemoryCalloc(1, dataStrLen); + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf; + /*pRetrieve->useconds = 0;*/ + pRetrieve->precision = TSDB_DEFAULT_PRECISION; + pRetrieve->compressed = 0; + pRetrieve->completed = 1; + pRetrieve->numOfRows = htonl(block.info.rows); + + // TODO enable compress + int32_t actualLen = 0; + blockCompressEncode(&block, pRetrieve->data, &actualLen, block.info.numOfCols, false); + actualLen += sizeof(SRetrieveTableRsp); + ASSERT(actualLen <= dataStrLen); + taosArrayPush(rsp.blockDataLen, &actualLen); + taosArrayPush(rsp.blockData, &buf); + rsp.blockNum++; + } + } else { + ASSERT(0); + } + + if (rsp.blockNum == 0) { + taosWUnLockLatch(&pExec->pushHandle.lock); + continue; + } + + ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum); + ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum); + + rsp.rspOffset = fetchOffset; + + int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, &rsp); + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + pMsg->code = -1; + return -1; + } + + ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; + ((SMqRspHead*)buf)->epoch = pExec->pushHandle.epoch; + ((SMqRspHead*)buf)->consumerId = pExec->pushHandle.consumerId; + + void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); + tEncodeSMqDataBlkRsp(&abuf, &rsp); + pMsg->pCont = buf; + pMsg->contLen = tlen; + pMsg->code = 0; + tmsgSendRsp(pMsg); + + atomic_store_ptr(&pExec->pushHandle.handle, NULL); + taosWUnLockLatch(&pExec->pushHandle.lock); + + vDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld", + TD_VID(pTq->pVnode), fetchOffset, pExec->pushHandle.consumerId, pExec->pushHandle.epoch, rsp.blockNum, + rsp.reqOffset, rsp.rspOffset); + + // TODO destroy + taosArrayDestroy(rsp.blockData); + taosArrayDestroy(rsp.blockDataLen); + } + + return 0; +} + int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version) { if (msgType != TDMT_VND_SUBMIT) return 0; + void* data = taosMemoryMalloc(msgLen); if (data == NULL) { return -1; @@ -71,6 +204,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t versi .pCont = data, .contLen = msgLen, }; + tmsgPutToQueue(&pTq->pVnode->msgCb, FETCH_QUEUE, &req); #if 0 @@ -240,6 +374,7 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SMqPollReqV2* pReq = pMsg->pCont; int64_t consumerId = pReq->consumerId; + int64_t waitTime = pReq->blockingTime; int32_t reqEpoch = pReq->epoch; int64_t fetchOffset; @@ -265,8 +400,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SMqDataBlkRsp rsp = {0}; rsp.reqOffset = pReq->currentOffset; - rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t)); rsp.blockData = taosArrayInit(0, sizeof(void*)); + rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t)); while (1) { consumerEpoch = atomic_load_32(&pExec->epoch); @@ -283,6 +418,28 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { // response to user vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset); + +#if 0 + // add to pushMgr + taosWLockLatch(&pExec->pushHandle.lock); + + pExec->pushHandle.consumerId = consumerId; + pExec->pushHandle.epoch = reqEpoch; + pExec->pushHandle.reqOffset = rsp.reqOffset; + pExec->pushHandle.skipLogNum = rsp.skipLogNum; + pExec->pushHandle.handle = pMsg; + + taosWUnLockLatch(&pExec->pushHandle.lock); + + // TODO add timer + + // TODO: the pointer will always be valid? + taosHashPut(pTq->pushMgr, &consumerId, sizeof(int64_t), &pExec, sizeof(void*)); + taosArrayDestroy(rsp.blockData); + taosArrayDestroy(rsp.blockDataLen); + return 0; +#endif + break; } @@ -325,7 +482,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { rsp.blockNum++; } } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { - STqReadHandle* pReader = pExec->pStreamReader[workerId]; + STqReadHandle* pReader = pExec->pExecReader[workerId]; tqReadHandleSetMsg(pReader, pCont, 0); while (tqNextDataBlock(pReader)) { SSDataBlock block = {0}; @@ -635,10 +792,10 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { req.qmsg = NULL; pExec->pWalReader = walOpenReadHandle(pTq->pVnode->pWal); - for (int32_t i = 0; i < 4; i++) { - pExec->pStreamReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); + for (int32_t i = 0; i < 5; i++) { + pExec->pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); SReadHandle handle = { - .reader = pExec->pStreamReader[i], + .reader = pExec->pExecReader[i], .meta = pTq->pVnode->pMeta, }; pExec->task[i] = qCreateStreamExecTaskInfo(pExec->qmsg, &handle); diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 1297fdbc27..893cf71a6f 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -85,11 +85,11 @@ int32_t tsem_wait(tsem_t* sem) { #include #include -static TdThread sem_thread; +static TdThread sem_thread; static TdThreadOnce sem_once; -static task_t sem_port; -static volatile int sem_inited = 0; -static semaphore_t sem_exit; +static task_t sem_port; +static volatile int sem_inited = 0; +static semaphore_t sem_exit; static void *sem_thread_routine(void *arg) { (void)arg; @@ -122,12 +122,12 @@ static void once_init(void) { struct tsem_s { #ifdef SEM_USE_PTHREAD - TdThreadMutex lock; - TdThreadCond cond; + TdThreadMutex lock; + TdThreadCond cond; volatile int64_t val; #elif defined(SEM_USE_POSIX) size_t id; - sem_t * sem; + sem_t *sem; #elif defined(SEM_USE_SEM) semaphore_t sem; #else // SEM_USE_PTHREAD @@ -140,7 +140,8 @@ struct tsem_s { int tsem_init(tsem_t *sem, int pshared, unsigned int value) { // fprintf(stderr, "==%s[%d]%s():[%p]==creating\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); if (*sem) { - fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, + sem); abort(); } struct tsem_s *p = (struct tsem_s *)taosMemoryCalloc(1, sizeof(*p)); @@ -180,20 +181,22 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) { int e = errno; if (e == EEXIST) continue; if (e == EINTR) continue; - fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem, e, - strerror(e)); + fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem, + e, strerror(e)); abort(); } while (p->sem == SEM_FAILED); #elif defined(SEM_USE_SEM) taosThreadOnce(&sem_once, once_init); if (sem_inited != 1) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal resource init failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal resource init failed\n", taosDirEntryBaseName(__FILE__), __LINE__, + __func__, sem); errno = ENOMEM; return -1; } kern_return_t ret = semaphore_create(sem_port, &p->sem, SYNC_POLICY_FIFO, value); if (ret != KERN_SUCCESS) { - fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, + sem); // we fail-fast here, because we have less-doc about semaphore_create for the moment abort(); } @@ -224,18 +227,21 @@ int tsem_wait(tsem_t *sem) { } #ifdef SEM_USE_PTHREAD if (taosThreadMutexLock(&p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, + sem); abort(); } p->val -= 1; if (p->val < 0) { if (taosThreadCondWait(&p->cond, &p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, + sem); abort(); } } if (taosThreadMutexUnlock(&p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, + sem); abort(); } return 0; @@ -260,18 +266,21 @@ int tsem_post(tsem_t *sem) { } #ifdef SEM_USE_PTHREAD if (taosThreadMutexLock(&p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, + sem); abort(); } p->val += 1; if (p->val <= 0) { if (taosThreadCondSignal(&p->cond)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, + sem); abort(); } } if (taosThreadMutexUnlock(&p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, + sem); abort(); } return 0; @@ -293,26 +302,30 @@ int tsem_destroy(tsem_t *sem) { } struct tsem_s *p = *sem; if (!p->valid) { - // fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); - // abort(); + // fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, + // sem); abort(); return 0; } #ifdef SEM_USE_PTHREAD if (taosThreadMutexLock(&p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, + sem); abort(); } p->valid = 0; if (taosThreadCondDestroy(&p->cond)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, + sem); abort(); } if (taosThreadMutexUnlock(&p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, + sem); abort(); } if (taosThreadMutexDestroy(&p->lock)) { - fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, + sem); abort(); } #elif defined(SEM_USE_POSIX) @@ -321,8 +334,8 @@ int tsem_destroy(tsem_t *sem) { int r = sem_unlink(name); if (r) { int e = errno; - fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem, e, - strerror(e)); + fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem, + e, strerror(e)); abort(); } #elif defined(SEM_USE_SEM) @@ -424,4 +437,17 @@ int32_t tsem_wait(tsem_t* sem) { return ret; } +int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { + int ret = 0; + + struct timespec tv = { + .tv_sec = 0, + .tv_nsec = nanosecs, + }; + + while ((ret = sem_timedwait(sem, &tv)) == -1 && errno == EINTR) continue; + + return ret; +} + #endif diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index a69f11f285..e2dfde0503 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -19,16 +19,16 @@ #include "tlog.h" #include "tutil.h" -#define CACHE_MAX_CAPACITY 1024*1024*16 -#define CACHE_DEFAULT_CAPACITY 1024*4 +#define CACHE_MAX_CAPACITY 1024 * 1024 * 16 +#define CACHE_DEFAULT_CAPACITY 1024 * 4 -static TdThread cacheRefreshWorker = {0}; +static TdThread cacheRefreshWorker = {0}; static TdThreadOnce cacheThreadInit = PTHREAD_ONCE_INIT; static TdThreadMutex guard = PTHREAD_MUTEX_INITIALIZER; -static SArray *pCacheArrayList = NULL; -static bool stopRefreshWorker = false; -static bool refreshWorkerNormalStopped = false; -static bool refreshWorkerUnexpectedStopped = false; +static SArray *pCacheArrayList = NULL; +static bool stopRefreshWorker = false; +static bool refreshWorkerNormalStopped = false; +static bool refreshWorkerUnexpectedStopped = false; typedef struct SCacheNode { uint64_t addedTime; // the added time when this element is added or updated into cache @@ -36,7 +36,7 @@ typedef struct SCacheNode { int64_t expireTime; // expire time uint64_t signature; struct STrashElem *pTNodeHeader; // point to trash node head - uint16_t keyLen: 15; // max key size: 32kb + uint16_t keyLen : 15; // max key size: 32kb bool inTrashcan : 1; // denote if it is in trash or not uint32_t size; // allocated size for current SCacheNode uint32_t dataLen; @@ -47,8 +47,8 @@ typedef struct SCacheNode { } SCacheNode; typedef struct SCacheEntry { - int32_t num; // number of elements in current entry - SRWLatch latch; // entry latch + int32_t num; // number of elements in current entry + SRWLatch latch; // entry latch SCacheNode *next; } SCacheEntry; @@ -75,24 +75,24 @@ typedef struct SCacheIter { * when the node in pTrash does not be referenced, it will be release at the expired expiredTime */ struct SCacheObj { - int64_t sizeInBytes; // total allocated buffer in this hash table, SCacheObj is not included. - int64_t refreshTime; - char *name; - SCacheStatis statistics; + int64_t sizeInBytes; // total allocated buffer in this hash table, SCacheObj is not included. + int64_t refreshTime; + char *name; + SCacheStatis statistics; - SCacheEntry *pEntryList; - size_t capacity; // number of slots - size_t numOfElems; // number of elements in cache - _hash_fn_t hashFp; // hash function - __cache_free_fn_t freeFp; + SCacheEntry *pEntryList; + size_t capacity; // number of slots + size_t numOfElems; // number of elements in cache + _hash_fn_t hashFp; // hash function + __cache_free_fn_t freeFp; - uint32_t numOfElemsInTrash; // number of element in trash - STrashElem *pTrash; + uint32_t numOfElemsInTrash; // number of element in trash + STrashElem *pTrash; - uint8_t deleting; // set the deleting flag to stop refreshing ASAP. - TdThread refreshWorker; - bool extendLifespan; // auto extend life span when one item is accessed. - int64_t checkTick; // tick used to record the check times of the refresh threads + uint8_t deleting; // set the deleting flag to stop refreshing ASAP. + TdThread refreshWorker; + bool extendLifespan; // auto extend life span when one item is accessed. + int64_t checkTick; // tick used to record the check times of the refresh threads #if defined(LINUX) TdThreadRwlock lock; #else @@ -182,7 +182,7 @@ TdThread doRegisterCacheObj(SCacheObj *pCacheObj) { * @return SCacheNode */ static SCacheNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, - uint64_t duration); + uint64_t duration); /** * addedTime object node into trash, and this object is closed for referencing if it is addedTime to trash @@ -267,7 +267,7 @@ static void pushfrontNodeInEntryList(SCacheEntry *pEntry, SCacheNode *pNode) { pEntry->num += 1; } -static void removeNodeInEntryList(SCacheEntry* pe, SCacheNode* prev, SCacheNode* pNode) { +static void removeNodeInEntryList(SCacheEntry *pe, SCacheNode *prev, SCacheNode *pNode) { if (prev == NULL) { ASSERT(pe->next == pNode); pe->next = pNode->pNext; @@ -279,14 +279,14 @@ static void removeNodeInEntryList(SCacheEntry* pe, SCacheNode* prev, SCacheNode* pe->num -= 1; } -static FORCE_INLINE SCacheEntry* doFindEntry(SCacheObj* pCacheObj, const void* key, size_t keyLen) { +static FORCE_INLINE SCacheEntry *doFindEntry(SCacheObj *pCacheObj, const void *key, size_t keyLen) { uint32_t hashVal = (*pCacheObj->hashFp)(key, keyLen); int32_t slot = hashVal % pCacheObj->capacity; return &pCacheObj->pEntryList[slot]; } -static FORCE_INLINE SCacheNode * -doSearchInEntryList(SCacheEntry *pe, const void *key, size_t keyLen, SCacheNode** prev) { +static FORCE_INLINE SCacheNode *doSearchInEntryList(SCacheEntry *pe, const void *key, size_t keyLen, + SCacheNode **prev) { SCacheNode *pNode = pe->next; while (pNode) { if ((pNode->keyLen == keyLen) && memcmp(pNode->key, key, keyLen) == 0) { @@ -299,9 +299,9 @@ doSearchInEntryList(SCacheEntry *pe, const void *key, size_t keyLen, SCacheNode* return pNode; } -static bool doRemoveExpiredFn(void *param, SCacheNode* pNode) { +static bool doRemoveExpiredFn(void *param, SCacheNode *pNode) { SCacheObjTravSup *ps = (SCacheObjTravSup *)param; - SCacheObj *pCacheObj = ps->pCacheObj; + SCacheObj *pCacheObj = ps->pCacheObj; if ((int64_t)pNode->expireTime < ps->time && T_REF_VAL_GET(pNode) <= 0) { taosCacheReleaseNode(pCacheObj, pNode); @@ -320,7 +320,7 @@ static bool doRemoveExpiredFn(void *param, SCacheNode* pNode) { static bool doRemoveNodeFn(void *param, SCacheNode *pNode) { SCacheObjTravSup *ps = (SCacheObjTravSup *)param; - SCacheObj *pCacheObj = ps->pCacheObj; + SCacheObj *pCacheObj = ps->pCacheObj; if (T_REF_VAL_GET(pNode) == 0) { taosCacheReleaseNode(pCacheObj, pNode); @@ -347,14 +347,14 @@ static FORCE_INLINE int32_t getCacheCapacity(int32_t length) { len = (len << 1u); } - return len > CACHE_MAX_CAPACITY? CACHE_MAX_CAPACITY:len; + return len > CACHE_MAX_CAPACITY ? CACHE_MAX_CAPACITY : len; } -SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn, +SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInMs, bool extendLifespan, __cache_free_fn_t fn, const char *cacheName) { const int32_t SLEEP_DURATION = 500; // 500 ms - if (refreshTimeInSeconds <= 0) { + if (refreshTimeInMs <= 0) { return NULL; } @@ -374,10 +374,10 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext } // set free cache node callback function - pCacheObj->hashFp = taosGetDefaultHashFunction(keyType); - pCacheObj->freeFp = fn; - pCacheObj->refreshTime = refreshTimeInSeconds * 1000; - pCacheObj->checkTick = pCacheObj->refreshTime / SLEEP_DURATION; + pCacheObj->hashFp = taosGetDefaultHashFunction(keyType); + pCacheObj->freeFp = fn; + pCacheObj->refreshTime = refreshTimeInMs; + pCacheObj->checkTick = pCacheObj->refreshTime / SLEEP_DURATION; pCacheObj->extendLifespan = extendLifespan; // the TTL after the last access if (__trashcan_lock_init(pCacheObj) != 0) { @@ -411,8 +411,8 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v taosWLockLatch(&pe->latch); - SCacheNode *prev = NULL; - SCacheNode* pNode = doSearchInEntryList(pe, key, keyLen, &prev); + SCacheNode *prev = NULL; + SCacheNode *pNode = doSearchInEntryList(pe, key, keyLen, &prev); if (pNode == NULL) { pushfrontNodeInEntryList(pe, pNode1); @@ -460,12 +460,12 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen return NULL; } - SCacheNode *prev = NULL; + SCacheNode *prev = NULL; SCacheEntry *pe = doFindEntry(pCacheObj, key, keyLen); taosRLockLatch(&pe->latch); - SCacheNode* pNode = doSearchInEntryList(pe, key, keyLen, &prev); + SCacheNode *pNode = doSearchInEntryList(pe, key, keyLen, &prev); if (pNode != NULL) { int32_t ref = T_REF_INC(pNode); ASSERT(ref > 0); @@ -589,7 +589,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { } else { // NOTE: remove it from hash in the first place, otherwise, the pNode may have been released by other thread // when reaches here. - SCacheNode * prev = NULL; + SCacheNode *prev = NULL; SCacheEntry *pe = doFindEntry(pCacheObj, pNode->key, pNode->keyLen); taosWLockLatch(&pe->latch); @@ -646,7 +646,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { } } -void doTraverseElems(SCacheObj* pCacheObj, bool (*fp)(void *param, SCacheNode* pNode), SCacheObjTravSup* pSup) { +void doTraverseElems(SCacheObj *pCacheObj, bool (*fp)(void *param, SCacheNode *pNode), SCacheObjTravSup *pSup) { int32_t numOfEntries = (int32_t)pCacheObj->capacity; for (int32_t i = 0; i < numOfEntries; ++i) { SCacheEntry *pEntry = &pCacheObj->pEntryList[i]; @@ -675,7 +675,7 @@ void doTraverseElems(SCacheObj* pCacheObj, bool (*fp)(void *param, SCacheNode* p } } -void taosCacheEmpty(SCacheObj* pCacheObj) { +void taosCacheEmpty(SCacheObj *pCacheObj) { SCacheObjTravSup sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; doTraverseElems(pCacheObj, doRemoveNodeFn, &sup); taosTrashcanEmpty(pCacheObj, false); @@ -710,20 +710,20 @@ SCacheNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pDat return NULL; } - pNewNode->data = (char*)pNewNode + sizeof(SCacheNode); + pNewNode->data = (char *)pNewNode + sizeof(SCacheNode); pNewNode->dataLen = size; memcpy(pNewNode->data, pData, size); - pNewNode->key = (char *)pNewNode + sizeof(SCacheNode) + size; + pNewNode->key = (char *)pNewNode + sizeof(SCacheNode) + size; pNewNode->keyLen = (uint16_t)keyLen; memcpy(pNewNode->key, key, keyLen); - pNewNode->addedTime = (uint64_t)taosGetTimestampMs(); - pNewNode->lifespan = duration; + pNewNode->addedTime = (uint64_t)taosGetTimestampMs(); + pNewNode->lifespan = duration; pNewNode->expireTime = pNewNode->addedTime + pNewNode->lifespan; - pNewNode->signature = (uint64_t)pNewNode; - pNewNode->size = (uint32_t)sizeInBytes; + pNewNode->signature = (uint64_t)pNewNode; + pNewNode->size = (uint32_t)sizeInBytes; return pNewNode; } @@ -914,21 +914,19 @@ void taosStopCacheRefreshWorker(void) { taosArrayDestroy(pCacheArrayList); } -size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj) { - return pCacheObj->numOfElems + pCacheObj->numOfElemsInTrash; -} +size_t taosCacheGetNumOfObj(const SCacheObj *pCacheObj) { return pCacheObj->numOfElems + pCacheObj->numOfElemsInTrash; } -SCacheIter* taosCacheCreateIter(const SCacheObj* pCacheObj) { +SCacheIter *taosCacheCreateIter(const SCacheObj *pCacheObj) { ASSERT(pCacheObj != NULL); - SCacheIter* pIter = taosMemoryCalloc(1, sizeof(SCacheIter)); - pIter->pCacheObj = (SCacheObj*) pCacheObj; + SCacheIter *pIter = taosMemoryCalloc(1, sizeof(SCacheIter)); + pIter->pCacheObj = (SCacheObj *)pCacheObj; pIter->entryIndex = -1; - pIter->index = -1; + pIter->index = -1; return pIter; } -bool taosCacheIterNext(SCacheIter* pIter) { - SCacheObj* pCacheObj = pIter->pCacheObj; +bool taosCacheIterNext(SCacheIter *pIter) { + SCacheObj *pCacheObj = pIter->pCacheObj; if (pIter->index + 1 >= pIter->numOfObj) { if (pIter->entryIndex + 1 >= pCacheObj->capacity) { @@ -936,9 +934,9 @@ bool taosCacheIterNext(SCacheIter* pIter) { } // release the reference for all objects in the snapshot - for(int32_t i = 0; i < pIter->numOfObj; ++i) { - char* p= pIter->pCurrent[i]->data; - taosCacheRelease(pCacheObj, (void**) &p, false); + for (int32_t i = 0; i < pIter->numOfObj; ++i) { + char *p = pIter->pCurrent[i]->data; + taosCacheRelease(pCacheObj, (void **)&p, false); pIter->pCurrent[i] = NULL; } @@ -967,7 +965,7 @@ bool taosCacheIterNext(SCacheIter* pIter) { pIter->pCurrent = (SCacheNode **)tmp; } - SCacheNode* pNode = pEntry->next; + SCacheNode *pNode = pEntry->next; for (int32_t i = 0; i < pEntry->num; ++i) { ASSERT(pNode != NULL); @@ -981,7 +979,7 @@ bool taosCacheIterNext(SCacheIter* pIter) { pIter->numOfObj = pEntry->num; taosRUnLockLatch(&pEntry->latch); - pIter->index = -1; + pIter->index = -1; break; } } @@ -990,19 +988,19 @@ bool taosCacheIterNext(SCacheIter* pIter) { return true; } -void* taosCacheIterGetData(const SCacheIter* pIter, size_t* len) { - SCacheNode* pNode = pIter->pCurrent[pIter->index]; +void *taosCacheIterGetData(const SCacheIter *pIter, size_t *len) { + SCacheNode *pNode = pIter->pCurrent[pIter->index]; *len = pNode->dataLen; return pNode->data; } -void* taosCacheIterGetKey(const SCacheIter* pIter, size_t* len) { - SCacheNode* pNode = pIter->pCurrent[pIter->index]; +void *taosCacheIterGetKey(const SCacheIter *pIter, size_t *len) { + SCacheNode *pNode = pIter->pCurrent[pIter->index]; *len = pNode->keyLen; return pNode->key; } -void taosCacheDestroyIter(SCacheIter* pIter) { +void taosCacheDestroyIter(SCacheIter *pIter) { taosMemoryFreeClear(pIter->pCurrent); taosMemoryFreeClear(pIter); -} \ No newline at end of file +} diff --git a/source/util/test/cacheTest.cpp b/source/util/test/cacheTest.cpp index 748cf31b67..45c887288c 100644 --- a/source/util/test/cacheTest.cpp +++ b/source/util/test/cacheTest.cpp @@ -1,5 +1,5 @@ -#include #include +#include #include "os.h" #include "taos.h" @@ -8,132 +8,134 @@ // test cache TEST(cacheTest, client_cache_test) { const int32_t REFRESH_TIME_IN_SEC = 2; - SCacheObj* tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC, 0, NULL, "test"); + SCacheObj* tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC * 1000, 0, NULL, "test"); const char* key1 = "test1"; - char data1[] = "test11"; + char data1[] = "test11"; - char* cachedObj = (char*) taosCachePut(tscMetaCache, key1, strlen(key1), data1, strlen(data1)+1, 1); - taosSsleep(REFRESH_TIME_IN_SEC+1); + char* cachedObj = (char*)taosCachePut(tscMetaCache, key1, strlen(key1), data1, strlen(data1) + 1, 1); + taosSsleep(REFRESH_TIME_IN_SEC + 1); printf("obj is still valid: %s\n", cachedObj); char data2[] = "test22"; - taosCacheRelease(tscMetaCache, (void**) &cachedObj, false); + taosCacheRelease(tscMetaCache, (void**)&cachedObj, false); /* the object is cleared by cache clean operation */ - cachedObj = (char*) taosCachePut(tscMetaCache, key1, strlen(key1), data2, strlen(data2)+1, 20); + cachedObj = (char*)taosCachePut(tscMetaCache, key1, strlen(key1), data2, strlen(data2) + 1, 20); printf("after updated: %s\n", cachedObj); printf("start to remove data from cache\n"); - taosCacheRelease(tscMetaCache, (void**) &cachedObj, false); + taosCacheRelease(tscMetaCache, (void**)&cachedObj, false); printf("end of removing data from cache\n"); const char* key3 = "test2"; const char* data3 = "kkkkkkk"; - char* cachedObj2 = (char*) taosCachePut(tscMetaCache, key3, strlen(key3), data3, strlen(data3) + 1, 1); + char* cachedObj2 = (char*)taosCachePut(tscMetaCache, key3, strlen(key3), data3, strlen(data3) + 1, 1); printf("%s\n", cachedObj2); - taosCacheRelease(tscMetaCache, (void**) &cachedObj2, false); + taosCacheRelease(tscMetaCache, (void**)&cachedObj2, false); taosSsleep(3); - char* d = (char*) taosCacheAcquireByKey(tscMetaCache, key3, strlen(key3)); + char* d = (char*)taosCacheAcquireByKey(tscMetaCache, key3, strlen(key3)); assert(d == NULL); char key5[] = "test5"; char data5[] = "data5kkkkk"; - cachedObj2 = (char*) taosCachePut(tscMetaCache, key5, strlen(key5), data5, strlen(data5) + 1, 20); + cachedObj2 = (char*)taosCachePut(tscMetaCache, key5, strlen(key5), data5, strlen(data5) + 1, 20); - const char* data6= "new Data after updated"; - taosCacheRelease(tscMetaCache, (void**) &cachedObj2, false); + const char* data6 = "new Data after updated"; + taosCacheRelease(tscMetaCache, (void**)&cachedObj2, false); - cachedObj2 = (char*) taosCachePut(tscMetaCache, key5, strlen(key5), data6, strlen(data6) + 1, 20); + cachedObj2 = (char*)taosCachePut(tscMetaCache, key5, strlen(key5), data6, strlen(data6) + 1, 20); printf("%s\n", cachedObj2); - taosCacheRelease(tscMetaCache, (void**) &cachedObj2, true); + taosCacheRelease(tscMetaCache, (void**)&cachedObj2, true); const char* data7 = "add call update procedure"; - cachedObj2 = (char*) taosCachePut(tscMetaCache, key5, strlen(key5), data7, strlen(data7) + 1, 20); + cachedObj2 = (char*)taosCachePut(tscMetaCache, key5, strlen(key5), data7, strlen(data7) + 1, 20); printf("%s\n=======================================\n\n", cachedObj2); - char* cc = (char*) taosCacheAcquireByKey(tscMetaCache, key5, strlen(key5)); + char* cc = (char*)taosCacheAcquireByKey(tscMetaCache, key5, strlen(key5)); - taosCacheRelease(tscMetaCache, (void**) &cachedObj2, true); - taosCacheRelease(tscMetaCache, (void**) &cc, false); + taosCacheRelease(tscMetaCache, (void**)&cachedObj2, true); + taosCacheRelease(tscMetaCache, (void**)&cc, false); const char* data8 = "ttft"; const char* key6 = "key6"; - char* ft = (char*) taosCachePut(tscMetaCache, key6, strlen(key6), data8, strlen(data8), 20); - taosCacheRelease(tscMetaCache, (void**) &ft, false); + char* ft = (char*)taosCachePut(tscMetaCache, key6, strlen(key6), data8, strlen(data8), 20); + taosCacheRelease(tscMetaCache, (void**)&ft, false); /** * 140ns */ uint64_t startTime = taosGetTimestampUs(); printf("Cache Performance Test\nstart time:%" PRIu64 "\n", startTime); - for(int32_t i=0; i<1000; ++i) { - char* dd = (char*) taosCacheAcquireByKey(tscMetaCache, key6, strlen(key6)); + for (int32_t i = 0; i < 1000; ++i) { + char* dd = (char*)taosCacheAcquireByKey(tscMetaCache, key6, strlen(key6)); if (dd != NULL) { -// printf("get the data\n"); + // printf("get the data\n"); } else { printf("data has been released\n"); } - taosCacheRelease(tscMetaCache, (void**) &dd, false); + taosCacheRelease(tscMetaCache, (void**)&dd, false); } uint64_t endTime = taosGetTimestampUs(); - int64_t el = endTime - startTime; + int64_t el = endTime - startTime; - printf("End of Test, %" PRIu64 "\nTotal Elapsed Time:%" PRIu64 " us.avg:%f us\n", endTime, el, el/1000.0); + printf("End of Test, %" PRIu64 "\nTotal Elapsed Time:%" PRIu64 " us.avg:%f us\n", endTime, el, el / 1000.0); taosCacheCleanup(tscMetaCache); } TEST(cacheTest, cache_iter_test) { const int32_t REFRESH_TIME_IN_SEC = 2; - auto* pCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC, false, NULL, "test"); + auto* pCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC * 1000, false, NULL, "test"); char key[256] = {0}; char data[1024] = "abcdefghijk"; -// int32_t len = strlen(data); + // int32_t len = strlen(data); uint64_t startTime = taosGetTimestampUs(); - int32_t num = 10000; + int32_t num = 10000; - for(int32_t i = 0; i < num; ++i) { + for (int32_t i = 0; i < num; ++i) { int32_t len = sprintf(key, "abc_%7d", i); taosCachePut(pCache, key, strlen(key), data, len, 3600); } uint64_t endTime = taosGetTimestampUs(); - printf("add %d object cost:%" PRIu64 " us, avg:%f us\n", num, endTime - startTime, (endTime-startTime)/(double)num); + printf("add %d object cost:%" PRIu64 " us, avg:%f us\n", num, endTime - startTime, + (endTime - startTime) / (double)num); startTime = taosGetTimestampUs(); - for(int32_t i = 0; i < num; ++i) { + for (int32_t i = 0; i < num; ++i) { int32_t len = sprintf(key, "abc_%7d", i); - void* k = taosCacheAcquireByKey(pCache, key, len); + void* k = taosCacheAcquireByKey(pCache, key, len); assert(k != 0); } endTime = taosGetTimestampUs(); - printf("retrieve %d object cost:%" PRIu64 " us,avg:%f\n", num, endTime - startTime, (endTime - startTime)/(double)num); + printf("retrieve %d object cost:%" PRIu64 " us,avg:%f\n", num, endTime - startTime, + (endTime - startTime) / (double)num); - int32_t count = 0; + int32_t count = 0; SCacheIter* pIter = taosCacheCreateIter(pCache); - while(taosCacheIterNext(pIter)) { + while (taosCacheIterNext(pIter)) { size_t keyLen = 0; size_t dataLen = 0; char* key1 = static_cast(taosCacheIterGetKey(pIter, &keyLen)); char* data1 = static_cast(taosCacheIterGetData(pIter, &dataLen)); -// char d[256] = {0}; -// memcpy(d, data1, dataLen); -// char k[256] = {0}; -// memcpy(k, key1, keyLen); + // char d[256] = {0}; + // memcpy(d, data1, dataLen); + // char k[256] = {0}; + // memcpy(k, key1, keyLen); } ASSERT_EQ(count, num); From b816d7091e0894d4ccf11bab45ccd3456967336c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 22 Apr 2022 20:10:44 +0800 Subject: [PATCH 80/83] add multi process case for tmq --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 51b188c8db..31b1fff826 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -76,7 +76,7 @@ ./test.sh -f tsim/insert/backquote.sim -m ./test.sh -f tsim/parser/fourArithmetic-basic.sim -m ./test.sh -f tsim/query/interval-offset.sim -m -#./test.sh -f tsim/tmq/basic.sim -m +./test.sh -f tsim/tmq/basic1.sim -m ./test.sh -f tsim/stable/vnode3.sim -m ./test.sh -f tsim/qnode/basic1.sim -m ./test.sh -f tsim/mnode/basic1.sim -m From aade540f0514ba9aa489fb040de70c35a01d1a85 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 22 Apr 2022 21:02:21 +0800 Subject: [PATCH 81/83] remove unused msg --- include/common/tmsgdef.h | 4 -- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 3 - source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 4 -- source/dnode/vnode/src/inc/tq.h | 34 +--------- source/dnode/vnode/src/tq/tq.c | 11 ++-- source/dnode/vnode/src/tq/tqPush.c | 70 --------------------- source/dnode/vnode/src/vnd/vnodeSvr.c | 15 ----- 7 files changed, 8 insertions(+), 133 deletions(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index d0f88a660c..b37001bde9 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -175,11 +175,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_MQ_QUERY, "vnode-mq-query", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONNECT, "vnode-mq-connect", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", SMqSetCVgReq, SMqSetCVgRsp) - TD_DEF_MSG_TYPE(TDMT_VND_MQ_REB, "vnode-mq-mv-rebalance", SMqMVRebReq, SMqMVRebRsp) - TD_DEF_MSG_TYPE(TDMT_VND_MQ_CANCEL_CONN, "vnode-mq-mv-cancel-conn", SMqCancelConnReq, SMqCancelConnRsp) TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp) - TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_TASK, "vnode-cancel-task", NULL, NULL) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 411eff5374..159e14a3d5 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -218,9 +218,6 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_MND_GET_INDEX, mmProcessReadMsg, DEFAULT_HANDLE); // Requests handled by VNODE - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 65180f84dd..4cc1b8527c 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -276,10 +276,6 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); - // dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, vmProcessWriteMsg, DEFAULT_HANDLE); - // dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, vmProcessWriteMsg, DEFAULT_HANDLE); - // dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN, vmProcessWriteMsg, DEFAULT_HANDLE); - // dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessFetchMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_CONSUME, vmProcessFetchMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, vmProcessWriteMsg, DEFAULT_HANDLE); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 2250aab35c..6e004a89fc 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -83,7 +83,6 @@ typedef struct STqOffsetStore STqOffsetStore; struct STqReadHandle { int64_t ver; - int64_t tbUid; SHashObj* tbIdHash; const SSubmitReq* pMsg; SSubmitBlk* pBlock; @@ -218,20 +217,6 @@ typedef struct { SArray* topics; // SArray } STqConsumer; -enum { - TQ_PUSHER_TYPE__CLIENT = 1, - TQ_PUSHER_TYPE__STREAM, -}; - -typedef struct { - int8_t type; - int8_t reserved[3]; - int32_t ttl; - int64_t consumerId; - SRpcMsg* pMsg; - // SMqPollRsp* rsp; -} STqClientPusher; - typedef struct { int8_t type; int8_t nodeType; @@ -241,10 +226,6 @@ typedef struct { // TODO sync function } STqStreamPusher; -typedef struct { - SHashObj* pHash; // -} STqPushMgr; - typedef struct { int8_t inited; tmr_h timer; @@ -260,14 +241,11 @@ void tqCleanUp(); STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal); void tqClose(STQ*); // required by vnode -int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version); +int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); int tqCommit(STQ*); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId); int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); -// int32_t tqProcessSetConnReq(STQ* pTq, char* msg); -// int32_t tqProcessRebReq(STQ* pTq, char* msg); -// int32_t tqProcessCancelConnReq(STQ* pTq, char* msg); int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId); int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId); @@ -308,16 +286,6 @@ int32_t tqOffsetCommit(STqOffsetStore* pStore, const char* subscribeKey, int64_t int32_t tqOffsetPersist(STqOffsetStore* pStore, const char* subscribeKey); int32_t tqOffsetPersistAll(STqOffsetStore* pStore); -// tqPush -int32_t tqPushMgrInit(); -void tqPushMgrCleanUp(); - -STqPushMgr* tqPushMgrOpen(); -void tqPushMgrClose(STqPushMgr* pushMgr); - -STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl); -STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet); - #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index d0018d2cac..0b3feb5010 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -15,9 +15,12 @@ #include "vnodeInt.h" -int32_t tqInit() { return tqPushMgrInit(); } +int32_t tqInit() { + // + return 0; +} -void tqCleanUp() { tqPushMgrCleanUp(); } +void tqCleanUp() {} STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { STQ* pTq = taosMemoryMalloc(sizeof(STQ)); @@ -184,7 +187,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ return 0; } -int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version) { +int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { if (msgType != TDMT_VND_SUBMIT) return 0; void* data = taosMemoryMalloc(msgLen); @@ -194,7 +197,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t versi memcpy(data, msg, msgLen); if (msgType == TDMT_VND_SUBMIT) { - if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg, version) != 0) { + if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg, ver) != 0) { return -1; } } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 4384255f89..f2f48bbc8a 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -12,73 +12,3 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - -#include "vnodeInt.h" - -int32_t tqPushMgrInit() { - // - int8_t old = atomic_val_compare_exchange_8(&tqPushMgmt.inited, 0, 1); - if (old == 1) return 0; - - tqPushMgmt.timer = taosTmrInit(0, 0, 0, "TQ"); - return 0; -} - -void tqPushMgrCleanUp() { - int8_t old = atomic_val_compare_exchange_8(&tqPushMgmt.inited, 1, 0); - if (old == 0) return; - taosTmrStop(tqPushMgmt.timer); - taosTmrCleanUp(tqPushMgmt.timer); -} - -STqPushMgr* tqPushMgrOpen() { - STqPushMgr* mgr = taosMemoryMalloc(sizeof(STqPushMgr)); - if (mgr == NULL) { - return NULL; - } - mgr->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - return mgr; -} - -void tqPushMgrClose(STqPushMgr* pushMgr) { - taosHashCleanup(pushMgr->pHash); - taosMemoryFree(pushMgr); -} - -STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl) { - STqClientPusher* clientPusher = taosMemoryMalloc(sizeof(STqClientPusher)); - if (clientPusher == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - clientPusher->type = TQ_PUSHER_TYPE__CLIENT; - clientPusher->pMsg = pMsg; - clientPusher->consumerId = consumerId; - clientPusher->ttl = ttl; - if (taosHashPut(pushMgr->pHash, &consumerId, sizeof(int64_t), &clientPusher, sizeof(void*)) < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(clientPusher); - // TODO send rsp back - return NULL; - } - return clientPusher; -} - -STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet) { - STqStreamPusher* streamPusher = taosMemoryMalloc(sizeof(STqStreamPusher)); - if (streamPusher == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - streamPusher->type = TQ_PUSHER_TYPE__STREAM; - streamPusher->nodeType = 0; - streamPusher->streamId = streamId; - /*memcpy(&streamPusher->epSet, pEpSet, sizeof(SEpSet));*/ - - if (taosHashPut(pushMgr->pHash, &streamId, sizeof(int64_t), &streamPusher, sizeof(void*)) < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(streamPusher); - return NULL; - } - return streamPusher; -} diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index e0c37f7eb7..aa90a04478 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -90,21 +90,6 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg // TODO: handle error } break; -#if 0 - case TDMT_VND_MQ_SET_CONN: { - if (tqProcessSetConnReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { - // TODO: handle error - } - } break; - case TDMT_VND_MQ_REB: { - if (tqProcessRebReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { - } - } break; - case TDMT_VND_MQ_CANCEL_CONN: { - if (tqProcessCancelConnReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { - } - } break; -#endif case TDMT_VND_TASK_DEPLOY: { if (tqProcessTaskDeploy(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead)) < 0) { From 5948ddd6971368d79fc3cd8417fdd6bc1c6528c6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 23 Apr 2022 00:13:09 +0800 Subject: [PATCH 82/83] fix: invalid free in unittest --- source/dnode/mgmt/implement/src/dmExec.c | 1 + source/dnode/mgmt/interface/src/dmEnv.c | 1 + source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/implement/src/dmExec.c b/source/dnode/mgmt/implement/src/dmExec.c index c0b85ed705..06001028b5 100644 --- a/source/dnode/mgmt/implement/src/dmExec.c +++ b/source/dnode/mgmt/implement/src/dmExec.c @@ -174,6 +174,7 @@ void dmStopNode(SMgmtWrapper *pWrapper) { void dmCloseNode(SMgmtWrapper *pWrapper) { dInfo("node:%s, start to close", pWrapper->name); + pWrapper->deployed = false; while (pWrapper->refCount > 0) { taosMsleep(10); diff --git a/source/dnode/mgmt/interface/src/dmEnv.c b/source/dnode/mgmt/interface/src/dmEnv.c index b6f309ec6f..92a6e018cb 100644 --- a/source/dnode/mgmt/interface/src/dmEnv.c +++ b/source/dnode/mgmt/interface/src/dmEnv.c @@ -53,6 +53,7 @@ void dmCleanup() { } monCleanup(); + syncCleanUp(); walCleanUp(); taosStopCacheRefreshWorker(); dInfo("dnode env is cleaned up"); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index ffbc6eb3b9..e784272581 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -274,7 +274,7 @@ static void vmCleanup(SMgmtWrapper *pWrapper) { taosMemoryFree(pMgmt); pWrapper->pMgmt = NULL; - syncCleanUp(); + // syncCleanUp(); dInfo("vnode-mgmt is cleaned up"); } From 5e4737f9a02ca36d90983054ea6fe4a2b6d8d361 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 23 Apr 2022 00:27:46 +0800 Subject: [PATCH 83/83] fix: sleep 1000ms while unit test creates db so vnode becomes leader --- source/dnode/mnode/impl/test/db/db.cpp | 1 + source/dnode/mnode/impl/test/sma/sma.cpp | 1 + source/dnode/mnode/impl/test/stb/stb.cpp | 8 ++++++++ 3 files changed, 10 insertions(+) diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index 6e72ce89d3..1fc1bec650 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -89,6 +89,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { void* pReq = rpcMallocCont(contLen); tSerializeSAlterDbReq(pReq, contLen, &alterdbReq); + taosMsleep(1000); // Wait for the vnode to become the leader SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); diff --git a/source/dnode/mnode/impl/test/sma/sma.cpp b/source/dnode/mnode/impl/test/sma/sma.cpp index 0d41d5de20..96c0c8e953 100644 --- a/source/dnode/mnode/impl/test/sma/sma.cpp +++ b/source/dnode/mnode/impl/test/sma/sma.cpp @@ -258,6 +258,7 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) { pReq = BuildCreateDbReq(dbname, &contLen); pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_EQ(pRsp->code, 0); + taosMsleep(1000); // Wait for the vnode to become the leader } { diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index 9270d38e11..0c54091aa9 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -304,6 +304,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); + taosMsleep(2000); // Wait for the vnode to become the leader } { @@ -438,6 +439,7 @@ TEST_F(MndTestStb, 02_Alter_Stb_AddTag) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); + taosMsleep(2000); // Wait for the vnode to become the leader } { @@ -497,6 +499,7 @@ TEST_F(MndTestStb, 03_Alter_Stb_DropTag) { void* pReq = BuildCreateDbReq(dbname, &contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_EQ(pRsp->code, 0); + taosMsleep(2000); // Wait for the vnode to become the leader } { @@ -538,6 +541,7 @@ TEST_F(MndTestStb, 04_Alter_Stb_AlterTagName) { void* pReq = BuildCreateDbReq(dbname, &contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_EQ(pRsp->code, 0); + taosMsleep(2000); // Wait for the vnode to become the leader } { @@ -602,6 +606,7 @@ TEST_F(MndTestStb, 05_Alter_Stb_AlterTagBytes) { void* pReq = BuildCreateDbReq(dbname, &contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_EQ(pRsp->code, 0); + taosMsleep(2000); // Wait for the vnode to become the leader } { @@ -655,6 +660,7 @@ TEST_F(MndTestStb, 06_Alter_Stb_AddColumn) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); + taosMsleep(2000); // Wait for the vnode to become the leader } { @@ -715,6 +721,7 @@ TEST_F(MndTestStb, 07_Alter_Stb_DropColumn) { void* pReq = BuildCreateDbReq(dbname, &contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_EQ(pRsp->code, 0); + taosMsleep(2000); // Wait for the vnode to become the leader } { @@ -775,6 +782,7 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) { void* pReq = BuildCreateDbReq(dbname, &contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_EQ(pRsp->code, 0); + taosMsleep(2000); // Wait for the vnode to become the leader } {