diff --git a/example/src/tstream.c b/example/src/tstream.c index 62d94b041d..51578bd27b 100644 --- a/example/src/tstream.c +++ b/example/src/tstream.c @@ -63,7 +63,7 @@ int32_t init_env() { } int32_t create_stream() { - printf("create topic\n"); + printf("create stream\n"); TAOS_RES* pRes; TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); if (pConn == NULL) { @@ -77,7 +77,9 @@ int32_t create_stream() { } taos_free_result(pRes); - const char* sql = "select ts,k from tu1"; + /*const char* sql = "select min(k), max(k), sum(k) from tu1";*/ + const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1"; + /*const char* sql = "select sum(k) from tu1 interval(10m)";*/ pRes = tmq_create_stream(pConn, "stream1", "out1", sql); if (taos_errno(pRes) != 0) { printf("failed to create stream out1, reason:%s\n", taos_errstr(pRes)); diff --git a/include/common/taosdef.h b/include/common/taosdef.h index f252143fa6..5f7bebdeb6 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -63,9 +63,12 @@ typedef enum { } ETsdbStatisStatus; typedef enum { - TSDB_SMA_STAT_OK = 0, // ready to provide service - TSDB_SMA_STAT_EXPIRED = 1, // not ready or expired -} ETsdbSmaStat; + TSDB_SMA_STAT_UNKNOWN = -1, // unknown + TSDB_SMA_STAT_OK = 0, // ready to provide service + TSDB_SMA_STAT_EXPIRED = 1, // not ready or expired + TSDB_SMA_STAT_DROPPED = 2, // sma dropped +} ETsdbSmaStat; // bit operation + typedef enum { TSDB_SMA_TYPE_BLOCK = 0, // Block-wise SMA @@ -75,6 +78,8 @@ typedef enum { extern char *qtypeStr[]; +#define TSDB_PORT_DNODEDNODE 5 +#define TSDB_PORT_SYNC 10 #define TSDB_PORT_HTTP 11 #ifdef __cplusplus diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9572d9d784..9fbd8e08cf 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -23,8 +23,8 @@ #include "tencode.h" #include "thash.h" #include "tlist.h" -#include "trow.h" #include "tname.h" +#include "trow.h" #include "tuuid.h" #ifdef __cplusplus @@ -109,6 +109,7 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_STREAMTABLES, TSDB_MGMT_TABLE_TP, TSDB_MGMT_TABLE_FUNC, + TSDB_MGMT_TABLE_INDEX, TSDB_MGMT_TABLE_MAX, } EShowType; @@ -270,9 +271,10 @@ typedef struct { int8_t igExists; int32_t numOfColumns; int32_t numOfTags; + int32_t commentLen; SArray* pColumns; SArray* pTags; - char comment[TSDB_STB_COMMENT_LEN]; + char *comment; } SMCreateStbReq; int32_t tSerializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pReq); @@ -473,10 +475,9 @@ typedef struct { int32_t code; } SQueryTableRsp; -int32_t tSerializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp); - -int32_t tDeserializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp); +int32_t tSerializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp); +int32_t tDeserializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp); typedef struct { char db[TSDB_DB_FNAME_LEN]; @@ -889,14 +890,14 @@ typedef struct { } SRetrieveTableRsp; typedef struct { - int64_t handle; - int64_t useconds; - int8_t completed; // all results are returned to client - int8_t precision; - int8_t compressed; - int32_t compLen; - int32_t numOfRows; - char data[]; + int64_t handle; + int64_t useconds; + int8_t completed; // all results are returned to client + int8_t precision; + int8_t compressed; + int32_t compLen; + int32_t numOfRows; + char data[]; } SRetrieveMetaTableRsp; typedef struct { @@ -1426,12 +1427,11 @@ int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq); void* tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq); typedef struct { - SArray* rspList; // SArray + SArray* rspList; // SArray } SVCreateTbBatchRsp; -int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp); -int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp); - +int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); +int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); typedef struct { int64_t ver; @@ -1941,12 +1941,47 @@ static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) } return buf; } + +typedef struct { + char name[TSDB_TABLE_FNAME_LEN]; + char stb[TSDB_TABLE_FNAME_LEN]; + int8_t igExists; + int8_t intervalUnit; + int8_t slidingUnit; + int8_t timezone; + int32_t dstVgId; // for stream + int64_t interval; + int64_t offset; + int64_t sliding; + int32_t exprLen; // strlen + 1 + int32_t tagsFilterLen; // strlen + 1 + int32_t sqlLen; // strlen + 1 + int32_t astLen; // strlen + 1 + char* expr; + char* tagsFilter; + char* sql; + char* ast; +} SMCreateSmaReq; + +int32_t tSerializeSMCreateSmaReq(void* buf, int32_t bufLen, SMCreateSmaReq* pReq); +int32_t tDeserializeSMCreateSmaReq(void* buf, int32_t bufLen, SMCreateSmaReq* pReq); +void tFreeSMCreateSmaReq(SMCreateSmaReq* pReq); + +typedef struct { + char name[TSDB_TABLE_FNAME_LEN]; + int8_t igNotExists; +} SMDropSmaReq; + +int32_t tSerializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); +int32_t tDeserializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); + typedef struct { int8_t version; // for compatibility(default 0) int8_t intervalUnit; // MACRO: TIME_UNIT_XXX int8_t slidingUnit; // MACRO: TIME_UNIT_XXX + int8_t timezoneInt; // sma data expired if timezone changes. char indexName[TSDB_INDEX_NAME_LEN]; - char timezone[TD_TIMEZONE_LEN]; // sma data expired if timezone changes. + char timezone[TD_TIMEZONE_LEN]; int32_t exprLen; int32_t tagsFilterLen; int64_t indexUid; @@ -2053,8 +2088,8 @@ static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) { tlen += taosEncodeFixedI8(buf, pSma->version); tlen += taosEncodeFixedI8(buf, pSma->intervalUnit); tlen += taosEncodeFixedI8(buf, pSma->slidingUnit); + tlen += taosEncodeFixedI8(buf, pSma->timezoneInt); tlen += taosEncodeString(buf, pSma->indexName); - tlen += taosEncodeString(buf, pSma->timezone); tlen += taosEncodeFixedI32(buf, pSma->exprLen); tlen += taosEncodeFixedI32(buf, pSma->tagsFilterLen); tlen += taosEncodeFixedI64(buf, pSma->indexUid); @@ -2088,8 +2123,8 @@ static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) { buf = taosDecodeFixedI8(buf, &pSma->version); buf = taosDecodeFixedI8(buf, &pSma->intervalUnit); buf = taosDecodeFixedI8(buf, &pSma->slidingUnit); + buf = taosDecodeFixedI8(buf, &pSma->timezoneInt); buf = taosDecodeStringTo(buf, pSma->indexName); - buf = taosDecodeStringTo(buf, pSma->timezone); buf = taosDecodeFixedI32(buf, &pSma->exprLen); buf = taosDecodeFixedI32(buf, &pSma->tagsFilterLen); buf = taosDecodeFixedI64(buf, &pSma->indexUid); @@ -2313,6 +2348,11 @@ enum { STREAM_TASK_STATUS__STOP, }; +enum { + STREAM_NEXT_OP_DST__VND = 1, + STREAM_NEXT_OP_DST__SND, +}; + typedef struct { void* inputHandle; void* executor; @@ -2327,6 +2367,7 @@ typedef struct { int8_t pipeSink; int8_t numOfRunners; int8_t parallelizable; + int8_t nextOpDst; // vnode or snode SEpSet NextOpEp; char* qmsg; // not applied to encoder and decoder @@ -2342,6 +2383,8 @@ static FORCE_INLINE SStreamTask* streamTaskNew(int64_t streamId, int32_t level) return NULL; } pTask->taskId = tGenIdPI32(); + pTask->streamId = streamId; + pTask->level = level; pTask->status = STREAM_TASK_STATUS__RUNNING; pTask->qmsg = NULL; return pTask; @@ -2369,6 +2412,13 @@ typedef struct { int32_t reserved; } SStreamTaskExecRsp; +typedef struct { + SMsgHead head; + int64_t streamId; + int64_t version; + SArray* res; // SArray +} SStreamSmaSinkReq; + #pragma pack(pop) #ifdef __cplusplus diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 73a78131dc..f9ce69925b 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -127,6 +127,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STB, "mnode-create-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STB, "mnode-alter-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_STB, "mnode-drop-stb", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_SMA, "mnode-create-sma", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_SMA, "mnode-drop-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TABLE_META, "mnode-table-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_VGROUP_LIST, "mnode-vgroup-list", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_QNODE_LIST, "mnode-qnode-list", NULL, NULL) @@ -191,6 +193,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp) TD_DEF_MSG_TYPE(TDMT_VND_TASK_DEPLOY, "vnode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp) TD_DEF_MSG_TYPE(TDMT_VND_TASK_EXEC, "vnode-task-exec", SStreamTaskExecReq, SStreamTaskExecRsp) + TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) diff --git a/include/common/trow.h b/include/common/trow.h index 4ae5a2277d..fc99cbc5b2 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -166,6 +166,7 @@ typedef struct { #define TD_ROW_HEAD_LEN (sizeof(STSRow)) #define TD_ROW_NCOLS_LEN (sizeof(col_id_t)) +#define TD_ROW_INFO(r) ((r)->info) #define TD_ROW_TYPE(r) ((r)->type) #define TD_ROW_DELETE(r) ((r)->del) #define TD_ROW_ENDIAN(r) ((r)->endian) @@ -180,6 +181,7 @@ typedef struct { // (int32_t)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined. #define TD_ROW_MAX_BYTES_FROM_SCHEMA(s) (schemaTLen(s) + TD_ROW_HEAD_LEN) +#define TD_ROW_SET_INFO(r, i) (TD_ROW_INFO(r) = (i)) #define TD_ROW_SET_TYPE(r, t) (TD_ROW_TYPE(r) = (t)) #define TD_ROW_SET_DELETE(r) (TD_ROW_DELETE(r) = 1) #define TD_ROW_SET_SVER(r, v) (TD_ROW_SVER(r) = (v)) @@ -473,6 +475,7 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { return terrno; } + TD_ROW_SET_INFO(pBuilder->pBuf, 0); TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType); uint32_t len = 0; diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index d04e9f817e..a28c093e85 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -119,10 +119,11 @@ typedef enum { SDB_CONSUMER = 13, SDB_TOPIC = 14, SDB_VGROUP = 15, - SDB_STB = 16, - SDB_DB = 17, - SDB_FUNC = 18, - SDB_MAX = 19 + SDB_SMA = 16, + SDB_STB = 17, + SDB_DB = 18, + SDB_FUNC = 19, + SDB_MAX = 20 } ESdbType; typedef struct SSdb SSdb; diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index a38431a1b2..a2f88490f0 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -35,6 +35,7 @@ typedef enum { TAOS_SYNC_STATE_FOLLOWER = 100, TAOS_SYNC_STATE_CANDIDATE = 101, TAOS_SYNC_STATE_LEADER = 102, + TAOS_SYNC_STATE_ERROR = 103, } ESyncState; typedef struct SSyncBuffer { @@ -68,17 +69,20 @@ typedef struct SSnapshot { typedef struct SSyncFSM { void* data; - // when value in pBuf finish a raft flow, FpCommitCb is called, code indicates the result + // when value in pMsg finish a raft flow, FpCommitCb is called, code indicates the result // user can do something according to the code and isWeak. for example, write data into tsdb - void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pBuf, SyncIndex index, bool isWeak, int32_t code); + void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state); - // when value in pBuf has been written into local log store, FpPreCommitCb is called, code indicates the result + // when value in pMsg has been written into local log store, FpPreCommitCb is called, code indicates the result // user can do something according to the code and isWeak. for example, write data into tsdb - void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pBuf, SyncIndex index, bool isWeak, int32_t code); + void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state); // when log entry is updated by a new one, FpRollBackCb is called // user can do something to roll back. for example, delete data from tsdb, or just ignore it - void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pBuf, SyncIndex index, bool isWeak, int32_t code); + void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state); // user should implement this function, use "data" to take snapshot into "snapshot" int32_t (*FpTakeSnapshot)(SSnapshot* snapshot); @@ -157,10 +161,14 @@ void syncCleanUp(); int64_t syncStart(const SSyncInfo* pSyncInfo); void syncStop(int64_t rid); int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg); -int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak); // use this function -int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak); // just for compatibility +int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak); ESyncState syncGetMyRole(int64_t rid); -void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole); + +// propose with sequence number, to implement linearizable semantics +int32_t syncPropose2(int64_t rid, const SRpcMsg* pMsg, bool isWeak, uint64_t seqNum); + +// for compatibility, the same as syncPropose +int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak); extern int32_t sDebugFlag; diff --git a/source/libs/transport/inc/rpcHead.h b/include/libs/transport/rpcHead.h similarity index 100% rename from source/libs/transport/inc/rpcHead.h rename to include/libs/transport/rpcHead.h diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 3c8cc8abd1..7af8dd37bf 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -51,10 +51,31 @@ extern "C" { #endif -#if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) +#if defined(WINDOWS) #define htobe64 htonll #endif +#if defined(WINDOWS) + #define TAOS_EPOLL_WAIT_TIME 100 + typedef SOCKET eventfd_t; + #define eventfd(a, b) -1 + typedef SOCKET EpollFd; + #define EpollClose(pollFd) epoll_close(pollFd) + #ifndef EPOLLWAKEUP + #define EPOLLWAKEUP (1u << 29) + #endif +#elif defined(_TD_DARWIN_64) + #define TAOS_EPOLL_WAIT_TIME 500 + typedef int32_t SOCKET; + typedef SOCKET EpollFd; + #define EpollClose(pollFd) epoll_close(pollFd) +#else + #define TAOS_EPOLL_WAIT_TIME 500 + typedef int32_t SOCKET; + typedef SOCKET EpollFd; + #define EpollClose(pollFd) taosCloseSocket(pollFd) +#endif + #if defined(_TD_DARWIN_64) // #define htobe64 htonll @@ -64,12 +85,12 @@ extern "C" { # define htole16(x) OSSwapHostToLittleInt16(x) # define be16toh(x) OSSwapBigToHostInt16(x) # define le16toh(x) OSSwapLittleToHostInt16(x) - + # define htobe32(x) OSSwapHostToBigInt32(x) # define htole32(x) OSSwapHostToLittleInt32(x) # define be32toh(x) OSSwapBigToHostInt32(x) # define le32toh(x) OSSwapLittleToHostInt32(x) - + # define htobe64(x) OSSwapHostToBigInt64(x) # define htole64(x) OSSwapHostToLittleInt64(x) # define be64toh(x) OSSwapBigToHostInt64(x) @@ -83,6 +104,17 @@ extern "C" { #define TAOS_EPOLL_WAIT_TIME 500 +typedef int32_t SocketFd; +typedef SocketFd EpollFd; + +typedef struct TdSocket { +#if SOCKET_WITH_LOCK + TdThreadRwlock rwlock; +#endif + int refId; + SocketFd fd; +} *TdSocketPtr, TdSocket; + typedef struct TdSocketServer *TdSocketServerPtr; typedef struct TdSocket *TdSocketPtr; typedef struct TdEpoll *TdEpollPtr; @@ -91,6 +123,7 @@ int32_t taosSendto(TdSocketPtr pSocket, void * msg, int len, unsigned int flags, int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len); int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len); int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen); +int32_t taosCloseSocketNoCheck1(SocketFd fd); int32_t taosCloseSocket(TdSocketPtr *ppSocket); int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer); int32_t taosShutDownSocketRD(TdSocketPtr pSocket); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 9e1e1c53dc..2fb4657407 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -274,18 +274,23 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_STREAM_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03F1) #define TSDB_CODE_MND_INVALID_STREAM_OPTION TAOS_DEF_ERROR_CODE(0, 0x03F2) +// mnode-sma +#define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0400) +#define TSDB_CODE_MND_SMA_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0401) +#define TSDB_CODE_MND_INVALID_SMA_OPTION TAOS_DEF_ERROR_CODE(0, 0x0402) + // dnode -#define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) -#define TSDB_CODE_DND_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0401) -#define TSDB_CODE_DND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0402) -#define TSDB_CODE_NODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0403) -#define TSDB_CODE_NODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0404) -#define TSDB_CODE_NODE_PARSE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0405) -#define TSDB_CODE_NODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0406) -#define TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0410) -#define TSDB_CODE_DND_VNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0411) -#define TSDB_CODE_DND_VNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0412) -#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0413) +#define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x04A0) +#define TSDB_CODE_DND_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x04A1) +#define TSDB_CODE_DND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x04A2) +#define TSDB_CODE_NODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x04A3) +#define TSDB_CODE_NODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x04A4) +#define TSDB_CODE_NODE_PARSE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x04A5) +#define TSDB_CODE_NODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x04A6) +#define TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x04A7) +#define TSDB_CODE_DND_VNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x04A8) +#define TSDB_CODE_DND_VNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x04A9) +#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x04AA) // vnode #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) @@ -309,7 +314,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_IS_SYNCING TAOS_DEF_ERROR_CODE(0, 0x0513) #define TSDB_CODE_VND_INVALID_TSDB_STATE TAOS_DEF_ERROR_CODE(0, 0x0514) #define TSDB_CODE_VND_TB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0515) -#define TSDB_CODE_VND_HASH_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x0516) +#define TSDB_CODE_VND_SMA_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0516) +#define TSDB_CODE_VND_HASH_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x0517) // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) @@ -336,8 +342,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615) #define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0616) #define TSDB_CODE_TDB_TABLE_RECREATED TAOS_DEF_ERROR_CODE(0, 0x0617) -#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x0618) -#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0619) +#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0618) +#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x0619) +#define TSDB_CODE_TDB_INVALID_SMA_STAT TAOS_DEF_ERROR_CODE(0, 0x0620) // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 87e3cf0c24..ed5441fe99 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -421,6 +421,7 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) { return NULL; } +#if 0 SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { SDataCols *pRet = tdNewDataCols(pDataCols->maxCols, pDataCols->maxPoints); if (pRet == NULL) return NULL; @@ -454,6 +455,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { return pRet; } +#endif void tdResetDataCols(SDataCols *pCols) { if (pCols != NULL) { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 841824a7c7..506e017deb 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -132,6 +132,9 @@ bool tsdbForceKeepFile = false; int32_t tsDiskCfgNum = 0; SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0}; +// stream scheduler +bool tsStreamSchedV = true; + /* * minimum scale for whole system, millisecond by default * for TSDB_TIME_PRECISION_MILLI: 86400000L @@ -585,4 +588,4 @@ void taosCfgDynamicOptions(const char *option, const char *value) { taosResetLog(); cfgDumpCfg(tsCfg, 1, false); } -} \ No newline at end of file +} diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 1ea00566f4..e39fe02f29 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -313,12 +313,12 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) { tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pBSmaCols[i]); } - if(pReq->rollup && NULL != pReq->stbCfg.pRSmaParam) { + if(pReq->rollup && pReq->stbCfg.pRSmaParam) { SRSmaParam *param = pReq->stbCfg.pRSmaParam; tlen += taosEncodeFixedU32(buf, (uint32_t)param->xFilesFactor); tlen += taosEncodeFixedI8(buf, param->delayUnit); tlen += taosEncodeFixedI8(buf, param->nFuncIds); - for(int8_t i=0; i< param->nFuncIds; ++i) { + for (int8_t i = 0; i < param->nFuncIds; ++i) { tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]); } tlen += taosEncodeFixedI64(buf, param->delay); @@ -336,16 +336,16 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes); tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name); } - tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols); - for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) { - tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pBSmaCols[i]); + tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nBSmaCols); + for (col_id_t i = 0; i < pReq->ntbCfg.nBSmaCols; ++i) { + tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pBSmaCols[i]); } - if(pReq->rollup && NULL != pReq->stbCfg.pRSmaParam) { - SRSmaParam *param = pReq->stbCfg.pRSmaParam; + if(pReq->rollup && pReq->ntbCfg.pRSmaParam) { + SRSmaParam *param = pReq->ntbCfg.pRSmaParam; tlen += taosEncodeFixedU32(buf, (uint32_t)param->xFilesFactor); tlen += taosEncodeFixedI8(buf, param->delayUnit); tlen += taosEncodeFixedI8(buf, param->nFuncIds); - for(int8_t i=0; i< param->nFuncIds; ++i) { + for (int8_t i = 0; i < param->nFuncIds; ++i) { tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]); } tlen += taosEncodeFixedI64(buf, param->delay); @@ -385,7 +385,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name); } buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols)); - if(pReq->stbCfg.nBSmaCols > 0) { + if (pReq->stbCfg.nBSmaCols > 0) { pReq->stbCfg.pBSmaCols = (col_id_t *)malloc(pReq->stbCfg.nBSmaCols * sizeof(col_id_t)); for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) { buf = taosDecodeFixedI16(buf, pReq->stbCfg.pBSmaCols + i); @@ -393,14 +393,14 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { } else { pReq->stbCfg.pBSmaCols = NULL; } - if(pReq->rollup) { + if (pReq->rollup) { pReq->stbCfg.pRSmaParam = (SRSmaParam *)malloc(sizeof(SRSmaParam)); SRSmaParam *param = pReq->stbCfg.pRSmaParam; - buf = taosDecodeFixedU32(buf, (uint32_t*)¶m->xFilesFactor); + buf = taosDecodeFixedU32(buf, (uint32_t *)¶m->xFilesFactor); buf = taosDecodeFixedI8(buf, ¶m->delayUnit); buf = taosDecodeFixedI8(buf, ¶m->nFuncIds); - if(param->nFuncIds > 0) { - for (int8_t i = 0; i< param->nFuncIds; ++i) { + if (param->nFuncIds > 0) { + for (int8_t i = 0; i < param->nFuncIds; ++i) { buf = taosDecodeFixedI32(buf, param->pFuncIds + i); } } else { @@ -424,23 +424,23 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes); buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name); } - buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols)); - if(pReq->stbCfg.nBSmaCols > 0) { - pReq->stbCfg.pBSmaCols = (col_id_t *)malloc(pReq->stbCfg.nBSmaCols * sizeof(col_id_t)); - for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) { - buf = taosDecodeFixedI16(buf, pReq->stbCfg.pBSmaCols + i); + buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols)); + if(pReq->ntbCfg.nBSmaCols > 0) { + pReq->ntbCfg.pBSmaCols = (col_id_t *)malloc(pReq->ntbCfg.nBSmaCols * sizeof(col_id_t)); + for (col_id_t i = 0; i < pReq->ntbCfg.nBSmaCols; ++i) { + buf = taosDecodeFixedI16(buf, pReq->ntbCfg.pBSmaCols + i); } } else { - pReq->stbCfg.pBSmaCols = NULL; + pReq->ntbCfg.pBSmaCols = NULL; } if(pReq->rollup) { - pReq->stbCfg.pRSmaParam = (SRSmaParam *)malloc(sizeof(SRSmaParam)); - SRSmaParam *param = pReq->stbCfg.pRSmaParam; + pReq->ntbCfg.pRSmaParam = (SRSmaParam *)malloc(sizeof(SRSmaParam)); + SRSmaParam *param = pReq->ntbCfg.pRSmaParam; buf = taosDecodeFixedU32(buf, (uint32_t*)¶m->xFilesFactor); buf = taosDecodeFixedI8(buf, ¶m->delayUnit); buf = taosDecodeFixedI8(buf, ¶m->nFuncIds); - if(param->nFuncIds > 0) { - for (int8_t i = 0; i< param->nFuncIds; ++i) { + if (param->nFuncIds > 0) { + for (int8_t i = 0; i < param->nFuncIds; ++i) { buf = taosDecodeFixedI32(buf, param->pFuncIds + i); } } else { @@ -448,7 +448,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { } buf = taosDecodeFixedI64(buf, ¶m->delay); } else { - pReq->stbCfg.pRSmaParam = NULL; + pReq->ntbCfg.pRSmaParam = NULL; } break; default: @@ -510,6 +510,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfColumns) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfTags) < 0) return -1; + if (tEncodeI32(&encoder, pReq->commentLen) < 0) return -1; for (int32_t i = 0; i < pReq->numOfColumns; ++i) { SField *pField = taosArrayGet(pReq->pColumns, i); @@ -525,7 +526,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq if (tEncodeCStr(&encoder, pField->name) < 0) return -1; } - if (tEncodeCStr(&encoder, pReq->comment) < 0) return -1; + if (tEncodeBinary(&encoder, pReq->comment, pReq->commentLen) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -542,6 +543,7 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfColumns) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfTags) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->commentLen) < 0) return -1; pReq->pColumns = taosArrayInit(pReq->numOfColumns, sizeof(SField)); pReq->pTags = taosArrayInit(pReq->numOfTags, sizeof(SField)); @@ -572,6 +574,12 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR } } + if (pReq->commentLen > 0) { + pReq->comment = malloc(pReq->commentLen); + if (pReq->comment == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->comment) < 0) return -1; + } + if (tDecodeCStrTo(&decoder, pReq->comment) < 0) return -1; tEndDecode(&decoder); @@ -669,6 +677,123 @@ void tFreeSMAltertbReq(SMAltertbReq *pReq) { pReq->pFields = NULL; } +int32_t tSerializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->stb) < 0) return -1; + if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; + if (tEncodeI8(&encoder, pReq->intervalUnit) < 0) return -1; + if (tEncodeI8(&encoder, pReq->slidingUnit) < 0) return -1; + if (tEncodeI8(&encoder, pReq->timezone) < 0) return -1; + if (tEncodeI32(&encoder, pReq->dstVgId) < 0) return -1; + if (tEncodeI64(&encoder, pReq->interval) < 0) return -1; + if (tEncodeI64(&encoder, pReq->offset) < 0) return -1; + if (tEncodeI64(&encoder, pReq->sliding) < 0) return -1; + if (tEncodeI32(&encoder, pReq->exprLen) < 0) return -1; + if (tEncodeI32(&encoder, pReq->tagsFilterLen) < 0) return -1; + if (tEncodeI32(&encoder, pReq->sqlLen) < 0) return -1; + if (tEncodeI32(&encoder, pReq->astLen) < 0) return -1; + if (pReq->exprLen > 0) { + if (tEncodeBinary(&encoder, pReq->expr, pReq->exprLen) < 0) return -1; + } + if (pReq->tagsFilterLen > 0) { + if (tEncodeBinary(&encoder, pReq->tagsFilter, pReq->tagsFilterLen) < 0) return -1; + } + if (pReq->sqlLen > 0) { + if (tEncodeBinary(&encoder, pReq->sql, pReq->sqlLen) < 0) return -1; + } + if (pReq->astLen > 0) { + if (tEncodeBinary(&encoder, pReq->ast, pReq->astLen) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->stb) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->intervalUnit) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->slidingUnit) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->timezone) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->dstVgId) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->interval) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->offset) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->sliding) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->exprLen) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->tagsFilterLen) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->sqlLen) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->astLen) < 0) return -1; + if (pReq->exprLen > 0) { + pReq->expr = malloc(pReq->exprLen); + if (pReq->expr == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->expr) < 0) return -1; + } + if (pReq->tagsFilterLen > 0) { + pReq->tagsFilter = malloc(pReq->tagsFilterLen); + if (pReq->tagsFilter == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->tagsFilter) < 0) return -1; + } + if (pReq->sqlLen > 0) { + pReq->sql = malloc(pReq->sqlLen); + if (pReq->sql == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1; + } + if (pReq->astLen > 0) { + pReq->ast = malloc(pReq->astLen); + if (pReq->ast == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1; + } + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; +} + +void tFreeSMCreateSmaReq(SMCreateSmaReq *pReq) { + tfree(pReq->expr); + tfree(pReq->tagsFilter); + tfree(pReq->sql); + tfree(pReq->ast); +} + +int32_t tSerializeSMDropSmaReq(void *buf, int32_t bufLen, SMDropSmaReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMDropSmaReq(void *buf, int32_t bufLen, SMDropSmaReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -2709,7 +2834,7 @@ int32_t tSerializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->code) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->code) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2736,13 +2861,13 @@ int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchR if (tStartEncode(&encoder) < 0) return -1; if (pRsp->rspList) { int32_t num = taosArrayGetSize(pRsp->rspList); - if (tEncodeI32(&encoder, num) < 0) return -1; + if (tEncodeI32(&encoder, num) < 0) return -1; for (int32_t i = 0; i < num; ++i) { SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i); - if (tEncodeI32(&encoder, rsp->code) < 0) return -1; + if (tEncodeI32(&encoder, rsp->code) < 0) return -1; } } else { - if (tEncodeI32(&encoder, 0) < 0) return -1; + if (tEncodeI32(&encoder, 0) < 0) return -1; } tEndEncode(&encoder); @@ -2752,7 +2877,7 @@ int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchR } int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { - SCoder decoder = {0}; + SCoder decoder = {0}; int32_t num = 0; tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); @@ -2775,7 +2900,6 @@ int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatc return 0; } - int32_t tSerializeSVCreateTSmaReq(void **buf, SVCreateTSmaReq *pReq) { int32_t tlen = 0; @@ -2824,7 +2948,8 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; if (tEncodeCStr(&encoder, pReq->outputTbName) < 0) return -1; if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1; + if (tEncodeI32(&encoder, sqlLen) < 0) return -1; + if (tEncodeI32(&encoder, astLen) < 0) return -1; if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1; if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1; @@ -2872,30 +2997,36 @@ void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) { } int32_t tEncodeSStreamTask(SCoder *pEncoder, const SStreamTask *pTask) { - if (tStartEncode(pEncoder) < 0) return -1; + /*if (tStartEncode(pEncoder) < 0) return -1;*/ if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->level) < 0) return -1; if (tEncodeI8(pEncoder, pTask->status) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->pipeSource) < 0) return -1; if (tEncodeI8(pEncoder, pTask->pipeSink) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->parallelizable) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->nextOpDst) < 0) return -1; // if (tEncodeI8(pEncoder, pTask->numOfRunners) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pTask->NextOpEp) < 0) return -1; if (tEncodeCStr(pEncoder, pTask->qmsg) < 0) return -1; - tEndEncode(pEncoder); + /*tEndEncode(pEncoder);*/ return pEncoder->pos; } int32_t tDecodeSStreamTask(SCoder *pDecoder, SStreamTask *pTask) { - if (tStartDecode(pDecoder) < 0) return -1; + /*if (tStartDecode(pDecoder) < 0) return -1;*/ if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->level) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->pipeSource) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->pipeSink) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->parallelizable) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->nextOpDst) < 0) return -1; // if (tDecodeI8(pDecoder, &pTask->numOfRunners) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pTask->NextOpEp) < 0) return -1; if (tDecodeCStrAlloc(pDecoder, &pTask->qmsg) < 0) return -1; - tEndDecode(pDecoder); + /*tEndDecode(pDecoder);*/ return 0; } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index db4bc49425..48a63799f5 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -78,7 +78,7 @@ static FORCE_INLINE void dataColSetNoneAt(SDataCol *pCol, int index, bool setBit setNull(POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * index), pCol->type, pCol->bytes); pCol->len += TYPE_BYTES[pCol->type]; } - if(setBitmap) { + if (setBitmap) { tdSetBitmapValType(pCol->pBitmap, index, TD_VTYPE_NONE); } } @@ -118,8 +118,8 @@ int trbWriteCol(SRowBuilder *pRB, void *pData, col_id_t cid) { #endif -STSRow* tdRowDup(STSRow *row) { - STSRow* trow = malloc(TD_ROW_LEN(row)); +STSRow *tdRowDup(STSRow *row) { + STSRow *trow = malloc(TD_ROW_LEN(row)); if (trow == NULL) return NULL; tdRowCpy(trow, row); @@ -176,7 +176,7 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols SDataCol *pDataCol = &(pCols->cols[0]); if (pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints); } while (dcol < pCols->numOfCols) { @@ -378,9 +378,7 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i } } - - -STSRow* mergeTwoRows(void *buffer, STSRow* row1, STSRow *row2, STSchema *pSchema1, STSchema *pSchema2) { +STSRow *mergeTwoRows(void *buffer, STSRow *row1, STSRow *row2, STSchema *pSchema1, STSchema *pSchema2) { #if 0 ASSERT(TD_ROW_KEY(row1) == TD_ROW_KEY(row2)); ASSERT(schemaVersion(pSchema1) == TD_ROW_SVER(row1)); @@ -473,6 +471,44 @@ STSRow* mergeTwoRows(void *buffer, STSRow* row1, STSRow *row2, STSchema *pSchema } taosArrayDestroy(stashRow); return buffer; - #endif +#endif return NULL; } + +SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { + SDataCols *pRet = tdNewDataCols(pDataCols->maxCols, pDataCols->maxPoints); + if (pRet == NULL) return NULL; + + pRet->numOfCols = pDataCols->numOfCols; + pRet->sversion = pDataCols->sversion; + if (keepData) pRet->numOfRows = pDataCols->numOfRows; + + for (int i = 0; i < pDataCols->numOfCols; i++) { + pRet->cols[i].type = pDataCols->cols[i].type; + pRet->cols[i].bitmap = pDataCols->cols[i].bitmap; + pRet->cols[i].colId = pDataCols->cols[i].colId; + pRet->cols[i].bytes = pDataCols->cols[i].bytes; + pRet->cols[i].offset = pDataCols->cols[i].offset; + + if (keepData) { + if (pDataCols->cols[i].len > 0) { + if (tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints) < 0) { + tdFreeDataCols(pRet); + return NULL; + } + pRet->cols[i].len = pDataCols->cols[i].len; + memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len); + if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) { + int dataOffSize = sizeof(VarDataOffsetT) * pDataCols->maxPoints; + memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize); + } + if (!TD_COL_ROWS_NORM(pRet->cols + i)) { + int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->maxPoints); + memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, nBitmapBytes); + } + } + } + } + + return pRet; +} \ No newline at end of file diff --git a/source/dnode/mgmt/mnode/src/mmMsg.c b/source/dnode/mgmt/mnode/src/mmMsg.c index 56a580ed93..6f719f3d8d 100644 --- a/source/dnode/mgmt/mnode/src/mmMsg.c +++ b/source/dnode/mgmt/mnode/src/mmMsg.c @@ -123,6 +123,8 @@ void mmInitMsgHandles(SMgmtWrapper *pWrapper) { dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, (NodeMsgFp)mmProcessWriteMsg, 0); dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, (NodeMsgFp)mmProcessWriteMsg, 0); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SMA, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SMA, (NodeMsgFp)mmProcessWriteMsg, 0); dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, (NodeMsgFp)mmProcessReadMsg, 0); dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, (NodeMsgFp)mmProcessReadMsg, 0); dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, (NodeMsgFp)mmProcessWriteMsg, 0); @@ -142,6 +144,8 @@ void mmInitMsgHandles(SMgmtWrapper *pWrapper) { dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, (NodeMsgFp)mmProcessWriteMsg, 0); dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, (NodeMsgFp)mmProcessWriteMsg, 0); dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, (NodeMsgFp)mmProcessReadMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); // Requests handled by VNODE dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); @@ -149,4 +153,6 @@ void mmInitMsgHandles(SMgmtWrapper *pWrapper) { dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); } diff --git a/source/dnode/mgmt/vnode/src/vmMsg.c b/source/dnode/mgmt/vnode/src/vmMsg.c index d4af82b382..a98dccbca3 100644 --- a/source/dnode/mgmt/vnode/src/vmMsg.c +++ b/source/dnode/mgmt/vnode/src/vmMsg.c @@ -268,6 +268,9 @@ void vmInitMsgHandles(SMgmtWrapper *pWrapper) { dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg, 0); dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, 0); dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, (NodeMsgFp)vmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, (NodeMsgFp)vmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, (NodeMsgFp)vmProcessWriteMsg, 0); dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg, 0); dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg, 0); dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, 0); @@ -277,6 +280,7 @@ void vmInitMsgHandles(SMgmtWrapper *pWrapper) { dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, 0); dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, 0); dndSetMsgHandle(pWrapper, TDMT_VND_TASK_EXEC, (NodeMsgFp)vmProcessFetchMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, (NodeMsgFp)vmProcessFetchMsg, 0); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, (NodeMsgFp)vmProcessMgmtMsg, 0); dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, (NodeMsgFp)vmProcessMgmtMsg, 0); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 909486aaac..d4f5fa1e3b 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -103,6 +103,8 @@ typedef enum { TRN_TYPE_CREATE_STB = 4001, TRN_TYPE_ALTER_STB = 4002, TRN_TYPE_DROP_STB = 4003, + TRN_TYPE_CREATE_SMA = 4004, + TRN_TYPE_DROP_SMA = 4005, TRN_TYPE_STB_SCOPE_END, } ETrnType; @@ -305,6 +307,31 @@ typedef struct { SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; } SVgObj; +typedef struct { + char name[TSDB_TABLE_FNAME_LEN]; + char stb[TSDB_TABLE_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + int64_t createdTime; + int64_t uid; + int64_t stbUid; + int64_t dbUid; + int8_t intervalUnit; + int8_t slidingUnit; + int8_t timezone; + int32_t dstVgId; // for stream + int64_t interval; + int64_t offset; + int64_t sliding; + int32_t exprLen; // strlen + 1 + int32_t tagsFilterLen; + int32_t sqlLen; + int32_t astLen; + char* expr; + char* tagsFilter; + char* sql; + char* ast; +} SSmaObj; + typedef struct { char name[TSDB_TABLE_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN]; @@ -316,10 +343,11 @@ typedef struct { int32_t nextColId; int32_t numOfColumns; int32_t numOfTags; + int32_t commentLen; SSchema* pColumns; SSchema* pTags; + char* comment; SRWLatch lock; - char comment[TSDB_STB_COMMENT_LEN]; } SStbObj; typedef struct { @@ -697,12 +725,12 @@ typedef struct { char* logicalPlan; char* physicalPlan; SArray* tasks; // SArray> + SArray* outputName; } SStreamObj; int32_t tEncodeSStreamObj(SCoder* pEncoder, const SStreamObj* pObj); int32_t tDecodeSStreamObj(SCoder* pDecoder, SStreamObj* pObj); - #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index d5bffada6a..20e85973be 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -122,9 +122,9 @@ typedef struct SMnode { SMsgCb msgCb; } SMnode; -void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); -uint64_t mndGenerateUid(char *name, int32_t len); -void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); +void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); +int64_t mndGenerateUid(char *name, int32_t len); +void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndSma.h b/source/dnode/mnode/impl/inc/mndSma.h new file mode 100644 index 0000000000..4a80f619d3 --- /dev/null +++ b/source/dnode/mnode/impl/inc/mndSma.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MND_SMA_H_ +#define _TD_MND_SMA_H_ + +#include "mndInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t mndInitSma(SMnode *pMnode); +void mndCleanupSma(SMnode *pMnode); +SSmaObj *mndAcquireSma(SMnode *pMnode, char *smaName); +void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_SMA_H_*/ diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index f81dead325..f0905f88d2 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -16,6 +16,7 @@ #include "mndDef.h" int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) { + int32_t outputNameSz = 0; if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->db) < 0) return -1; if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1; @@ -43,6 +44,15 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) { } else { tEncodeI32(pEncoder, 0); } + + if (pObj->outputName != NULL) { + outputNameSz = taosArrayGetSize(pObj->outputName); + } + if (tEncodeI32(pEncoder, outputNameSz) < 0) return -1; + for (int32_t i = 0; i < outputNameSz; i++) { + char *name = taosArrayGetP(pObj->outputName, i); + if (tEncodeCStr(pEncoder, name) < 0) return -1; + } return pEncoder->pos; } @@ -76,5 +86,16 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) { } else { pObj->tasks = NULL; } + int32_t outputNameSz; + if (tDecodeI32(pDecoder, &outputNameSz) < 0) return -1; + pObj->outputName = taosArrayInit(outputNameSz, sizeof(void *)); + if (pObj->outputName == NULL) { + return -1; + } + for (int32_t i = 0; i < outputNameSz; i++) { + char *name; + if (tDecodeCStrAlloc(pDecoder, &name) < 0) return -1; + taosArrayPush(pObj->outputName, &name); + } return 0; } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index b95574ea41..8ccfd6be5d 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -32,20 +32,23 @@ #include "tname.h" #include "tuuid.h" -int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet* pEpSet, tmsg_t type) { +extern bool tsStreamSchedV; + +int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet* pEpSet, tmsg_t type, int32_t nodeId) { SCoder encoder; tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); tEncodeSStreamTask(&encoder, pTask); - int32_t tlen = sizeof(SMsgHead) + encoder.pos; + int32_t size = encoder.pos; + int32_t tlen = sizeof(SMsgHead) + size; tCoderClear(&encoder); void* buf = malloc(tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - ((SMsgHead*)buf)->streamTaskId = pTask->taskId; + ((SMsgHead*)buf)->streamTaskId = htonl(nodeId); void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, tlen, TD_ENCODER); + tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, size, TD_ENCODER); tEncodeSStreamTask(&encoder, pTask); tCoderClear(&encoder); @@ -70,7 +73,7 @@ int32_t mndAssignTaskToVg(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SS terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } - mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet, TDMT_VND_TASK_DEPLOY); + mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet, TDMT_VND_TASK_DEPLOY, pVgroup->vgId); return 0; } @@ -90,7 +93,7 @@ int32_t mndAssignTaskToSnode(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } - mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet, TDMT_SND_TASK_DEPLOY); + mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet, TDMT_SND_TASK_DEPLOY, 0); return 0; } @@ -106,6 +109,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { int32_t totLevel = LIST_LENGTH(pPlan->pSubplans); pStream->tasks = taosArrayInit(totLevel, sizeof(SArray)); + int32_t lastUsedVgId = 0; for (int32_t level = 0; level < totLevel; level++) { SArray* taskOneLevel = taosArrayInit(0, sizeof(SStreamTask)); @@ -113,9 +117,9 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { int32_t opNum = LIST_LENGTH(inner->pNodeList); ASSERT(opNum == 1); - SSubplan* plan = nodesListGetNode(inner->pNodeList, level); + SSubplan* plan = nodesListGetNode(inner->pNodeList, 0); if (level == 0) { - ASSERT(plan->type == SUBPLAN_TYPE_SCAN); + ASSERT(plan->subplanType == SUBPLAN_TYPE_SCAN); void* pIter = NULL; while (1) { pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); @@ -125,11 +129,14 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { continue; } + lastUsedVgId = pVgroup->vgId; pStream->vgNum++; // send to vnode SStreamTask* pTask = streamTaskNew(pStream->uid, level); + pTask->pipeSource = 1; pTask->pipeSink = level == totLevel - 1 ? 1 : 0; + pTask->parallelizable = 1; // TODO: set to if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); @@ -140,19 +147,35 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { } } else { SStreamTask* pTask = streamTaskNew(pStream->uid, level); + pTask->pipeSource = 0; pTask->pipeSink = level == totLevel - 1 ? 1 : 0; - SSnodeObj* pSnode = mndSchedFetchSnode(pMnode); - if (pSnode != NULL) { - if (mndAssignTaskToSnode(pMnode, pTrans, pTask, plan, pSnode) < 0) { - sdbRelease(pSdb, pSnode); + pTask->parallelizable = plan->subplanType == SUBPLAN_TYPE_SCAN; + pTask->nextOpDst = STREAM_NEXT_OP_DST__VND; + + if (tsStreamSchedV) { + ASSERT(lastUsedVgId != 0); + SVgObj* pVg = mndAcquireVgroup(pMnode, lastUsedVgId); + if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVg) < 0) { + sdbRelease(pSdb, pVg); qDestroyQueryPlan(pPlan); return -1; } - sdbRelease(pMnode->pSdb, pSnode); + sdbRelease(pSdb, pVg); } else { - // TODO: assign to one vg - ASSERT(0); + SSnodeObj* pSnode = mndSchedFetchSnode(pMnode); + if (pSnode != NULL) { + if (mndAssignTaskToSnode(pMnode, pTrans, pTask, plan, pSnode) < 0) { + sdbRelease(pSdb, pSnode); + qDestroyQueryPlan(pPlan); + return -1; + } + sdbRelease(pMnode->pSdb, pSnode); + } else { + // TODO: assign to one vg + ASSERT(0); + } } + taosArrayPush(taskOneLevel, pTask); } taosArrayPush(pStream->tasks, taskOneLevel); diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 8536ee8981..eb12347e64 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -427,6 +427,8 @@ char *mndShowStr(int32_t showType) { return "show topics"; case TSDB_MGMT_TABLE_FUNC: return "show functions"; + case TSDB_MGMT_TABLE_INDEX: + return "show indexes"; default: return "undefined"; } diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c new file mode 100644 index 0000000000..2bd13f395f --- /dev/null +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -0,0 +1,762 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndSma.h" +#include "mndAuth.h" +#include "mndDb.h" +#include "mndDnode.h" +#include "mndInfoSchema.h" +#include "mndMnode.h" +#include "mndShow.h" +#include "mndStb.c" +#include "mndTrans.h" +#include "mndUser.h" +#include "mndVgroup.h" +#include "tname.h" + +#define TSDB_SMA_VER_NUMBER 1 +#define TSDB_SMA_RESERVE_SIZE 64 + +static SSdbRaw *mndSmaActionEncode(SSmaObj *pSma); +static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw); +static int32_t mndSmaActionInsert(SSdb *pSdb, SSmaObj *pSma); +static int32_t mndSmaActionDelete(SSdb *pSdb, SSmaObj *pSpSmatb); +static int32_t mndSmaActionUpdate(SSdb *pSdb, SSmaObj *pOld, SSmaObj *pNew); +static int32_t mndProcessMCreateSmaReq(SNodeMsg *pReq); +static int32_t mndProcessMDropSmaReq(SNodeMsg *pReq); +static int32_t mndProcessVCreateSmaRsp(SNodeMsg *pRsp); +static int32_t mndProcessVDropSmaRsp(SNodeMsg *pRsp); +static int32_t mndGetSmaMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextSma(SMnode *pMnode, void *pIter); + +int32_t mndInitSma(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_SMA, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndSmaActionEncode, + .decodeFp = (SdbDecodeFp)mndSmaActionDecode, + .insertFp = (SdbInsertFp)mndSmaActionInsert, + .updateFp = (SdbUpdateFp)mndSmaActionUpdate, + .deleteFp = (SdbDeleteFp)mndSmaActionDelete}; + + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_SMA, mndProcessMCreateSmaReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_SMA, mndProcessMDropSmaReq); + mndSetMsgHandle(pMnode, TDMT_VND_CREATE_SMA_RSP, mndProcessVCreateSmaRsp); + mndSetMsgHandle(pMnode, TDMT_VND_DROP_SMA_RSP, mndProcessVDropSmaRsp); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndGetSmaMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndRetrieveSma); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndCancelGetNextSma); + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupSma(SMnode *pMnode) {} + +static SSdbRaw *mndSmaActionEncode(SSmaObj *pSma) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + + int32_t size = sizeof(SSmaObj) + pSma->exprLen + pSma->tagsFilterLen + TSDB_SMA_RESERVE_SIZE; + SSdbRaw *pRaw = sdbAllocRaw(SDB_SMA, TSDB_SMA_VER_NUMBER, size); + if (pRaw == NULL) goto _OVER; + + int32_t dataPos = 0; + + SDB_SET_BINARY(pRaw, dataPos, pSma->name, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pSma->stb, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pSma->db, TSDB_DB_FNAME_LEN, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->createdTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->uid, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->stbUid, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->dbUid, _OVER) + SDB_SET_INT8(pRaw, dataPos, pSma->intervalUnit, _OVER) + SDB_SET_INT8(pRaw, dataPos, pSma->slidingUnit, _OVER) + SDB_SET_INT8(pRaw, dataPos, pSma->timezone, _OVER) + SDB_SET_INT32(pRaw, dataPos, pSma->dstVgId, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->interval, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->offset, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->sliding, _OVER) + SDB_SET_INT32(pRaw, dataPos, pSma->exprLen, _OVER) + SDB_SET_INT32(pRaw, dataPos, pSma->tagsFilterLen, _OVER) + SDB_SET_INT32(pRaw, dataPos, pSma->sqlLen, _OVER) + SDB_SET_INT32(pRaw, dataPos, pSma->astLen, _OVER) + if (pSma->exprLen > 0) { + SDB_SET_BINARY(pRaw, dataPos, pSma->expr, pSma->exprLen, _OVER) + } + if (pSma->tagsFilterLen > 0) { + SDB_SET_BINARY(pRaw, dataPos, pSma->tagsFilter, pSma->tagsFilterLen, _OVER) + } + if (pSma->sqlLen > 0) { + SDB_SET_BINARY(pRaw, dataPos, pSma->sql, pSma->sqlLen, _OVER) + } + if (pSma->astLen > 0) { + SDB_SET_BINARY(pRaw, dataPos, pSma->ast, pSma->astLen, _OVER) + } + + SDB_SET_RESERVE(pRaw, dataPos, TSDB_SMA_RESERVE_SIZE, _OVER) + SDB_SET_DATALEN(pRaw, dataPos, _OVER) + terrno = 0; + +_OVER: + if (terrno != 0) { + mError("sma:%s, failed to encode to raw:%p since %s", pSma->name, pRaw, terrstr()); + sdbFreeRaw(pRaw); + return NULL; + } + + mTrace("sma:%s, encode to raw:%p, row:%p", pSma->name, pRaw, pSma); + return pRaw; +} + +static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; + + if (sver != TSDB_SMA_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + goto _OVER; + } + + SSdbRow *pRow = sdbAllocRow(sizeof(SSmaObj)); + if (pRow == NULL) goto _OVER; + + SSmaObj *pSma = sdbGetRowObj(pRow); + if (pSma == NULL) goto _OVER; + + int32_t dataPos = 0; + + SDB_GET_BINARY(pRaw, dataPos, pSma->name, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pSma->stb, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pSma->db, TSDB_DB_FNAME_LEN, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->createdTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->uid, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->stbUid, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->dbUid, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pSma->intervalUnit, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pSma->slidingUnit, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pSma->timezone, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pSma->dstVgId, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->interval, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->offset, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->sliding, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pSma->exprLen, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pSma->tagsFilterLen, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pSma->sqlLen, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pSma->astLen, _OVER) + + if (pSma->exprLen > 0) { + pSma->expr = calloc(pSma->exprLen, 1); + if (pSma->expr == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, pSma->expr, pSma->exprLen, _OVER) + } + + if (pSma->tagsFilterLen > 0) { + pSma->tagsFilter = calloc(pSma->tagsFilterLen, 1); + if (pSma->tagsFilter == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, pSma->tagsFilter, pSma->tagsFilterLen, _OVER) + } + + if (pSma->sqlLen > 0) { + pSma->sql = calloc(pSma->sqlLen, 1); + if (pSma->sql == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, pSma->sql, pSma->sqlLen, _OVER) + } + + if (pSma->astLen > 0) { + pSma->ast = calloc(pSma->astLen, 1); + if (pSma->ast == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, pSma->ast, pSma->astLen, _OVER) + } + + SDB_GET_RESERVE(pRaw, dataPos, TSDB_SMA_RESERVE_SIZE, _OVER) + terrno = 0; + +_OVER: + if (terrno != 0) { + mError("sma:%s, failed to decode from raw:%p since %s", pSma->name, pRaw, terrstr()); + tfree(pSma->expr); + tfree(pSma->tagsFilter); + tfree(pRow); + return NULL; + } + + mTrace("sma:%s, decode from raw:%p, row:%p", pSma->name, pRaw, pSma); + return pRow; +} + +static int32_t mndSmaActionInsert(SSdb *pSdb, SSmaObj *pSma) { + mTrace("sma:%s, perform insert action, row:%p", pSma->name, pSma); + return 0; +} + +static int32_t mndSmaActionDelete(SSdb *pSdb, SSmaObj *pSma) { + mTrace("sma:%s, perform delete action, row:%p", pSma->name, pSma); + tfree(pSma->tagsFilter); + tfree(pSma->expr); + return 0; +} + +static int32_t mndSmaActionUpdate(SSdb *pSdb, SSmaObj *pOld, SSmaObj *pNew) { + mTrace("sma:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); + return 0; +} + +SSmaObj *mndAcquireSma(SMnode *pMnode, char *smaName) { + SSdb *pSdb = pMnode->pSdb; + SSmaObj *pSma = sdbAcquire(pSdb, SDB_SMA, smaName); + if (pSma == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { + terrno = TSDB_CODE_MND_SMA_NOT_EXIST; + } + return pSma; +} + +void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pSma); +} + +SDbObj *mndAcquireDbBySma(SMnode *pMnode, const char *smaName) { + SName name = {0}; + tNameFromString(&name, smaName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + char db[TSDB_TABLE_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, db); + + return mndAcquireDb(pMnode, db); +} + +static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, int32_t *pContLen) { + SName name = {0}; + tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + SVCreateTSmaReq req = {0}; + req.tSma.version = 0; + req.tSma.intervalUnit = pSma->intervalUnit; + req.tSma.slidingUnit = pSma->slidingUnit; + req.tSma.timezoneInt = pSma->timezone; + tstrncpy(req.tSma.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN); + req.tSma.exprLen = pSma->exprLen; + req.tSma.tagsFilterLen = pSma->tagsFilterLen; + req.tSma.indexUid = pSma->uid; + req.tSma.tableUid = pSma->stbUid; + req.tSma.interval = pSma->interval; + req.tSma.offset = pSma->offset; + req.tSma.sliding = pSma->sliding; + req.tSma.expr = pSma->expr; + req.tSma.tagsFilter = pSma->tagsFilter; + + int32_t contLen = tSerializeSVCreateTSmaReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead *pHead = malloc(contLen); + if (pHead == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + + void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); + tSerializeSVCreateTSmaReq(&pBuf, &req); + + *pContLen = contLen; + return pHead; +} + +static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, int32_t *pContLen) { + SName name = {0}; + tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + SVDropTSmaReq req = {0}; + req.ver = 0; + req.indexUid = pSma->uid; + tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN); + + int32_t contLen = tSerializeSVDropTSmaReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead *pHead = malloc(contLen); + if (pHead == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + + void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); + tDeserializeSVDropTSmaReq(&pBuf, &req); + + *pContLen = contLen; + return pHead; +} + +static int32_t mndSetCreateSmaRedoLogs(SMnode *pMnode, STrans *pTrans, SSmaObj *pSma) { + SSdbRaw *pRedoRaw = mndSmaActionEncode(pSma); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; + + return 0; +} + +static int32_t mndSetCreateSmaCommitLogs(SMnode *pMnode, STrans *pTrans, SSmaObj *pSma) { + SSdbRaw *pCommitRaw = mndSmaActionEncode(pSma); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +static int32_t mndSetCreateSmaRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SSmaObj *pSma) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + int32_t contLen; + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pDb->uid) { + sdbRelease(pSdb, pVgroup); + continue; + } + + void *pReq = mndBuildVCreateSmaReq(pMnode, pVgroup, pSma, &contLen); + if (pReq == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_CREATE_SMA; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pReq); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + +static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCreate, SDbObj *pDb, SStbObj *pStb) { + SSmaObj smaObj = {0}; + memcpy(smaObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); + memcpy(smaObj.stb, pStb->name, TSDB_TABLE_FNAME_LEN); + memcpy(smaObj.db, pDb->name, TSDB_DB_FNAME_LEN); + smaObj.createdTime = taosGetTimestampMs(); + smaObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); + smaObj.stbUid = pStb->uid; + smaObj.dbUid = pStb->dbUid; + smaObj.intervalUnit = pCreate->intervalUnit; + smaObj.slidingUnit = pCreate->slidingUnit; + smaObj.timezone = pCreate->timezone; + smaObj.dstVgId = pCreate->dstVgId; + smaObj.interval = pCreate->interval; + smaObj.offset = pCreate->offset; + smaObj.sliding = pCreate->sliding; + smaObj.exprLen = pCreate->exprLen; + smaObj.tagsFilterLen = pCreate->tagsFilterLen; + smaObj.sqlLen = pCreate->sqlLen; + smaObj.astLen = pCreate->astLen; + + if (smaObj.exprLen > 0) { + smaObj.expr = malloc(smaObj.exprLen); + if (smaObj.expr == NULL) goto _OVER; + memcpy(smaObj.expr, pCreate->expr, smaObj.exprLen); + } + + if (smaObj.tagsFilterLen > 0) { + smaObj.tagsFilter = malloc(smaObj.tagsFilterLen); + if (smaObj.tagsFilter == NULL) goto _OVER; + memcpy(smaObj.tagsFilter, pCreate->tagsFilter, smaObj.tagsFilterLen); + } + + if (smaObj.sqlLen > 0) { + smaObj.sql = malloc(smaObj.sqlLen); + if (smaObj.sql == NULL) goto _OVER; + memcpy(smaObj.sql, pCreate->sql, smaObj.sqlLen); + } + + if (smaObj.astLen > 0) { + smaObj.ast = malloc(smaObj.astLen); + if (smaObj.ast == NULL) goto _OVER; + memcpy(smaObj.ast, pCreate->ast, smaObj.astLen); + } + + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_SMA, &pReq->rpcMsg); + if (pTrans == NULL) goto _OVER; + + mDebug("trans:%d, used to create sma:%s", pTrans->id, pCreate->name); + mndTransSetDbInfo(pTrans, pDb); + + if (mndSetCreateSmaRedoLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER; + if (mndSetCreateSmaCommitLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER; + if (mndSetCreateSmaRedoActions(pMnode, pTrans, pDb, &smaObj) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + + code = 0; + +_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndCheckCreateSmaReq(SMCreateSmaReq *pCreate) { + terrno = TSDB_CODE_MND_INVALID_SMA_OPTION; + if (pCreate->name[0] == 0) return -1; + if (pCreate->stb[0] == 0) return -1; + if (pCreate->igExists < 0 || pCreate->igExists > 1) return -1; + if (pCreate->intervalUnit < 0) return -1; + if (pCreate->slidingUnit < 0) return -1; + if (pCreate->timezone < 0) return -1; + if (pCreate->dstVgId < 0) return -1; + if (pCreate->interval < 0) return -1; + if (pCreate->offset < 0) return -1; + if (pCreate->sliding < 0) return -1; + if (pCreate->exprLen < 0) return -1; + if (pCreate->tagsFilterLen < 0) return -1; + if (pCreate->sqlLen < 0) return -1; + if (pCreate->astLen < 0) return -1; + if (pCreate->exprLen != 0 && strlen(pCreate->expr) + 1 != pCreate->exprLen) return -1; + if (pCreate->tagsFilterLen != 0 && strlen(pCreate->tagsFilter) + 1 != pCreate->tagsFilterLen) return -1; + if (pCreate->sqlLen != 0 && strlen(pCreate->sql) + 1 != pCreate->sqlLen) return -1; + if (pCreate->astLen != 0 && strlen(pCreate->ast) + 1 != pCreate->astLen) return -1; + + SName smaName = {0}; + if (tNameFromString(&smaName, pCreate->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE) < 0) return -1; + if (*(char *)tNameGetTableName(&smaName) == 0) return -1; + + terrno = 0; + return 0; +} + +static int32_t mndProcessMCreateSmaReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; + int32_t code = -1; + SStbObj *pStb = NULL; + SSmaObj *pSma = NULL; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SMCreateSmaReq createReq = {0}; + + if (tDeserializeSMCreateSmaReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + mDebug("sma:%s, start to create", createReq.name); + if (mndCheckCreateSmaReq(&createReq) != 0) { + goto _OVER; + } + + pStb = mndAcquireStb(pMnode, createReq.stb); + if (pStb == NULL) { + mError("sma:%s, failed to create since stb:%s not exist", createReq.name, createReq.stb); + goto _OVER; + } + + pSma = mndAcquireSma(pMnode, createReq.name); + if (pSma != NULL) { + if (createReq.igExists) { + mDebug("sma:%s, already exist in sma:%s, ignore exist is set", createReq.name, pSma->name); + code = 0; + goto _OVER; + } else { + terrno = TSDB_CODE_MND_SMA_ALREADY_EXIST; + goto _OVER; + } + } + + pDb = mndAcquireDbBySma(pMnode, createReq.name); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + goto _OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto _OVER; + } + + if (mndCheckWriteAuth(pUser, pDb) != 0) { + goto _OVER; + } + + code = mndCreateSma(pMnode, pReq, &createReq, pDb, pStb); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("sma:%s, failed to create since %s", createReq.name, terrstr()); + } + + mndReleaseStb(pMnode, pStb); + mndReleaseSma(pMnode, pSma); + mndReleaseDb(pMnode, pDb); + mndReleaseUser(pMnode, pUser); + tFreeSMCreateSmaReq(&createReq); + + return code; +} + +static int32_t mndProcessVCreateSmaRsp(SNodeMsg *pRsp) { + mndTransProcessRsp(pRsp); + return 0; +} + +static int32_t mndSetDropSmaRedoLogs(SMnode *pMnode, STrans *pTrans, SSmaObj *pSma) { + SSdbRaw *pRedoRaw = mndSmaActionEncode(pSma); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropSmaCommitLogs(SMnode *pMnode, STrans *pTrans, SSmaObj *pSma) { + SSdbRaw *pCommitRaw = mndSmaActionEncode(pSma); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropSmaRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SSmaObj *pSma) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + int32_t contLen; + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pDb->uid) { + sdbRelease(pSdb, pVgroup); + continue; + } + + int32_t contLen = 0; + void *pReq = mndBuildVDropSmaReq(pMnode, pVgroup, pSma, &contLen); + if (pReq == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_DROP_SMA; + action.acceptableCode = TSDB_CODE_VND_SMA_NOT_EXIST; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pReq); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + +static int32_t mndDropSma(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pDb, SSmaObj *pSma) { + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_SMA, &pReq->rpcMsg); + if (pTrans == NULL) goto _OVER; + + mDebug("trans:%d, used to drop sma:%s", pTrans->id, pSma->name); + mndTransSetDbInfo(pTrans, pDb); + + if (mndSetDropSmaRedoLogs(pMnode, pTrans, pSma) != 0) goto _OVER; + if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER; + if (mndSetDropSmaRedoActions(pMnode, pTrans, pDb, pSma) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + + code = 0; + +_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessMDropSmaReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; + int32_t code = -1; + SUserObj *pUser = NULL; + SDbObj *pDb = NULL; + SSmaObj *pSma = NULL; + SMDropSmaReq dropReq = {0}; + + if (tDeserializeSMDropSmaReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + mDebug("sma:%s, start to drop", dropReq.name); + + pSma = mndAcquireSma(pMnode, dropReq.name); + if (pSma == NULL) { + if (dropReq.igNotExists) { + mDebug("sma:%s, not exist, ignore not exist is set", dropReq.name); + code = 0; + goto _OVER; + } else { + terrno = TSDB_CODE_MND_SMA_NOT_EXIST; + goto _OVER; + } + } + + pDb = mndAcquireDbBySma(pMnode, dropReq.name); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + goto _OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto _OVER; + } + + if (mndCheckWriteAuth(pUser, pDb) != 0) { + goto _OVER; + } + + code = mndDropSma(pMnode, pReq, pDb, pSma); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("sma:%s, failed to drop since %s", dropReq.name, terrstr()); + } + + mndReleaseDb(pMnode, pDb); + mndReleaseSma(pMnode, pSma); + mndReleaseUser(pMnode, pUser); + + return code; +} + +static int32_t mndProcessVDropSmaRsp(SNodeMsg *pRsp) { + mndTransProcessRsp(pRsp); + return 0; +} + +static int32_t mndGetSmaMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchemas; + + pShow->bytes[cols] = TSDB_INDEX_NAME_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "name"); + pSchema[cols].bytes = pShow->bytes[cols]; + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = pShow->bytes[cols]; + cols++; + + pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "stb"); + pSchema[cols].bytes = pShow->bytes[cols]; + cols++; + + pMeta->numOfColumns = cols; + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_SMA); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbName, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SSmaObj *pSma = NULL; + int32_t cols = 0; + char *pWrite; + char prefix[TSDB_DB_FNAME_LEN] = {0}; + + SDbObj *pDb = mndAcquireDb(pMnode, pShow->db); + if (pDb == NULL) return 0; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_SMA, pShow->pIter, (void **)&pSma); + if (pShow->pIter == NULL) break; + + if (pSma->dbUid != pDb->uid) { + sdbRelease(pSdb, pSma); + continue; + } + + cols = 0; + + SName smaName = {0}; + tNameFromString(&smaName, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, (char *)tNameGetTableName(&smaName)); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pSma->createdTime; + cols++; + + SName stbName = {0}; + tNameFromString(&stbName, pSma->stb, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, (char *)tNameGetTableName(&stbName)); + cols++; + + numOfRows++; + sdbRelease(pSdb, pSma); + } + + mndReleaseDb(pMnode, pDb); + pShow->numOfReads += numOfRows; + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + return numOfRows; +} + +static void mndCancelGetNextSma(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 7b0de9d8bd..e8d6157661 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -13,20 +13,19 @@ * along with this program. If not, see . */ -#define _DEFAULT_SOURCE #include "mndStb.h" #include "mndAuth.h" #include "mndDb.h" #include "mndDnode.h" +#include "mndInfoSchema.h" #include "mndMnode.h" #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" -#include "mndInfoSchema.h" #include "tname.h" -#define TSDB_STB_VER_NUMBER 1 +#define TSDB_STB_VER_NUMBER 1 #define TSDB_STB_RESERVE_SIZE 64 static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw); @@ -88,6 +87,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { SDB_SET_INT32(pRaw, dataPos, pStb->nextColId, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags, STB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->commentLen, STB_ENCODE_OVER) for (int32_t i = 0; i < pStb->numOfColumns; ++i) { SSchema *pSchema = &pStb->pColumns[i]; @@ -105,7 +105,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_ENCODE_OVER) } - SDB_SET_BINARY(pRaw, dataPos, pStb->comment, TSDB_STB_COMMENT_LEN, STB_ENCODE_OVER) + SDB_SET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, STB_ENCODE_OVER) SDB_SET_RESERVE(pRaw, dataPos, TSDB_STB_RESERVE_SIZE, STB_ENCODE_OVER) SDB_SET_DATALEN(pRaw, dataPos, STB_ENCODE_OVER) @@ -150,6 +150,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pStb->nextColId, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->numOfColumns, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->numOfTags, STB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->commentLen, STB_DECODE_OVER) pStb->pColumns = calloc(pStb->numOfColumns, sizeof(SSchema)); pStb->pTags = calloc(pStb->numOfTags, sizeof(SSchema)); @@ -173,7 +174,11 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_DECODE_OVER) } - SDB_GET_BINARY(pRaw, dataPos, pStb->comment, TSDB_STB_COMMENT_LEN, STB_DECODE_OVER) + if (pStb->commentLen > 0) { + pStb->comment = calloc(pStb->commentLen, 1); + if (pStb->comment == NULL) goto STB_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, STB_DECODE_OVER) + } SDB_GET_RESERVE(pRaw, dataPos, TSDB_STB_RESERVE_SIZE, STB_DECODE_OVER) terrno = 0; @@ -183,6 +188,7 @@ STB_DECODE_OVER: mError("stb:%s, failed to decode from raw:%p since %s", pStb->name, pRaw, terrstr()); tfree(pStb->pColumns); tfree(pStb->pTags); + tfree(pStb->comment); tfree(pRow); return NULL; } @@ -200,6 +206,7 @@ static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) { mTrace("stb:%s, perform delete action, row:%p", pStb->name, pStb); tfree(pStb->pColumns); tfree(pStb->pTags); + tfree(pStb->comment); return 0; } @@ -502,6 +509,13 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre stbObj.nextColId = 1; stbObj.numOfColumns = pCreate->numOfColumns; stbObj.numOfTags = pCreate->numOfTags; + stbObj.commentLen = pCreate->commentLen; + stbObj.comment = calloc(stbObj.commentLen, 1); + if (stbObj.comment == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + memcpy(stbObj.comment, pCreate->comment, stbObj.commentLen); stbObj.pColumns = malloc(stbObj.numOfColumns * sizeof(SSchema)); stbObj.pTags = malloc(stbObj.numOfTags * sizeof(SSchema)); @@ -1162,7 +1176,7 @@ static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * static int32_t mndDropStb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pDb, SStbObj *pStb) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK,TRN_TYPE_DROP_STB, &pReq->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_STB, &pReq->rpcMsg); if (pTrans == NULL) goto DROP_STB_OVER; mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name); @@ -1568,7 +1582,11 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32 cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_TO_VARSTR(pWrite, pStb->comment); + if (pStb->commentLen != 0) { + STR_TO_VARSTR(pWrite, pStb->comment); + } else { + STR_TO_VARSTR(pWrite, ""); + } cols++; numOfRows++; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 99aded0292..fb6906b5c9 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -21,6 +21,7 @@ #include "mndScheduler.h" #include "mndShow.h" #include "mndStb.h" +#include "mndTopic.h" #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" @@ -33,6 +34,7 @@ static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream); static int32_t mndProcessCreateStreamReq(SNodeMsg *pReq); +static int32_t mndProcessTaskDeployInternalRsp(SNodeMsg *pRsp); /*static int32_t mndProcessDropStreamReq(SNodeMsg *pReq);*/ /*static int32_t mndProcessDropStreamInRsp(SNodeMsg *pRsp);*/ static int32_t mndProcessStreamMetaReq(SNodeMsg *pReq); @@ -50,6 +52,8 @@ int32_t mndInitStream(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndStreamActionDelete}; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STREAM, mndProcessCreateStreamReq); + mndSetMsgHandle(pMnode, TDMT_VND_TASK_DEPLOY_RSP, mndProcessTaskDeployInternalRsp); + mndSetMsgHandle(pMnode, TDMT_SND_TASK_DEPLOY_RSP, mndProcessTaskDeployInternalRsp); /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq);*/ /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM_RSP, mndProcessDropStreamInRsp);*/ @@ -68,7 +72,7 @@ SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) { SCoder encoder; tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); - if (tEncodeSStreamObj(NULL, pStream) < 0) { + if (tEncodeSStreamObj(&encoder, pStream) < 0) { tCoderClear(&encoder); goto STREAM_ENCODE_OVER; } @@ -83,7 +87,7 @@ SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) { if (buf == NULL) goto STREAM_ENCODE_OVER; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, tlen, TD_ENCODER); - if (tEncodeSStreamObj(NULL, pStream) < 0) { + if (tEncodeSStreamObj(&encoder, pStream) < 0) { tCoderClear(&encoder); goto STREAM_ENCODE_OVER; } @@ -135,7 +139,7 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, buf, tlen, STREAM_DECODE_OVER); SCoder decoder; - tCoderInit(&decoder, TD_LITTLE_ENDIAN, NULL, 0, TD_DECODER); + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, tlen + 1, TD_DECODER); if (tDecodeSStreamObj(&decoder, pStream) < 0) { goto STREAM_DECODE_OVER; } @@ -191,6 +195,11 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) { sdbRelease(pSdb, pStream); } +static int32_t mndProcessTaskDeployInternalRsp(SNodeMsg *pRsp) { + mndTransProcessRsp(pRsp); + return 0; +} + static SDbObj *mndAcquireDbByStream(SMnode *pMnode, char *streamName) { SName name = {0}; tNameFromString(&name, streamName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); @@ -209,6 +218,55 @@ static int32_t mndCheckCreateStreamReq(SCMCreateStreamReq *pCreate) { return 0; } +static SArray *mndExtractNamesFromAst(const SNode *pAst) { + if (pAst->type != QUERY_NODE_SELECT_STMT) return NULL; + + SArray *names = taosArrayInit(0, sizeof(void *)); + if (names == NULL) { + return NULL; + } + SSelectStmt *pSelect = (SSelectStmt *)pAst; + SNodeList *pNodes = pSelect->pProjectionList; + SListCell *pCell = pNodes->pHead; + while (pCell != NULL) { + if (pCell->pNode->type != QUERY_NODE_FUNCTION) { + continue; + } + SFunctionNode *pFunction = (SFunctionNode *)pCell->pNode; + char *name = strdup(pFunction->node.aliasName); + taosArrayPush(names, &name); + pCell = pCell->pNext; + } + return names; +} + +static int32_t mndStreamGetPlanString(const SCMCreateStreamReq *pCreate, char **pStr) { + if (NULL == pCreate->ast) { + return TSDB_CODE_SUCCESS; + } + + SNode *pAst = NULL; + int32_t code = nodesStringToNode(pCreate->ast, &pAst); + + SQueryPlan *pPlan = NULL; + if (TSDB_CODE_SUCCESS == code) { + SPlanContext cxt = { + .pAstRoot = pAst, + .topicQuery = false, + .streamQuery = true, + }; + code = qCreateQueryPlan(&cxt, &pPlan, NULL); + } + + if (TSDB_CODE_SUCCESS == code) { + code = nodesNodeToString(pPlan, false, pStr, NULL); + } + nodesDestroyNode(pAst); + nodesDestroyNode(pPlan); + terrno = code; + return code; +} + static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) { mDebug("stream:%s to create", pCreate->name); SStreamObj streamObj = {0}; @@ -220,10 +278,28 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe streamObj.dbUid = pDb->uid; streamObj.version = 1; streamObj.sql = pCreate->sql; - streamObj.physicalPlan = ""; - streamObj.logicalPlan = ""; + /*streamObj.physicalPlan = "";*/ + streamObj.logicalPlan = "not implemented"; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg); + SNode *pAst = NULL; + if (nodesStringToNode(pCreate->ast, &pAst) < 0) { + return -1; + } + SArray *names = mndExtractNamesFromAst(pAst); + printf("|"); + for (int i = 0; i < taosArrayGetSize(names); i++) { + printf(" %15s |", (char *)taosArrayGetP(names, i)); + } + printf("\n=======================================================\n"); + + streamObj.outputName = names; + + if (TSDB_CODE_SUCCESS != mndStreamGetPlanString(pCreate, &streamObj.physicalPlan)) { + mError("topic:%s, failed to get plan since %s", pCreate->name, terrstr()); + return -1; + } + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg); if (pTrans == NULL) { mError("stream:%s, failed to create since %s", pCreate->name, terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index bd0fd0e612..95dce5d8f1 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -236,7 +236,7 @@ static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) { return 0; } -static int32_t mndGetPlanString(SCMCreateTopicReq *pCreate, char **pStr) { +static int32_t mndGetPlanString(const SCMCreateTopicReq *pCreate, char **pStr) { if (NULL == pCreate->ast) { return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index e2a6e49b56..aa39c740ad 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -406,6 +406,10 @@ static const char *mndTransType(ETrnType type) { return "alter-stb"; case TRN_TYPE_DROP_STB: return "drop-stb"; + case TRN_TYPE_CREATE_SMA: + return "create-sma"; + case TRN_TYPE_DROP_SMA: + return "drop-sma"; default: return "invalid"; } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 6400bf69f1..76687fc5cc 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -28,6 +28,7 @@ #include "mndProfile.h" #include "mndQnode.h" #include "mndShow.h" +#include "mndSma.h" #include "mndSnode.h" #include "mndStb.h" #include "mndStream.h" @@ -204,6 +205,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-offset", mndInitOffset, mndCleanupOffset) != 0) return -1; if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-stb", mndInitSma, mndCleanupSma) != 0) return -1; if (mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1; @@ -409,15 +411,15 @@ void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) { } // Note: uid 0 is reserved -uint64_t mndGenerateUid(char *name, int32_t len) { +int64_t mndGenerateUid(char *name, int32_t len) { int32_t hashval = MurmurHash3_32(name, len); do { int64_t us = taosGetTimestampUs(); - uint64_t x = (us & 0x000000FFFFFFFFFF) << 24; - uint64_t uuid = x + ((hashval & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); + int64_t x = (us & 0x000000FFFFFFFFFF) << 24; + int64_t uuid = x + ((hashval & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); if (uuid) { - return uuid; + return abs(uuid); } } while (true); } diff --git a/source/dnode/mnode/impl/test/CMakeLists.txt b/source/dnode/mnode/impl/test/CMakeLists.txt index df0510f783..61201f33c3 100644 --- a/source/dnode/mnode/impl/test/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/CMakeLists.txt @@ -12,5 +12,6 @@ add_subdirectory(dnode) add_subdirectory(mnode) add_subdirectory(db) add_subdirectory(stb) +add_subdirectory(sma) add_subdirectory(func) add_subdirectory(topic) diff --git a/source/dnode/mnode/impl/test/sma/CMakeLists.txt b/source/dnode/mnode/impl/test/sma/CMakeLists.txt new file mode 100644 index 0000000000..943695abf3 --- /dev/null +++ b/source/dnode/mnode/impl/test/sma/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. SMA_SRC) +add_executable(mnode_test_sma ${SMA_SRC}) +target_link_libraries( + mnode_test_sma + PUBLIC sut +) + +add_test( + NAME mnode_test_sma + COMMAND mnode_test_sma +) diff --git a/source/dnode/mnode/impl/test/sma/sma.cpp b/source/dnode/mnode/impl/test/sma/sma.cpp new file mode 100644 index 0000000000..5b48906681 --- /dev/null +++ b/source/dnode/mnode/impl/test/sma/sma.cpp @@ -0,0 +1,236 @@ +/** + * @file sma.cpp + * @author slguan (slguan@taosdata.com) + * @brief MNODE module sma tests + * @version 1.0 + * @date 2022-03-23 + * + * @copyright Copyright (c) 2022 + * + */ + +#include "sut.h" + +class MndTestSma : public ::testing::Test { + protected: + static void SetUpTestSuite() { test.Init("/tmp/mnode_test_sma", 9035); } + static void TearDownTestSuite() { test.Cleanup(); } + + static Testbase test; + + public: + void SetUp() override {} + void TearDown() override {} + + void* BuildCreateDbReq(const char* dbname, int32_t* pContLen); + void* BuildDropDbReq(const char* dbname, int32_t* pContLen); + void* BuildCreateStbReq(const char* stbname, int32_t* pContLen); + void* BuildDropStbReq(const char* stbname, int32_t* pContLen); + void* BuildCreateSmaReq(const char* smaname, const char* stbname, int8_t igExists, const char* expr, + const char* tagsFilter, const char* sql, const char* ast, int32_t* pContLen); + void* BuildDropSmaReq(const char* smaname, int8_t igNotExists, int32_t* pContLen); +}; + +Testbase MndTestSma::test; + +void* MndTestSma::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { + SCreateDbReq createReq = {0}; + strcpy(createReq.db, dbname); + createReq.numOfVgroups = 2; + createReq.cacheBlockSize = 16; + createReq.totalBlocks = 10; + createReq.daysPerFile = 10; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.commitTime = 3600; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.quorum = 1; + createReq.update = 0; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; + + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); + + *pContLen = contLen; + return pReq; +} + +void* MndTestSma::BuildDropDbReq(const char* dbname, int32_t* pContLen) { + SDropDbReq dropdbReq = {0}; + strcpy(dropdbReq.db, dbname); + + int32_t contLen = tSerializeSDropDbReq(NULL, 0, &dropdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropDbReq(pReq, contLen, &dropdbReq); + + *pContLen = contLen; + return pReq; +} + +void* MndTestSma::BuildCreateStbReq(const char* stbname, int32_t* pContLen) { + SMCreateStbReq createReq = {0}; + createReq.numOfColumns = 3; + createReq.numOfTags = 1; + createReq.igExists = 0; + createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField)); + createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField)); + strcpy(createReq.name, stbname); + + { + SField field = {0}; + field.bytes = 8; + field.type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(field.name, "ts"); + taosArrayPush(createReq.pColumns, &field); + } + + { + SField field = {0}; + field.bytes = 2; + field.type = TSDB_DATA_TYPE_TINYINT; + strcpy(field.name, "col1"); + taosArrayPush(createReq.pColumns, &field); + } + + { + SField field = {0}; + field.bytes = 8; + field.type = TSDB_DATA_TYPE_BIGINT; + strcpy(field.name, "col2"); + taosArrayPush(createReq.pColumns, &field); + } + + { + SField field = {0}; + field.bytes = 2; + field.type = TSDB_DATA_TYPE_TINYINT; + strcpy(field.name, "tag1"); + taosArrayPush(createReq.pTags, &field); + } + + int32_t tlen = tSerializeSMCreateStbReq(NULL, 0, &createReq); + void* pHead = rpcMallocCont(tlen); + tSerializeSMCreateStbReq(pHead, tlen, &createReq); + tFreeSMCreateStbReq(&createReq); + *pContLen = tlen; + return pHead; +} + +void* MndTestSma::BuildDropStbReq(const char* stbname, int32_t* pContLen) { + SMDropStbReq dropstbReq = {0}; + strcpy(dropstbReq.name, stbname); + + int32_t contLen = tSerializeSMDropStbReq(NULL, 0, &dropstbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMDropStbReq(pReq, contLen, &dropstbReq); + + *pContLen = contLen; + return pReq; +} + +void* MndTestSma::BuildCreateSmaReq(const char* smaname, const char* stbname, int8_t igExists, const char* expr, + const char* tagsFilter, const char* sql, const char* ast, int32_t* pContLen) { + SMCreateSmaReq createReq = {0}; + strcpy(createReq.name, smaname); + strcpy(createReq.stb, stbname); + createReq.igExists = igExists; + createReq.intervalUnit = 1; + createReq.slidingUnit = 2; + createReq.timezone = 3; + createReq.dstVgId = 4; + createReq.interval = 10; + createReq.offset = 5; + createReq.sliding = 6; + createReq.expr = (char*)expr; + createReq.exprLen = strlen(createReq.expr) + 1; + createReq.tagsFilter = (char*)tagsFilter; + createReq.tagsFilterLen = strlen(createReq.tagsFilter) + 1; + createReq.sql = (char*)sql; + createReq.sqlLen = strlen(createReq.sql) + 1; + createReq.ast = (char*)expr; + createReq.astLen = strlen(createReq.ast) + 1; + + int32_t tlen = tSerializeSMCreateSmaReq(NULL, 0, &createReq); + void* pHead = rpcMallocCont(tlen); + tSerializeSMCreateSmaReq(pHead, tlen, &createReq); + *pContLen = tlen; + return pHead; +} + +void* MndTestSma::BuildDropSmaReq(const char* smaname, int8_t igNotExists, int32_t* pContLen) { + SMDropSmaReq dropsmaReq = {0}; + dropsmaReq.igNotExists = igNotExists; + strcpy(dropsmaReq.name, smaname); + + int32_t contLen = tSerializeSMDropSmaReq(NULL, 0, &dropsmaReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMDropSmaReq(pReq, contLen, &dropsmaReq); + + *pContLen = contLen; + return pReq; +} + +TEST_F(MndTestSma, 01_Create_Show_Meta_Drop_Restart_Stb) { + const char* dbname = "1.d1"; + const char* stbname = "1.d1.stb"; + const char* smaname = "1.d1.sma"; + int32_t contLen = 0; + void* pReq; + SRpcMsg* pRsp; + + { + pReq = BuildCreateDbReq(dbname, &contLen); + pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + } + + { + pReq = BuildCreateStbReq(stbname, &contLen); + pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + } + + { + pReq = BuildCreateSmaReq(smaname, stbname, 0, "expr", "tagsFilter", "sql", "ast", &contLen); + pRsp = test.SendReq(TDMT_MND_CREATE_SMA, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + test.SendShowMetaReq(TSDB_MGMT_TABLE_INDEX, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + } + + // restart + test.Restart(); + + { + test.SendShowMetaReq(TSDB_MGMT_TABLE_INDEX, dbname); + CHECK_META("show indexes", 3); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckBinary("sma", TSDB_INDEX_NAME_LEN); + CheckTimestamp(); + CheckBinary("stb", TSDB_TABLE_NAME_LEN); + } + + { + pReq = BuildDropSmaReq(smaname, 0, &contLen); + pRsp = test.SendReq(TDMT_MND_DROP_SMA, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + test.SendShowMetaReq(TSDB_MGMT_TABLE_INDEX, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 0); + } +} diff --git a/source/dnode/vnode/inc/meta.h b/source/dnode/vnode/inc/meta.h index a48f437c97..149aac1206 100644 --- a/source/dnode/vnode/inc/meta.h +++ b/source/dnode/vnode/inc/meta.h @@ -51,7 +51,7 @@ int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg); int metaDropTable(SMeta *pMeta, tb_uid_t uid); int metaCommit(SMeta *pMeta); int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg); -int32_t metaDropTSma(SMeta *pMeta, char *indexName); +int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid); // For Query STbCfg * metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid); diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index 7b93f7580d..2b10c885b9 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -96,6 +96,7 @@ int tsdbCommit(STsdb *pTsdb); */ int32_t tsdbInsertTSmaData(STsdb *pTsdb, char *msg); int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg); +int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid); /** * @brief Insert RSma(Time-range-wise Rollup SMA) data. diff --git a/source/dnode/vnode/src/inc/metaDef.h b/source/dnode/vnode/src/inc/metaDef.h index 16a53baef0..bc1017f0c7 100644 --- a/source/dnode/vnode/src/inc/metaDef.h +++ b/source/dnode/vnode/src/inc/metaDef.h @@ -34,7 +34,7 @@ void metaCloseDB(SMeta* pMeta); int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg); int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg); -int metaRemoveSmaFromDb(SMeta* pMeta, const char* indexName); +int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid); // SMetaCache int metaOpenCache(SMeta* pMeta); diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index 4d4bb12a21..deb3cae617 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -167,6 +167,7 @@ struct STQ { STqMetaStore* tqMeta; STqPushMgr* tqPushMgr; SHashObj* pStreamTasks; + SVnode* pVnode; SWal* pWal; SMeta* pVnodeMeta; }; diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h index f934b0263d..c15a17469c 100644 --- a/source/dnode/vnode/src/inc/tsdbSma.h +++ b/source/dnode/vnode/src/inc/tsdbSma.h @@ -16,15 +16,15 @@ #ifndef _TD_TSDB_SMA_H_ #define _TD_TSDB_SMA_H_ -typedef struct SSmaStat SSmaStat; -typedef struct SSmaEnv SSmaEnv; +typedef struct SSmaStat SSmaStat; +typedef struct SSmaEnv SSmaEnv; struct SSmaEnv { TdThreadRwlock lock; - SDiskID did; - TDBEnv dbEnv; // TODO: If it's better to put it in smaIndex level? - char * path; // relative path - SSmaStat * pStat; + SDiskID did; + TDBEnv dbEnv; // TODO: If it's better to put it in smaIndex level? + char *path; // relative path + SSmaStat *pStat; }; #define SMA_ENV_LOCK(env) ((env)->lock) diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index ed9aad9277..78ff9f1062 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -82,12 +82,6 @@ struct SVnode { int vnodeScheduleTask(SVnodeTask* task); -int32_t vnodePutToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq); -int32_t vnodePutToVFetchQ(SVnode* pVnode, struct SRpcMsg* pReq); -int32_t vnodeSendReq(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq); -int32_t vnodeSendMnodeReq(SVnode* pVnode, struct SRpcMsg* pReq); -void vnodeSendRsp(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pRsp); - #define vFatal(...) \ do { \ if (vDebugFlag & DEBUG_FATAL) { \ @@ -177,19 +171,20 @@ int tqInit(); void tqCleanUp(); // open in each vnode -STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac); +STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, + SMemAllocatorFactory* allocFac); void tqClose(STQ*); // required by vnode -int tqPushMsg(STQ*, void* msg, tmsg_t msgType, int64_t version); +int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version); int tqCommit(STQ*); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessSetConnReq(STQ* pTq, char* msg); int32_t tqProcessRebReq(STQ* pTq, char* msg); int32_t tqProcessTaskExec(STQ* pTq, SRpcMsg* msg); - int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen); +int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index 2dd8386d7a..e4bad9e94b 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -259,7 +259,7 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { return 0; } -int metaRemoveSmaFromDb(SMeta *pMeta, const char *indexName) { +int metaRemoveSmaFromDb(SMeta *pMeta, int64_t indexUid) { // TODO #if 0 DBT key = {0}; diff --git a/source/dnode/vnode/src/meta/metaIdx.c b/source/dnode/vnode/src/meta/metaIdx.c index 881ea4f46d..818da14738 100644 --- a/source/dnode/vnode/src/meta/metaIdx.c +++ b/source/dnode/vnode/src/meta/metaIdx.c @@ -121,11 +121,11 @@ int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg) { return TSDB_CODE_SUCCESS; } -int32_t metaDropTSma(SMeta *pMeta, char* indexName) { +int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) { // TODO: Validate the cfg // TODO: add atomicity - if (metaRemoveSmaFromDb(pMeta, indexName) < 0) { + if (metaRemoveSmaFromDb(pMeta, indexUid) < 0) { // TODO: handle error return -1; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 3af79ca461..25fe963799 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -17,11 +17,14 @@ #include "tqInt.h" #include "tqMetaStore.h" +void tqDebugShowSSData(SArray* dataBlocks); + int32_t tqInit() { return tqPushMgrInit(); } void tqCleanUp() { tqPushMgrCleanUp(); } -STQ* tqOpen(const char* path, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) { +STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, + SMemAllocatorFactory* allocFac) { STQ* pTq = malloc(sizeof(STQ)); if (pTq == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; @@ -29,6 +32,7 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, S } pTq->path = strdup(path); pTq->tqConfig = tqConfig; + pTq->pVnode = pVnode; pTq->pWal = pWal; pTq->pVnodeMeta = pVnodeMeta; #if 0 @@ -68,46 +72,19 @@ void tqClose(STQ* pTq) { // TODO } -int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) { +int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version) { if (msgType != TDMT_VND_SUBMIT) return 0; - - void* pIter = NULL; - - while (1) { - pIter = taosHashIterate(pTq->pStreamTasks, pIter); - if (pIter == NULL) break; - SStreamTask* pTask = (SStreamTask*)pIter; - if (!pTask->pipeSource) continue; - - int32_t workerId = 0; - void* exec = pTask->runner[workerId].executor; - qSetStreamInput(exec, msg, STREAM_DATA_TYPE_SUBMIT_BLOCK); - SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); - while (1) { - SSDataBlock* output; - uint64_t ts; - if (qExecTask(exec, &output, &ts) < 0) { - ASSERT(false); - } - if (output == NULL) { - break; - } - taosArrayPush(pRes, output); - } - if (pTask->pipeSink) { - // write back - } else { - int32_t tlen = sizeof(SStreamExecMsgHead) + tEncodeDataBlocks(NULL, pRes); - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - return -1; - } - void* abuf = POINTER_SHIFT(buf, sizeof(SStreamExecMsgHead)); - tEncodeDataBlocks(abuf, pRes); - // serialize - // to next level - } + void* data = malloc(msgLen); + if (data == NULL) { + return -1; } + memcpy(data, msg, msgLen); + SRpcMsg req = { + .msgType = TDMT_VND_STREAM_TRIGGER, + .pCont = data, + .contLen = msgLen, + }; + tmsgPutToQueue(&pTq->pVnode->msgCb, FETCH_QUEUE, &req); #if 0 void* pIter = taosHashIterate(pTq->tqPushMgr->pHash, NULL); @@ -483,7 +460,9 @@ int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { } SCoder decoder; tCoderInit(&decoder, TD_LITTLE_ENDIAN, (uint8_t*)msg, msgLen, TD_DECODER); - tDecodeSStreamTask(&decoder, pTask); + if (tDecodeSStreamTask(&decoder, pTask) < 0) { + ASSERT(0); + } tCoderClear(&decoder); tqExpandTask(pTq, pTask, 8); @@ -560,7 +539,11 @@ void tqDebugShowSSData(SArray* dataBlocks) { break; case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_UINT: - printf(" %15u |", *(uint32_t*)var); + printf(" %15d |", *(int32_t*)var); + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + printf(" %15ld |", *(int64_t*)var); break; } } @@ -569,6 +552,62 @@ void tqDebugShowSSData(SArray* dataBlocks) { } } +int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen) { + void* pIter = NULL; + + while (1) { + pIter = taosHashIterate(pTq->pStreamTasks, pIter); + if (pIter == NULL) break; + SStreamTask* pTask = (SStreamTask*)pIter; + if (!pTask->pipeSource) continue; + + int32_t workerId = 0; + void* exec = pTask->runner[workerId].executor; + qSetStreamInput(exec, data, STREAM_DATA_TYPE_SUBMIT_BLOCK); + SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); + while (1) { + SSDataBlock* output; + uint64_t ts; + if (qExecTask(exec, &output, &ts) < 0) { + ASSERT(false); + } + if (output == NULL) { + break; + } + taosArrayPush(pRes, output); + } + if (pTask->pipeSink) { + // write back + /*printf("reach end\n");*/ + tqDebugShowSSData(pRes); + } else { + int32_t tlen = sizeof(SStreamExecMsgHead) + tEncodeDataBlocks(NULL, pRes); + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + return -1; + } + void* abuf = POINTER_SHIFT(buf, sizeof(SStreamExecMsgHead)); + tEncodeDataBlocks(abuf, pRes); + tmsg_t type; + + if (pTask->nextOpDst == STREAM_NEXT_OP_DST__VND) { + type = TDMT_VND_TASK_EXEC; + } else { + type = TDMT_SND_TASK_EXEC; + } + + SRpcMsg reqMsg = { + .pCont = buf, + .contLen = tlen, + .code = 0, + .msgType = type, + }; + tmsgSendReq(&pTq->pVnode->msgCb, &pTask->NextOpEp, &reqMsg); + } + } + return 0; +} + int32_t tqProcessTaskExec(STQ* pTq, SRpcMsg* msg) { SStreamTaskExecReq* pReq = msg->pCont; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 0c4b933c19..8c58b5ce07 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -126,6 +126,36 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { if (pArray == NULL) { return NULL; } + int32_t colMeta = 0; + int32_t colNeed = 0; + while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { + SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; + int16_t colIdSchema = pColSchema->colId; + int16_t colIdNeed = *(int16_t*)taosArrayGet(pHandle->pColIdList, colNeed); + if (colIdSchema < colIdNeed) { + colMeta++; + } else if (colIdSchema > colIdNeed) { + colNeed++; + } else { + SColumnInfoData colInfo = {0}; + int sz = numOfRows * pColSchema->bytes; + colInfo.info.bytes = pColSchema->bytes; + colInfo.info.colId = pColSchema->colId; + colInfo.info.type = pColSchema->type; + + colInfo.pData = calloc(1, sz); + if (colInfo.pData == NULL) { + // TODO free + taosArrayDestroy(pArray); + return NULL; + } + + blockDataEnsureColumnCapacity(&colInfo, numOfRows); + taosArrayPush(pArray, &colInfo); + colMeta++; + colNeed++; + } + } int j = 0; for (int32_t i = 0; i < colNumNeed; i++) { @@ -163,11 +193,23 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { tdSTSRowIterReset(&iter, row); // get all wanted col of that block + int32_t colTot = taosArrayGetSize(pArray); + for (int32_t i = 0; i < colTot; i++) { + SColumnInfoData* pColData = taosArrayGet(pArray, i); + SCellVal sVal = {0}; + if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) { + break; + } + memcpy(POINTER_SHIFT(pColData->pData, curRow * pColData->info.bytes), sVal.val, pColData->info.bytes); + } +#if 0 for (int32_t i = 0; i < colNumNeed; i++) { SColumnInfoData* pColData = taosArrayGet(pArray, i); STColumn* pCol = schemaColAt(pTschema, i); // TODO - ASSERT(pCol->colId == pColData->info.colId); + if(pCol->colId != pColData->info.colId) { + continue; + } // void* val = tdGetMemRowDataOfColEx(row, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx); SCellVal sVal = {0}; if (!tdSTSRowIterNext(&iter, pCol->colId, pCol->type, &sVal)) { @@ -176,6 +218,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { } memcpy(POINTER_SHIFT(pColData->pData, curRow * pCol->bytes), sVal.val, pCol->bytes); } +#endif curRow++; } return pArray; diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 0eb2d525b3..80f8158139 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -25,6 +25,7 @@ static const char *TSDB_SMA_DNAME[] = { #define SMA_STORAGE_TSDB_TIMES 10 #define SMA_STORAGE_SPLIT_HOURS 24 #define SMA_KEY_LEN 18 // tableUid_colId_TSKEY 8+2+8 +#define SMA_DROP_EXPIRED_TIME 10 // default is 10 seconds #define SMA_STATE_HASH_SLOT 4 #define SMA_STATE_ITEM_HASH_SLOT 32 @@ -60,10 +61,11 @@ typedef struct { typedef struct { /** * @brief The field 'state' is here to demonstrate if one smaIndex is ready to provide service. - * - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open, - * without information about its previous state. * - TSDB_SMA_STAT_OK: 1) The sma calculation of history data is finished; 2) Or recevied information from * Streaming Module or TSDB local persistence. + * - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open, + * without information about its previous state. + * - TSDB_SMA_STAT_DROPPED: 1)sma dropped */ int8_t state; // ETsdbSmaStat SHashObj *expiredWindows; // key: skey of time window, value: N/A @@ -80,6 +82,7 @@ struct SSmaStat { // expired window static int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg); static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat); +static void * tsdbFreeSmaStatItem(SSmaStatItem *pSmaStatItem); static int32_t tsdbDestroySmaState(SSmaStat *pSmaStat); static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did); static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SDiskID did, SSmaEnv **pEnv); @@ -109,7 +112,55 @@ static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg); static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg); +// mgmt interface +static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid); + // implementation +static FORCE_INLINE int8_t tsdbSmaStat(SSmaStatItem *pStatItem) { + if (pStatItem) { + return atomic_load_8(&pStatItem->state); + } + return TSDB_SMA_STAT_UNKNOWN; +} + +static FORCE_INLINE bool tsdbSmaStatIsOK(SSmaStatItem *pStatItem, int8_t *state) { + if(!pStatItem) { + return false; + } + + if (state) { + *state = atomic_load_8(&pStatItem->state); + return *state == TSDB_SMA_STAT_OK; + } + return atomic_load_8(&pStatItem->state) == TSDB_SMA_STAT_OK; +} + +static FORCE_INLINE bool tsdbSmaStatIsExpired(SSmaStatItem *pStatItem) { + return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_EXPIRED) : true; +} + +static FORCE_INLINE bool tsdbSmaStatIsDropped(SSmaStatItem *pStatItem) { + return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_DROPPED) : true; +} + +static FORCE_INLINE void tsdbSmaStatSetOK(SSmaStatItem *pStatItem) { + if (pStatItem) { + atomic_store_8(&pStatItem->state, TSDB_SMA_STAT_OK); + } +} + +static FORCE_INLINE void tsdbSmaStatSetExpired(SSmaStatItem *pStatItem) { + if (pStatItem) { + atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_EXPIRED); + } +} + +static FORCE_INLINE void tsdbSmaStatSetDropped(SSmaStatItem *pStatItem) { + if (pStatItem) { + atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_DROPPED); + } +} + static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) { snprintf(dirName, TSDB_FILENAME_LEN, "vnode%svnode%d%stsdb%s%s", TD_DIRSEP, vgId, TD_DIRSEP, TD_DIRSEP, TSDB_SMA_DNAME[smaType]); @@ -252,6 +303,16 @@ static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) { return pItem; } +static void *tsdbFreeSmaStatItem(SSmaStatItem *pSmaStatItem) { + if (pSmaStatItem != NULL) { + tdDestroyTSma(pSmaStatItem->pSma); + tfree(pSmaStatItem->pSma); + taosHashCleanup(pSmaStatItem->expiredWindows); + tfree(pSmaStatItem); + } + return NULL; +} + /** * @brief Release resources allocated for its member fields, not including itself. * @@ -264,12 +325,7 @@ int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) { void *item = taosHashIterate(pSmaStat->smaStatItems, NULL); while (item != NULL) { SSmaStatItem *pItem = *(SSmaStatItem **)item; - if (pItem != NULL) { - tdDestroyTSma(pItem->pSma); - tfree(pItem->pSma); - taosHashCleanup(pItem->expiredWindows); - tfree(pItem); - } + tsdbFreeSmaStatItem(pItem); item = taosHashIterate(pSmaStat->smaStatItems, item); } taosHashCleanup(pSmaStat->smaStatItems); @@ -437,6 +493,15 @@ static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t ind skey, indexUid); return TSDB_CODE_FAILED; } + // TODO: use a standalone interface to received state upate notification from stream computing module. + /** + * @brief state + * - When SMA env init in TSDB, its status is TSDB_SMA_STAT_OK. + * - In startup phase of stream computing module, it should notify the SMA env in TSDB to expired if needed(e.g. + * when batch data caculation not finised) + * - When TSDB_SMA_STAT_OK, the stream computing module should also notify that to the SMA env in TSDB. + */ + pItem->state = TSDB_SMA_STAT_OK; } else { // error handling tsdbUnRefSmaStat(pTsdb, pStat); @@ -711,6 +776,150 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { STsdbCfg * pCfg = REPO_CFG(pTsdb); STSmaDataWrapper *pData = (STSmaDataWrapper *)msg; SSmaEnv * pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); + int64_t indexUid = SMA_TEST_INDEX_UID; + + + if (pEnv == NULL) { + terrno = TSDB_CODE_INVALID_PTR; + tsdbWarn("vgId:%d insert tSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); + return terrno; + } + + if (pData->dataLen <= 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return TSDB_CODE_FAILED; + } + + STSmaWriteH tSmaH = {0}; + + if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData) != 0) { + return TSDB_CODE_FAILED; + } + + SSmaStat *pStat = SMA_ENV_STAT(pTsdb->pTSmaEnv); + SSmaStatItem *pItem = NULL; + + tsdbRefSmaStat(pTsdb, pStat); + + if (pStat && pStat->smaStatItems) { + pItem = taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid)); + } + + if ((pItem == NULL) || ((pItem = *(SSmaStatItem **)pItem) == NULL) || tsdbSmaStatIsDropped(pItem)) { + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + tsdbUnRefSmaStat(pTsdb, pStat); + return TSDB_CODE_FAILED; + } + + char rPath[TSDB_FILENAME_LEN] = {0}; + char aPath[TSDB_FILENAME_LEN] = {0}; + snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); + tfsAbsoluteName(REPO_TFS(pTsdb), SMA_ENV_DID(pEnv), rPath, aPath); + if (!taosCheckExistFile(aPath)) { + if (tfsMkdirRecurAt(REPO_TFS(pTsdb), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) { + tsdbUnRefSmaStat(pTsdb, pStat); + return TSDB_CODE_FAILED; + } + } + + // Step 1: Judge the storage level and days + int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit); + int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); + int32_t fid = (int32_t)(TSDB_KEY_FID(pData->skey, daysPerFile, pCfg->precision)); + + // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file + // - Set and open the DFile or the B+Tree file + // TODO: tsdbStartTSmaCommit(); + tsdbSetTSmaDataFile(&tSmaH, pData, indexUid, fid); + if (tsdbOpenDBF(pTsdb->pTSmaEnv->dbEnv, &tSmaH.dFile) != 0) { + tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb), + tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); + tsdbDestroyTSmaWriteH(&tSmaH); + tsdbUnRefSmaStat(pTsdb, pStat); + return TSDB_CODE_FAILED; + } + + if (tsdbInsertTSmaDataSection(&tSmaH, pData) != 0) { + tsdbWarn("vgId:%d insert tSma data section failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); + tsdbDestroyTSmaWriteH(&tSmaH); + tsdbUnRefSmaStat(pTsdb, pStat); + return TSDB_CODE_FAILED; + } + // TODO:tsdbEndTSmaCommit(); + + // Step 3: reset the SSmaStat + tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey); + + tsdbDestroyTSmaWriteH(&tSmaH); + tsdbUnRefSmaStat(pTsdb, pStat); + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Drop tSma data and local cache + * - insert/query reference + * @param pTsdb + * @param msg + * @return int32_t + */ +static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid) { + SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); + + // clear local cache + if (pEnv) { + tsdbDebug("vgId:%d drop tSma local cache for %" PRIi64, REPO_ID(pTsdb), indexUid); + + SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); + if ((pItem != NULL) || ((pItem = *(SSmaStatItem **)pItem) != NULL)) { + if (tsdbSmaStatIsDropped(pItem)) { + tsdbDebug("vgId:%d tSma stat is already dropped for %" PRIi64, REPO_ID(pTsdb), indexUid); + return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode + } + + tsdbWLockSma(pEnv); + if (tsdbSmaStatIsDropped(pItem)) { + tsdbUnLockSma(pEnv); + tsdbDebug("vgId:%d tSma stat is already dropped for %" PRIi64, REPO_ID(pTsdb), indexUid); + return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode + } + tsdbSmaStatSetDropped(pItem); + tsdbUnLockSma(pEnv); + + int32_t nSleep = 0; + while (true) { + if (T_REF_VAL_GET(SMA_ENV_STAT(pEnv)) <= 0) { + break; + } + taosSsleep(1); + if (++nSleep > SMA_DROP_EXPIRED_TIME) { + break; + }; + } + + tsdbFreeSmaStatItem(pItem); + tsdbDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64 " in local cache", REPO_ID(pTsdb), indexUid); + } + } + // clear sma data files + // TODO: + +} + +static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t fid) { + STsdb *pTsdb = pSmaH->pTsdb; + + char tSmaFile[TSDB_FILENAME_LEN] = {0}; + snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.rsma", REPO_ID(pTsdb), fid); + pSmaH->dFile.path = strdup(tSmaFile); + + return TSDB_CODE_SUCCESS; +} + +static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { + STsdbCfg * pCfg = REPO_CFG(pTsdb); + STSmaDataWrapper *pData = (STSmaDataWrapper *)msg; + SSmaEnv * pEnv = atomic_load_ptr(&pTsdb->pRSmaEnv); if (pEnv == NULL) { terrno = TSDB_CODE_INVALID_PTR; @@ -771,51 +980,6 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { return TSDB_CODE_SUCCESS; } -static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t fid) { - STsdb *pTsdb = pSmaH->pTsdb; - - char tSmaFile[TSDB_FILENAME_LEN] = {0}; - snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.rsma", REPO_ID(pTsdb), fid); - pSmaH->dFile.path = strdup(tSmaFile); - - return TSDB_CODE_SUCCESS; -} - -static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { - STsdbCfg * pCfg = REPO_CFG(pTsdb); - STSmaDataWrapper *pData = (STSmaDataWrapper *)msg; - STSmaWriteH tSmaH = {0}; - - tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData); - - if (pData->dataLen <= 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - - // Step 1: Judge the storage level - int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit); - int32_t daysPerFile = storageLevel == SMA_STORAGE_LEVEL_TSDB ? SMA_STORAGE_TSDB_DAYS : pCfg->daysPerFile; - - // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file - // - Set and open the DFile or the B+Tree file - - int32_t fid = (int32_t)(TSDB_KEY_FID(pData->skey, daysPerFile, pCfg->precision)); - - // Save all the TSma data to one file - // TODO: tsdbStartTSmaCommit(); - tsdbSetTSmaDataFile(&tSmaH, pData, storageLevel, fid); - - tsdbInsertTSmaDataSection(&tSmaH, pData); - // TODO:tsdbEndTSmaCommit(); - - // reset the SSmaStat - tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pRSmaEnv), pData->indexUid, pData->skey); - - return TSDB_CODE_SUCCESS; -} - /** * @brief * @@ -934,6 +1098,15 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_ #endif #if 1 + int8_t smaStat = 0; + if (!tsdbSmaStatIsOK(pItem, &smaStat)) { // TODO: multiple check for large scale sma query + tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv)); + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + tsdbWarn("vgId:%d getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, REPO_ID(pTsdb), indexUid, + tstrerror(terrno), smaStat); + return TSDB_CODE_FAILED; + } + if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY)) != NULL) { // TODO: mark this window as expired. tsdbDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, REPO_ID(pTsdb), @@ -1086,6 +1259,20 @@ int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg) { return code; } +/** + * @brief Get tSma data + * + * @param pTsdb + * @param pData + * @param indexUid + * @param interval + * @param intervalUnit + * @param tableUid + * @param colId + * @param querySKey + * @param nMaxResult + * @return int32_t + */ int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySKey, int32_t nMaxResult) { int32_t code = TSDB_CODE_SUCCESS; @@ -1094,4 +1281,19 @@ int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, tsdbWarn("vgId:%d get tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); } return code; +} + +/** + * @brief Drop tSma Data and caches + * + * @param pTsdb + * @param msg + * @return int32_t + */ +int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tsdbDropTSmaDataImpl(pTsdb, indexUid)) < 0) { + tsdbWarn("vgId:%d drop tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); + } + return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c index 86e670d533..70f4117976 100644 --- a/source/dnode/vnode/src/vnd/vnodeMain.c +++ b/source/dnode/vnode/src/vnd/vnodeMain.c @@ -115,7 +115,8 @@ static int vnodeOpenImpl(SVnode *pVnode) { // Open tsdb sprintf(dir, "%s/tsdb", pVnode->path); - pVnode->pTsdb = tsdbOpen(dir, pVnode->vgId, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode), pVnode->pMeta, pVnode->pTfs); + pVnode->pTsdb = + tsdbOpen(dir, pVnode->vgId, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode), pVnode->pMeta, pVnode->pTfs); if (pVnode->pTsdb == NULL) { // TODO: handle error return -1; @@ -131,7 +132,7 @@ static int vnodeOpenImpl(SVnode *pVnode) { // Open TQ sprintf(dir, "%s/tq", pVnode->path); - pVnode->pTq = tqOpen(dir, pVnode->pWal, pVnode->pMeta, &(pVnode->config.tqCfg), vBufPoolGetMAF(pVnode)); + pVnode->pTq = tqOpen(dir, pVnode, pVnode->pWal, pVnode->pMeta, &(pVnode->config.tqCfg), vBufPoolGetMAF(pVnode)); if (pVnode->pTq == NULL) { // TODO: handle error return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 2f5f94b55f..a3258044c8 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -68,6 +68,8 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg) { return tqProcessPollReq(pVnode->pTq, pMsg); case TDMT_VND_TASK_EXEC: return tqProcessTaskExec(pVnode->pTq, pMsg); + case TDMT_VND_STREAM_TRIGGER: + return tqProcessStreamTrigger(pVnode->pTq, pMsg->pCont, pMsg->contLen); case TDMT_VND_QUERY_HEARTBEAT: return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg); default: @@ -163,7 +165,6 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { memcpy(POINTER_SHIFT(metaRsp.pSchemas, sizeof(SSchema) * pSW->nCols), pTagSchema, sizeof(SSchema) * nTagCols); } - _exit: rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); @@ -179,7 +180,6 @@ _exit: } tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp); - tFreeSTableMetaRsp(&metaRsp); if (pSW != NULL) { tfree(pSW->pSchema); diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index ade9adb1e1..3ef45ecb09 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -59,7 +59,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // todo: change the interface here int64_t ver; taosDecodeFixedI64(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &ver); - if (tqPushMsg(pVnode->pTq, ptr, pMsg->msgType, ver) < 0) { + if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, ver) < 0) { // TODO: handle error } @@ -85,10 +85,10 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { for (int i = 0; i < reqNum; i++) { SVCreateTbReq *pCreateTbReq = taosArrayGet(vCreateTbBatchReq.pArray, i); - char tableFName[TSDB_TABLE_FNAME_LEN]; + char tableFName[TSDB_TABLE_FNAME_LEN]; SMsgHead *pHead = (SMsgHead *)pMsg->pCont; sprintf(tableFName, "%s.%s", pCreateTbReq->dbFName, pCreateTbReq->name); - + int32_t code = vnodeValidateTableHash(&pVnode->config, tableFName); if (code) { SVCreateTbRsp rsp; @@ -96,7 +96,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { taosArrayPush(vCreateTbBatchRsp.rspList, &rsp); } - + if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) { // TODO: handle error vError("vgId:%d, failed to create table: %s", pVnode->vgId, pCreateTbReq->name); @@ -116,10 +116,10 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { taosArrayDestroy(vCreateTbBatchReq.pArray); if (vCreateTbBatchRsp.rspList) { int32_t contLen = tSerializeSVCreateTbBatchRsp(NULL, 0, &vCreateTbBatchRsp); - void *msg = rpcMallocCont(contLen); + void *msg = rpcMallocCont(contLen); tSerializeSVCreateTbBatchRsp(msg, contLen, &vCreateTbBatchRsp); taosArrayDestroy(vCreateTbBatchRsp.rspList); - + *pRsp = calloc(1, sizeof(SRpcMsg)); (*pRsp)->msgType = TDMT_VND_CREATE_TABLE_RSP; (*pRsp)->pCont = msg; @@ -168,6 +168,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } } break; case TDMT_VND_CREATE_SMA: { // timeRangeSMA +#if 0 SSmaCfg vCreateSmaReq = {0}; if (tDeserializeSVCreateTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateSmaReq) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -189,26 +190,37 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // } tdDestroyTSma(&vCreateSmaReq.tSma); // TODO: return directly or go on follow steps? +#endif } break; case TDMT_VND_CANCEL_SMA: { // timeRangeSMA } break; case TDMT_VND_DROP_SMA: { // timeRangeSMA +#if 0 SVDropTSmaReq vDropSmaReq = {0}; if (tDeserializeSVDropTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vDropSmaReq) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - if (metaDropTSma(pVnode->pMeta, vDropSmaReq.indexName) < 0) { - // TODO: handle error - return -1; - } // TODO: send msg to stream computing to drop tSma // if ((send msg to stream computing) < 0) { // tdDestroyTSma(&vCreateSmaReq); // return -1; // } + // + + if (metaDropTSma(pVnode->pMeta, vDropSmaReq.indexUid) < 0) { + // TODO: handle error + return -1; + } + + if(tsdbDropTSmaData(pVnode->pTsdb, vDropSmaReq.indexUid) < 0) { + // TODO: handle error + return -1; + } + // TODO: return directly or go on follow steps? +#endif } break; default: ASSERT(0); diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 3c51ad5d71..3fa5799f54 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -272,8 +272,8 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { taosArrayDestroy(pUids); // resource release - metaRemoveSmaFromDb(pMeta, smaIndexName1); - metaRemoveSmaFromDb(pMeta, smaIndexName2); + metaRemoveSmaFromDb(pMeta, indexUid1); + metaRemoveSmaFromDb(pMeta, indexUid2); tdDestroyTSma(&tSma); metaClose(pMeta); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 19100d7560..e89bc5df0e 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -30,7 +30,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, int32_t t qError("join not supported for stream block scan, %s" PRIx64, id); return TSDB_CODE_QRY_APP_ERROR; } - + pOperator->status = OP_NOT_OPENED; return doSetStreamBlock(pOperator->pDownstream[0], input, type, id); } else { SStreamBlockScanInfo* pInfo = pOperator->info; diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index d979e0df15..2196d6c207 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -47,6 +47,8 @@ SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore); SSyncRaftEntry* logStoreGetLastEntry(SSyncLogStore* pLogStore); cJSON* logStore2Json(SSyncLogStore* pLogStore); char* logStore2Str(SSyncLogStore* pLogStore); +cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore); +char* logStoreSimple2Str(SSyncLogStore* pLogStore); // for debug void logStorePrint(SSyncLogStore* pLogStore); @@ -54,6 +56,11 @@ void logStorePrint2(char* s, SSyncLogStore* pLogStore); void logStoreLog(SSyncLogStore* pLogStore); void logStoreLog2(char* s, SSyncLogStore* pLogStore); +void logStoreSimplePrint(SSyncLogStore* pLogStore); +void logStoreSimplePrint2(char* s, SSyncLogStore* pLogStore); +void logStoreSimpleLog(SSyncLogStore* pLogStore); +void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index e4df93ca47..1116c1a905 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -120,6 +120,11 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { // reject request if ((pMsg->term < ths->pRaftStore->currentTerm) || ((pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK)) { + sTrace( + "syncNodeOnAppendEntriesCb --> reject, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, ths->state:%d, " + "logOK:%d", + pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK); + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); pReply->srcId = ths->myRaftId; pReply->destId = pMsg->srcId; @@ -137,6 +142,11 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { // return to follower state if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE) { + sTrace( + "syncNodeOnAppendEntriesCb --> return to follower, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, " + "ths->state:%d, logOK:%d", + pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK); + syncNodeBecomeFollower(ths); // ret or reply? @@ -145,89 +155,60 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { // accept request if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_FOLLOWER && logOK) { - bool preMatch = false; - if (pMsg->prevLogIndex == SYNC_INDEX_INVALID && - ths->pLogStore->getLastIndex(ths->pLogStore) == SYNC_INDEX_INVALID) { - preMatch = true; - } - if (pMsg->prevLogIndex >= SYNC_INDEX_BEGIN && pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { - SSyncRaftEntry* pPreEntry = logStoreGetEntry(ths->pLogStore, pMsg->prevLogIndex); - assert(pPreEntry != NULL); - if (pMsg->prevLogTerm == pPreEntry->term) { - preMatch = true; + // preIndex = -1, or has preIndex entry in local log + assert(pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)); + + // has extra entries (> preIndex) in local log + bool hasExtraEntries = pMsg->prevLogIndex < ths->pLogStore->getLastIndex(ths->pLogStore); + + // has entries in SyncAppendEntries msg + bool hasAppendEntries = pMsg->dataLen > 0; + + sTrace( + "syncNodeOnAppendEntriesCb --> accept, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, ths->state:%d, " + "logOK:%d, hasExtraEntries:%d, hasAppendEntries:%d", + pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK, hasExtraEntries, hasAppendEntries); + + if (hasExtraEntries && hasAppendEntries) { + // not conflict by default + bool conflict = false; + + SyncIndex extraIndex = pMsg->prevLogIndex + 1; + SSyncRaftEntry* pExtraEntry = logStoreGetEntry(ths->pLogStore, extraIndex); + assert(pExtraEntry != NULL); + + SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + assert(pAppendEntry != NULL); + + // log not match, conflict + assert(extraIndex == pAppendEntry->index); + if (pExtraEntry->term != pAppendEntry->term) { + conflict = true; } - syncEntryDestory(pPreEntry); - } - if (preMatch) { - // must has preIndex in local log - assert(pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)); + if (conflict) { + // roll back + SyncIndex delBegin = ths->pLogStore->getLastIndex(ths->pLogStore); + SyncIndex delEnd = extraIndex; - bool hasExtraEntries = pMsg->prevLogIndex < ths->pLogStore->getLastIndex(ths->pLogStore); - bool hasAppendEntries = pMsg->dataLen > 0; + sTrace("syncNodeOnAppendEntriesCb --> conflict:%d, delBegin:%ld, delEnd:%ld", conflict, delBegin, delEnd); - if (hasExtraEntries && hasAppendEntries) { - // conflict - bool conflict = false; + // notice! reverse roll back! + for (SyncIndex index = delEnd; index >= delBegin; --index) { + if (ths->pFsm->FpRollBackCb != NULL) { + SSyncRaftEntry* pRollBackEntry = logStoreGetEntry(ths->pLogStore, index); + assert(pRollBackEntry != NULL); - SyncIndex extraIndex = pMsg->prevLogIndex + 1; - SSyncRaftEntry* pExtraEntry = logStoreGetEntry(ths->pLogStore, extraIndex); - assert(pExtraEntry != NULL); - - SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); - assert(pAppendEntry != NULL); - - assert(extraIndex == pAppendEntry->index); - if (pExtraEntry->term == pAppendEntry->term) { - conflict = true; + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); + ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, pRollBackEntry->index, pRollBackEntry->isWeak, 0, ths->state); + rpcFreeCont(rpcMsg.pCont); + syncEntryDestory(pRollBackEntry); + } } - if (conflict) { - // roll back - SyncIndex delBegin = ths->pLogStore->getLastIndex(ths->pLogStore); - SyncIndex delEnd = extraIndex; - - // notice! reverse roll back! - for (SyncIndex index = delEnd; index >= delBegin; --index) { - if (ths->pFsm->FpRollBackCb != NULL) { - SSyncRaftEntry* pRollBackEntry = logStoreGetEntry(ths->pLogStore, index); - assert(pRollBackEntry != NULL); - - SRpcMsg rpcMsg; - syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); - ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, pRollBackEntry->index, pRollBackEntry->isWeak, 0); - rpcFreeCont(rpcMsg.pCont); - syncEntryDestory(pRollBackEntry); - } - } - - // delete confict entries - ths->pLogStore->truncate(ths->pLogStore, extraIndex); - - // append new entries - ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); - - // pre commit - SRpcMsg rpcMsg; - syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); - if (ths->pFsm != NULL) { - if (ths->pFsm->FpPreCommitCb != NULL) { - ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 0); - } - } - rpcFreeCont(rpcMsg.pCont); - } - - // free memory - syncEntryDestory(pExtraEntry); - syncEntryDestory(pAppendEntry); - - } else if (hasExtraEntries && !hasAppendEntries) { - // do nothing - - } else if (!hasExtraEntries && hasAppendEntries) { - SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); - assert(pAppendEntry != NULL); + // delete confict entries + ths->pLogStore->truncate(ths->pLogStore, extraIndex); // append new entries ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); @@ -237,53 +218,63 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); if (ths->pFsm != NULL) { if (ths->pFsm->FpPreCommitCb != NULL) { - ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 0); + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 2, ths->state); } } rpcFreeCont(rpcMsg.pCont); - - // free memory - syncEntryDestory(pAppendEntry); - - } else if (!hasExtraEntries && !hasAppendEntries) { - // do nothing - - } else { - assert(0); } - SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); - pReply->srcId = ths->myRaftId; - pReply->destId = pMsg->srcId; - pReply->term = ths->pRaftStore->currentTerm; - pReply->success = true; + // free memory + syncEntryDestory(pExtraEntry); + syncEntryDestory(pAppendEntry); - if (hasAppendEntries) { - pReply->matchIndex = pMsg->prevLogIndex + 1; - } else { - pReply->matchIndex = pMsg->prevLogIndex; - } + } else if (hasExtraEntries && !hasAppendEntries) { + // do nothing + } else if (!hasExtraEntries && hasAppendEntries) { + SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + assert(pAppendEntry != NULL); + + // append new entries + ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); + + // pre commit SRpcMsg rpcMsg; - syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); - syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); + if (ths->pFsm != NULL) { + if (ths->pFsm->FpPreCommitCb != NULL) { + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 3, ths->state); + } + } + rpcFreeCont(rpcMsg.pCont); - syncAppendEntriesReplyDestroy(pReply); + // free memory + syncEntryDestory(pAppendEntry); + + } else if (!hasExtraEntries && !hasAppendEntries) { + // do nothing } else { - SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); - pReply->srcId = ths->myRaftId; - pReply->destId = pMsg->srcId; - pReply->term = ths->pRaftStore->currentTerm; - pReply->success = false; - pReply->matchIndex = SYNC_INDEX_INVALID; - - SRpcMsg rpcMsg; - syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); - syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); - syncAppendEntriesReplyDestroy(pReply); + assert(0); } + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->success = true; + + if (hasAppendEntries) { + pReply->matchIndex = pMsg->prevLogIndex + 1; + } else { + pReply->matchIndex = pMsg->prevLogIndex; + } + + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + // maybe update commit index from leader if (pMsg->commitIndex > ths->commitIndex) { // has commit entry in local @@ -308,7 +299,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { syncEntry2OriginalRpc(pEntry, &rpcMsg); if (ths->pFsm->FpCommitCb != NULL) { - ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0); + ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, ths->state); } rpcFreeCont(rpcMsg.pCont); diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 817974fd26..790ac7f8e1 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -48,6 +48,9 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p return ret; } + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== before pNextIndex", ths->pNextIndex); + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== before pMatchIndex", ths->pMatchIndex); + // no need this code, because if I receive reply.term, then I must have sent for that term. // if (pMsg->term > ths->pRaftStore->currentTerm) { // syncNodeUpdateTerm(ths, pMsg->term); @@ -77,5 +80,8 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex); } + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== after pNextIndex", ths->pNextIndex); + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== after pMatchIndex", ths->pMatchIndex); + return ret; } diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 0d4df8e6cf..f581b31c4b 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -63,7 +63,14 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { if (pEntry->term == pSyncNode->pRaftStore->currentTerm) { // update commit index newCommitIndex = index; + sTrace("syncMaybeAdvanceCommitIndex maybe to update, newCommitIndex:%ld commit, pSyncNode->commitIndex:%ld", + newCommitIndex, pSyncNode->commitIndex); break; + } else { + sTrace( + "syncMaybeAdvanceCommitIndex can not commit due to term not equal, pEntry->term:%lu, " + "pSyncNode->pRaftStore->currentTerm:%lu", + pEntry->term, pSyncNode->pRaftStore->currentTerm); } } } @@ -91,7 +98,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { syncEntry2OriginalRpc(pEntry, &rpcMsg); if (pSyncNode->pFsm->FpCommitCb != NULL) { - pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0); + pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, pSyncNode->state); } rpcFreeCont(rpcMsg.pCont); diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 19121632e2..b05d2e397d 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -257,13 +257,6 @@ static void *syncIOConsumerFunc(void *param) { assert(pSyncMsg != NULL); io->FpOnSyncPing(io->pSyncNode, pSyncMsg); syncPingDestroy(pSyncMsg); - /* - pSyncMsg = syncPingBuild(pRpcMsg->contLen); - syncPingFromRpcMsg(pRpcMsg, pSyncMsg); - // memcpy(pSyncMsg, tmpRpcMsg.pCont, tmpRpcMsg.contLen); - io->FpOnSyncPing(io->pSyncNode, pSyncMsg); - syncPingDestroy(pSyncMsg); - */ } } else if (pRpcMsg->msgType == SYNC_PING_REPLY) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index b92317ef3f..f2341ef521 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -104,29 +104,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { } int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { - int32_t ret = 0; - - // todo : get pointer from rid - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); - if (pSyncNode == NULL) { - return -1; - } - assert(rid == pSyncNode->rid); - - if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { - SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, 0, isWeak); - SRpcMsg rpcMsg; - syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); - pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); - syncClientRequestDestroy(pSyncMsg); - ret = 0; - - } else { - sTrace("syncForwardToPeer not leader, %s", syncUtilState2String(pSyncNode->state)); - ret = -1; // todo : need define err code !! - } - - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + int32_t ret = syncPropose2(rid, pMsg, isWeak, 0); return ret; } @@ -136,12 +114,38 @@ int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { } ESyncState syncGetMyRole(int64_t rid) { - // todo : get pointer from rid - SSyncNode* pSyncNode = NULL; + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return TAOS_SYNC_STATE_ERROR; + } + assert(rid == pSyncNode->rid); return pSyncNode->state; } -void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {} +int32_t syncPropose2(int64_t rid, const SRpcMsg* pMsg, bool isWeak, uint64_t seqNum) { + int32_t ret = 0; + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return -1; + } + assert(rid == pSyncNode->rid); + + if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { + SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, seqNum, isWeak); + SRpcMsg rpcMsg; + syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); + pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); + syncClientRequestDestroy(pSyncMsg); + ret = 0; + + } else { + sTrace("syncPropose not leader, %s", syncUtilState2String(pSyncNode->state)); + ret = -1; // todo : need define err code !! + } + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return ret; +} // open/close -------------- SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { @@ -848,7 +852,7 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg if (ths->pFsm != NULL) { if (ths->pFsm->FpPreCommitCb != NULL) { - ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0); + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, ths->state); } } rpcFreeCont(rpcMsg.pCont); @@ -863,7 +867,7 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg if (ths->pFsm != NULL) { if (ths->pFsm->FpPreCommitCb != NULL) { - ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, -2); + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 1, ths->state); } } rpcFreeCont(rpcMsg.pCont); diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 3ac4a7a1c0..f2344c6b6d 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -834,7 +834,7 @@ cJSON* syncRequestVote2Json(const SyncRequestVote* pMsg) { snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogIndex); + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->lastLogIndex); cJSON_AddStringToObject(pRoot, "lastLogIndex", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogTerm); cJSON_AddStringToObject(pRoot, "lastLogTerm", u64buf); @@ -1127,14 +1127,14 @@ cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg) { snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->prevLogIndex); - cJSON_AddStringToObject(pRoot, "pre_log_index", u64buf); + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->prevLogIndex); + cJSON_AddStringToObject(pRoot, "prevLogIndex", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->prevLogTerm); cJSON_AddStringToObject(pRoot, "pre_log_term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->commitIndex); - cJSON_AddStringToObject(pRoot, "commit_index", u64buf); + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->commitIndex); + cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); char* s; @@ -1288,7 +1288,7 @@ cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg) { snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); cJSON_AddNumberToObject(pRoot, "success", pMsg->success); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->matchIndex); + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->matchIndex); cJSON_AddStringToObject(pRoot, "matchIndex", u64buf); } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 620e923629..f569c5d621 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -165,6 +165,34 @@ char* logStore2Str(SSyncLogStore* pLogStore) { return serialized; } +cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore) { + char u64buf[128]; + SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; + cJSON* pRoot = cJSON_CreateObject(); + + if (pData != NULL && pData->pWal != NULL) { + snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); + cJSON_AddStringToObject(pRoot, "pWal", u64buf); + snprintf(u64buf, sizeof(u64buf), "%ld", logStoreLastIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastTerm(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncLogStoreSimple", pRoot); + return pJson; +} + +char* logStoreSimple2Str(SSyncLogStore* pLogStore) { + cJSON* pJson = logStoreSimple2Json(pLogStore); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + // for debug ----------------- void logStorePrint(SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); @@ -191,3 +219,30 @@ void logStoreLog2(char* s, SSyncLogStore* pLogStore) { sTrace("logStorePrint | len:%lu | %s | %s", strlen(serialized), s, serialized); free(serialized); } + +// for debug ----------------- +void logStoreSimplePrint(SSyncLogStore* pLogStore) { + char* serialized = logStoreSimple2Str(pLogStore); + printf("logStoreSimplePrint | len:%lu | %s \n", strlen(serialized), serialized); + fflush(NULL); + free(serialized); +} + +void logStoreSimplePrint2(char* s, SSyncLogStore* pLogStore) { + char* serialized = logStoreSimple2Str(pLogStore); + printf("logStoreSimplePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + free(serialized); +} + +void logStoreSimpleLog(SSyncLogStore* pLogStore) { + char* serialized = logStoreSimple2Str(pLogStore); + sTrace("logStoreSimpleLog | len:%lu | %s", strlen(serialized), serialized); + free(serialized); +} + +void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore) { + char* serialized = logStoreSimple2Str(pLogStore); + sTrace("logStoreSimpleLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + free(serialized); +} \ No newline at end of file diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 0bb701fc09..943b268cd3 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -49,6 +49,10 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { assert(pSyncNode->state == TAOS_SYNC_STATE_LEADER); + syncIndexMgrLog2("==syncNodeAppendEntriesPeers== pNextIndex", pSyncNode->pNextIndex); + syncIndexMgrLog2("==syncNodeAppendEntriesPeers== pMatchIndex", pSyncNode->pMatchIndex); + logStoreSimpleLog2("==syncNodeAppendEntriesPeers==", pSyncNode->pLogStore); + int32_t ret = 0; for (int i = 0; i < pSyncNode->peersNum; ++i) { SRaftId* pDestId = &(pSyncNode->peersId[i]); @@ -99,6 +103,8 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { pMsg->prevLogTerm = preLogTerm; pMsg->commitIndex = pSyncNode->commitIndex; + syncAppendEntriesLog2("==syncNodeAppendEntriesPeers==", pMsg); + // send AppendEntries syncNodeAppendEntries(pSyncNode, pDestId, pMsg); syncAppendEntriesDestroy(pMsg); diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index f02ba29218..dcf380e7e4 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -29,10 +29,14 @@ add_executable(syncPingTimerTest2 "") add_executable(syncPingSelfTest "") add_executable(syncElectTest "") add_executable(syncElectTest2 "") +add_executable(syncElectTest3 "") add_executable(syncEncodeTest "") add_executable(syncWriteTest "") add_executable(syncReplicateTest "") +add_executable(syncReplicateTest2 "") +add_executable(syncReplicateLoadTest "") add_executable(syncRefTest "") +add_executable(syncLogStoreCheck "") target_sources(syncTest @@ -159,6 +163,10 @@ target_sources(syncElectTest2 PRIVATE "syncElectTest2.cpp" ) +target_sources(syncElectTest3 + PRIVATE + "syncElectTest3.cpp" +) target_sources(syncEncodeTest PRIVATE "syncEncodeTest.cpp" @@ -171,10 +179,22 @@ target_sources(syncReplicateTest PRIVATE "syncReplicateTest.cpp" ) +target_sources(syncReplicateTest2 + PRIVATE + "syncReplicateTest2.cpp" +) +target_sources(syncReplicateLoadTest + PRIVATE + "syncReplicateLoadTest.cpp" +) target_sources(syncRefTest PRIVATE "syncRefTest.cpp" ) +target_sources(syncLogStoreCheck + PRIVATE + "syncLogStoreCheck.cpp" +) target_include_directories(syncTest @@ -332,6 +352,11 @@ target_include_directories(syncElectTest2 "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncElectTest3 + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_include_directories(syncEncodeTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" @@ -347,11 +372,26 @@ target_include_directories(syncReplicateTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncReplicateTest2 + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncReplicateLoadTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_include_directories(syncRefTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncLogStoreCheck + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -478,6 +518,10 @@ target_link_libraries(syncElectTest2 sync gtest_main ) +target_link_libraries(syncElectTest3 + sync + gtest_main +) target_link_libraries(syncEncodeTest sync gtest_main @@ -490,10 +534,22 @@ target_link_libraries(syncReplicateTest sync gtest_main ) +target_link_libraries(syncReplicateTest2 + sync + gtest_main +) +target_link_libraries(syncReplicateLoadTest + sync + gtest_main +) target_link_libraries(syncRefTest sync gtest_main ) +target_link_libraries(syncLogStoreCheck + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncElectTest3.cpp b/source/libs/sync/test/syncElectTest3.cpp new file mode 100644 index 0000000000..45c5488c60 --- /dev/null +++ b/source/libs/sync/test/syncElectTest3.cpp @@ -0,0 +1,138 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "tref.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SWal* pWal; + +int64_t syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "./elect3_test_%d", myIndex); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + + char tmpdir[128]; + snprintf(tmpdir, sizeof(tmpdir), "./elect3_test_wal_%d", myIndex); + pWal = walOpen(tmpdir, &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + int64_t rid = syncStart(&syncInfo); + assert(rid > 0); + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + pSyncNode->hbBaseLine = 500; + pSyncNode->electBaseLine = 1500; + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->pSyncNode = pSyncNode; + + syncNodeRelease(pSyncNode); + + return rid; +} + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +int main(int argc, char** argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncInit(); + assert(ret == 0); + + int64_t rid = syncNodeInit(); + assert(rid > 0); + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + syncNodePrint2((char*)"", pSyncNode); + initRaftId(pSyncNode); + + //--------------------------- + while (1) { + sTrace( + "elect sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d", + pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS); + taosMsleep(1000); + } + + syncNodeRelease(pSyncNode); + + return 0; +} diff --git a/source/libs/sync/test/syncLogStoreCheck.cpp b/source/libs/sync/test/syncLogStoreCheck.cpp new file mode 100644 index 0000000000..85e39a61c5 --- /dev/null +++ b/source/libs/sync/test/syncLogStoreCheck.cpp @@ -0,0 +1,105 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 1; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SWal* pWal; +SSyncNode* pSyncNode; + +SSyncNode* syncNodeInit(const char* path) { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./log_check"); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + pWal = walOpen(path, &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +SSyncNode* logStoreCheck(const char* path) { return syncNodeInit(path); } + +int main(int argc, char** argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + pSyncNode = logStoreCheck(argv[1]); + assert(pSyncNode != NULL); + + logStorePrint2((char*)"logStoreCheck", pSyncNode->pLogStore); + + return 0; +} diff --git a/source/libs/sync/test/syncReplicateLoadTest.cpp b/source/libs/sync/test/syncReplicateLoadTest.cpp new file mode 100644 index 0000000000..d53ceca473 --- /dev/null +++ b/source/libs/sync/test/syncReplicateLoadTest.cpp @@ -0,0 +1,188 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftEntry.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM *pFsm; +SWal * pWal; + +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, index, isWeak, + code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void initFsm() { + pFsm = (SSyncFSM *)malloc(sizeof(SSyncFSM)); + pFsm->FpCommitCb = CommitCb; + pFsm->FpPreCommitCb = PreCommitCb; + pFsm->FpRollBackCb = RollBackCb; +} + +int64_t syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "./replicate2_test_%d", myIndex); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + + char tmpdir[128]; + snprintf(tmpdir, sizeof(tmpdir), "./replicate2_test_wal_%d", myIndex); + pWal = walOpen(tmpdir, &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + int64_t rid = syncStart(&syncInfo); + assert(rid > 0); + + SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + // pSyncNode->hbBaseLine = 500; + // pSyncNode->electBaseLine = 1500; + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; + gSyncIO->pSyncNode = pSyncNode; + + syncNodeRelease(pSyncNode); + + return rid; +} + +void initRaftId(SSyncNode *pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char *s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +SRpcMsg *step0(int i) { + SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + memset(pMsg, 0, sizeof(SRpcMsg)); + pMsg->msgType = 9999; + pMsg->contLen = 128; + pMsg->pCont = malloc(pMsg->contLen); + snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i); + return pMsg; +} + +SyncClientRequest *step1(const SRpcMsg *pMsg) { + SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true); + return pRetMsg; +} + +int main(int argc, char **argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + void logTest(); + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + initFsm(); + + ret = syncInit(); + assert(ret == 0); + + int64_t rid = syncNodeInit(); + assert(rid > 0); + + SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + syncNodePrint2((char *)"", pSyncNode); + initRaftId(pSyncNode); + + // only load ... + + while (1) { + sTrace( + "replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " + "electTimerMS:%d", + pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS); + taosMsleep(1000); + } + + return 0; +} diff --git a/source/libs/sync/test/syncReplicateTest.cpp b/source/libs/sync/test/syncReplicateTest.cpp index 4d6e6f3a25..47399eeb3c 100644 --- a/source/libs/sync/test/syncReplicateTest.cpp +++ b/source/libs/sync/test/syncReplicateTest.cpp @@ -28,19 +28,29 @@ SSyncFSM * pFsm; SWal * pWal; SSyncNode *gSyncNode; -void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) { - printf("==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code); - syncRpcMsgPrint2((char *)"==CommitCb==", (SRpcMsg *)pBuf); +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); } -void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) { - printf("==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code); - syncRpcMsgPrint2((char *)"==PreCommitCb==", (SRpcMsg *)pBuf); +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, index, isWeak, + code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); } -void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) { - printf("==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code); - syncRpcMsgPrint2((char *)"==RollBackCb==", (SRpcMsg *)pBuf); +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); } void initFsm() { diff --git a/source/libs/sync/test/syncReplicateTest2.cpp b/source/libs/sync/test/syncReplicateTest2.cpp new file mode 100644 index 0000000000..09dbc0e2ed --- /dev/null +++ b/source/libs/sync/test/syncReplicateTest2.cpp @@ -0,0 +1,203 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftEntry.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM *pFsm; +SWal * pWal; + +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, index, isWeak, + code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void initFsm() { + pFsm = (SSyncFSM *)malloc(sizeof(SSyncFSM)); + pFsm->FpCommitCb = CommitCb; + pFsm->FpPreCommitCb = PreCommitCb; + pFsm->FpRollBackCb = RollBackCb; +} + +int64_t syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "./replicate2_test_%d", myIndex); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + + char tmpdir[128]; + snprintf(tmpdir, sizeof(tmpdir), "./replicate2_test_wal_%d", myIndex); + pWal = walOpen(tmpdir, &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + int64_t rid = syncStart(&syncInfo); + assert(rid > 0); + + SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + // pSyncNode->hbBaseLine = 500; + // pSyncNode->electBaseLine = 1500; + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; + gSyncIO->pSyncNode = pSyncNode; + + syncNodeRelease(pSyncNode); + + return rid; +} + +void initRaftId(SSyncNode *pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char *s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +SRpcMsg *step0(int i) { + SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + memset(pMsg, 0, sizeof(SRpcMsg)); + pMsg->msgType = 9999; + pMsg->contLen = 128; + pMsg->pCont = malloc(pMsg->contLen); + snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i); + return pMsg; +} + +SyncClientRequest *step1(const SRpcMsg *pMsg) { + SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true); + return pRetMsg; +} + +int main(int argc, char **argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + void logTest(); + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + initFsm(); + + ret = syncInit(); + assert(ret == 0); + + int64_t rid = syncNodeInit(); + assert(rid > 0); + + SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + syncNodePrint2((char *)"", pSyncNode); + initRaftId(pSyncNode); + + for (int i = 0; i < 30; ++i) { + // step0 + SRpcMsg *pMsg0 = step0(i); + syncRpcMsgPrint2((char *)"==step0==", pMsg0); + + syncPropose(rid, pMsg0, true); + taosMsleep(1000); + + free(pMsg0); + + sTrace( + "syncPropose sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " + "electTimerMS:%d", + pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS); + } + + while (1) { + sTrace( + "replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " + "electTimerMS:%d", + pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS); + taosMsleep(1000); + } + + return 0; +} diff --git a/source/libs/sync/test/syncWriteTest.cpp b/source/libs/sync/test/syncWriteTest.cpp index 2598abbddd..732bcf140b 100644 --- a/source/libs/sync/test/syncWriteTest.cpp +++ b/source/libs/sync/test/syncWriteTest.cpp @@ -28,19 +28,29 @@ SSyncFSM * pFsm; SWal * pWal; SSyncNode *gSyncNode; -void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) { - printf("==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code); - syncRpcMsgPrint2((char *)"==CommitCb==", (SRpcMsg *)pBuf); +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); } -void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) { - printf("==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code); - syncRpcMsgPrint2((char *)"==PreCommitCb==", (SRpcMsg *)pBuf); +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, index, isWeak, + code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); } -void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) { - printf("==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code); - syncRpcMsgPrint2((char *)"==RollBackCb==", (SRpcMsg *)pBuf); +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); } void initFsm() { diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 0e6abb5785..f693ad74de 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -47,9 +47,6 @@ #endif #endif -typedef int32_t SocketFd; -typedef SocketFd EpollFd; - typedef struct TdSocketServer { #if SOCKET_WITH_LOCK TdThreadRwlock rwlock; @@ -58,14 +55,6 @@ typedef struct TdSocketServer { SocketFd fd; } *TdSocketServerPtr, TdSocketServer; -typedef struct TdSocket { -#if SOCKET_WITH_LOCK - TdThreadRwlock rwlock; -#endif - int refId; - SocketFd fd; -} *TdSocketPtr, TdSocket; - typedef struct TdEpoll { #if SOCKET_WITH_LOCK TdThreadRwlock rwlock; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index b7ab007cf5..005995c8e2 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -270,6 +270,11 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CANT_PARALLEL, "Invalid stage to kill // mnode-topic TAOS_DEFINE_ERROR(TSDB_CODE_MND_UNSUPPORTED_TOPIC, "Topic with aggregation is unsupported") +// mnode-sma +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_ALREADY_EXIST, "SMA already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_NOT_EXIST, "SMA does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SMA_OPTION, "Invalid sma option") + // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_DND_OFFLINE, "Dnode is offline") @@ -305,6 +310,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, "Database write operat TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_SYNCING, "Database is syncing") TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_TSDB_STATE, "Invalid tsdb state") TAOS_DEFINE_ERROR(TSDB_CODE_VND_TB_NOT_EXIST, "Table not exists") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_SMA_NOT_EXIST, "SMA not exists") TAOS_DEFINE_ERROR(TSDB_CODE_VND_HASH_MISMATCH, "Hash value mismatch") // tsdb @@ -332,8 +338,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_MESSED_MSG, "TSDB messed message") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVLD_TAG_VAL, "TSDB invalid tag value") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_CACHE_LAST_ROW, "TSDB no cache last row data") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECREATED, "Table re-created") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_SMA_INDEX_IN_META, "No sma index in meta") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR, "TDB env open error") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_SMA_INDEX_IN_META, "No sma index in meta") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_SMA_STAT, "Invalid sma state") + // query TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, "Invalid handle") diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 821789de0a..c526bfe5c6 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -10,18 +10,20 @@ ./test.sh -f tsim/db/basic7.sim ./test.sh -f tsim/db/error1.sim -# ---- table -./test.sh -f tsim/table/basic1.sim - # ---- dnode ./test.sh -f tsim/dnode/basic1.sim # ---- insert -#./test.sh -f tsim/insert/basic0.sim +./test.sh -f tsim/insert/basic0.sim +./test.sh -f tsim/insert/null.sim # ---- query ./test.sh -f tsim/query/interval.sim +# ---- table +./test.sh -f tsim/table/basic1.sim + # ---- tmq ./test.sh -f tsim/tmq/basic.sim + #======================b1-end=============== diff --git a/tests/script/tsim/insert/basic0.sim b/tests/script/tsim/insert/basic0.sim index 8592624cf3..46aa99127b 100644 --- a/tests/script/tsim/insert/basic0.sim +++ b/tests/script/tsim/insert/basic0.sim @@ -36,14 +36,14 @@ endi print =============== insert data, mode1: one row one table in sql print =============== insert data, mode1: mulit rows one table in sql -print =============== insert data, mode1: one rows mulit table in sql -print =============== insert data, mode1: mulit rows mulit table in sql +#print =============== insert data, mode1: one rows mulit table in sql +#print =============== insert data, mode1: mulit rows mulit table in sql sql insert into ct1 values(now+0s, 10, 2.0, 3.0) -sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 13, 2.3, 3.3) +sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) sql insert into ct2 values(now+0s, 10, 2.0, 3.0) -sql insert into ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 13, 2.3, 3.3) -sql insert into ct1 values(now+4s, -14, -2.4, -3.4) ct2 values(now+4s, -14, -2.4, -3.4) -sql insert into ct1 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) ct2 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) +sql insert into ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) +#sql insert into ct1 values(now+4s, -14, -2.4, -3.4) ct2 values(now+4s, -14, -2.4, -3.4) +#sql insert into ct1 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) ct2 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) sql insert into ct3 values('2021-01-01 00:00:00.000', 10, 2.0, 3.0) @@ -52,14 +52,11 @@ sql insert into ct3 values('2021-01-01 00:00:00.000', 10, 2.0, 3.0) print =============== query data from child table sql select * from ct1 print rows: $rows -print $data00 $data01 -print $data10 $data11 -print $data20 $data21 -print $data30 $data31 -print $data40 $data41 -print $data50 $data51 -print $data60 $data61 -if $rows != 7 then +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +if $rows != 4 then return -1 endi if $data01 != 10 then @@ -84,28 +81,29 @@ endi print =============== select count(*) from child table sql select count(*) from ct1 +print rows: $rows +print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi - -print $data00 $data01 $data02 -if $data00 != 7 then +if $data00 != 4 then return -1 endi print =============== select count(column) from child table sql select count(ts), count(c1), count(c2), count(c3) from ct1 -print $data00 $data01 $data02 $data03 -if $data00 != 7 then +print rows: $rows +print $data00 $data01 $data02 $data03 +if $data00 != 4 then return -1 endi -if $data01 != 7 then +if $data01 != 4 then return -1 endi -if $data02 != 7 then +if $data02 != 4 then return -1 endi -if $data03 != 7 then +if $data03 != 4 then return -1 endi @@ -115,17 +113,18 @@ endi print =============== select min(column) from child table sql select min(c1), min(c2), min(c3) from ct1 -print $data00 $data01 $data02 $data03 +print rows: $rows +print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != -16 then +if $data00 != -13 then return -1 endi -if $data01 != -2.60000 then +if $data01 != -2.30000 then return -1 endi -if $data02 != -3.600000000 then +if $data02 != -3.300000000 then return -1 endi @@ -135,13 +134,13 @@ print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != 13 then +if $data00 != 11 then return -1 endi -if $data01 != 2.30000 then +if $data01 != 2.10000 then return -1 endi -if $data02 != 3.300000000 then +if $data02 != 3.100000000 then return -1 endi @@ -151,49 +150,53 @@ print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != 1 then +if $data00 != -4 then return -1 endi -if $data01 != 1.099999905 then +if $data01 != -0.400000095 then return -1 endi -if $data02 != 2.100000000 then +if $data02 != -0.400000000 then return -1 endi -print =============== select column, from child table +print =============== select column without timestamp, from child table sql select c1, c2, c3 from ct1 +print rows: $rows print $data00 $data01 $data02 -#if $rows != 4 then -# return -1 -#endi -#if $data00 != 10 then -# return -1 -#endi -#if $data01 != 2.00000 then -# return -1 -#endi -#if $data02 != 3.000000000 then -# return -1 -#endi -#if $data10 != 11 then -# return -1 -#endi -#if $data11 != 2.10000 then -# return -1 -#endi -#if $data12 != 3.100000000 then -# return -1 -#endi -#if $data30 != 13 then -# return -1 -#endi -#if $data31 != 2.30000 then -# return -1 -#endi -#if $data32 != 3.300000000 then -# return -1 -#endi +print $data10 $data11 $data12 +print $data20 $data21 $data22 +print $data30 $data31 $data32 +if $rows != 4 then + return -1 +endi +if $data00 != 10 then + return -1 +endi +if $data01 != 2.00000 then + return -1 +endi +if $data02 != 3.000000000 then + return -1 +endi +if $data10 != 11 then + return -1 +endi +if $data11 != 2.10000 then + return -1 +endi +if $data12 != 3.100000000 then + return -1 +endi +if $data30 != -13 then + return -1 +endi +if $data31 != -2.30000 then + return -1 +endi +if $data32 != -3.300000000 then + return -1 +endi #=================================================================== #=================================================================== @@ -202,17 +205,17 @@ print $data00 $data01 $data02 #if $rows != 4 then # return -1 #endi + #print =============== select count(*) from supter table #sql select count(*) from stb +#print $data00 $data01 $data02 #if $rows != 1 then # return -1 #endi -# -#print $data00 $data01 $data02 -#if $data00 != 8 then +#if $data00 != 9 then # return -1 #endi -# + #print =============== select count(column) from supter table #sql select count(ts), count(c1), count(c2), count(c3) from stb #print $data00 $data01 $data02 $data03 @@ -237,9 +240,31 @@ print =============== stop and restart taosd, then again do query above system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -sleep 2000 +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +print =============== query data from child table sql select * from ct1 -if $rows != 7 then +print rows: $rows +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +if $rows != 4 then return -1 endi if $data01 != 10 then @@ -264,28 +289,29 @@ endi print =============== select count(*) from child table sql select count(*) from ct1 +print rows: $rows +print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi - -print $data00 $data01 $data02 -if $data00 != 7 then +if $data00 != 4 then return -1 endi print =============== select count(column) from child table sql select count(ts), count(c1), count(c2), count(c3) from ct1 -print $data00 $data01 $data02 $data03 -if $data00 != 7 then +print rows: $rows +print $data00 $data01 $data02 $data03 +if $data00 != 4 then return -1 endi -if $data01 != 7 then +if $data01 != 4 then return -1 endi -if $data02 != 7 then +if $data02 != 4 then return -1 endi -if $data03 != 7 then +if $data03 != 4 then return -1 endi @@ -295,17 +321,18 @@ endi print =============== select min(column) from child table sql select min(c1), min(c2), min(c3) from ct1 -print $data00 $data01 $data02 $data03 +print rows: $rows +print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != -16 then +if $data00 != -13 then return -1 endi -if $data01 != -2.60000 then +if $data01 != -2.30000 then return -1 endi -if $data02 != -3.600000000 then +if $data02 != -3.300000000 then return -1 endi @@ -315,13 +342,13 @@ print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != 13 then +if $data00 != 11 then return -1 endi -if $data01 != 2.30000 then +if $data01 != 2.10000 then return -1 endi -if $data02 != 3.300000000 then +if $data02 != 3.100000000 then return -1 endi @@ -331,14 +358,87 @@ print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != 1 then +if $data00 != -4 then return -1 endi -if $data01 != 1.099999905 then +if $data01 != -0.400000095 then return -1 endi -if $data02 != 2.100000000 then +if $data02 != -0.400000000 then return -1 endi +print =============== select column without timestamp, from child table +sql select c1, c2, c3 from ct1 +print rows: $rows +print $data00 $data01 $data02 +print $data10 $data11 $data12 +print $data20 $data21 $data22 +print $data30 $data31 $data32 +if $rows != 4 then + return -1 +endi +if $data00 != 10 then + return -1 +endi +if $data01 != 2.00000 then + return -1 +endi +if $data02 != 3.000000000 then + return -1 +endi +if $data10 != 11 then + return -1 +endi +if $data11 != 2.10000 then + return -1 +endi +if $data12 != 3.100000000 then + return -1 +endi +if $data30 != -13 then + return -1 +endi +if $data31 != -2.30000 then + return -1 +endi +if $data32 != -3.300000000 then + return -1 +endi +#=================================================================== +#=================================================================== + +#print =============== query data from stb +#sql select * from stb +#if $rows != 4 then +# return -1 +#endi + +#print =============== select count(*) from supter table +#sql select count(*) from stb +#print $data00 $data01 $data02 +#if $rows != 1 then +# return -1 +#endi +#if $data00 != 9 then +# return -1 +#endi + +#print =============== select count(column) from supter table +#sql select count(ts), count(c1), count(c2), count(c3) from stb +#print $data00 $data01 $data02 $data03 +#if $data00 != 8 then +# return -1 +#endi +#if $data01 != 8 then +# return -1 +#endi +#if $data02 != 8 then +# return -1 +#endi +#if $data03 != 8 then +# return -1 +#endi + + #system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/insert/null.sim b/tests/script/tsim/insert/null.sim index 9dcc435486..fbaef8cc94 100644 --- a/tests/script/tsim/insert/null.sim +++ b/tests/script/tsim/insert/null.sim @@ -7,7 +7,7 @@ sql connect print =============== create database sql create database d0 sql show databases -if $rows != 1 then +if $rows != 2 then return -1 endi @@ -40,7 +40,7 @@ sql insert into ct1 values (now+7s, 14, NULL, 3.5, 95) sql insert into ct1 values (now+8s, 15, 2.5, NULL, 96) sql insert into ct1 values (now+9s, 16, 2.6, 3.6, NULL) sql insert into ct1 values (now+10s, NULL, NULL, NULL, NULL) -sql insert into ct1 values (now+11s, -2147483648, 2.7, 3.7, 97) +sql insert into ct1 values (now+11s, -2147483647, 2.7, 3.7, 97) #=================================================================== #=================================================================== @@ -75,7 +75,6 @@ endi # return -1 #endi - print =============== select count(*) from child table sql select count(*) from ct1 print ===> select count(*) from ct1 @@ -84,9 +83,7 @@ print ===> rows0: $data00 $data01 $data02 $data03 $data04 if $rows != 1 then return -1 endi - -print $data00 $data01 $data02 -if $data00 != 4 then +if $data00 != 12 then return -1 endi @@ -95,17 +92,16 @@ sql select count(ts), count(c1), count(c2), count(c3) from ct1 print ===> select count(ts), count(c1), count(c2), count(c3) from ct1 print ===> rows: $rows print ===> rows0: $data00 $data01 $data02 $data03 $data04 - -if $data00 != 4 then +if $data00 != 12 then return -1 endi -if $data01 != 4 then +if $data01 != 8 then return -1 endi -if $data02 != 4 then +if $data02 != 8 then return -1 endi -if $data03 != 4 then +if $data03 != 8 then return -1 endi @@ -121,7 +117,7 @@ print ===> rows0: $data00 $data01 $data02 $data03 $data04 if $rows != 1 then return -1 endi -if $data00 != 10 then +if $data00 != -2147483647 then return -1 endi if $data01 != 2.00000 then @@ -139,13 +135,13 @@ print ===> rows0: $data00 $data01 $data02 $data03 $data04 if $rows != 1 then return -1 endi -if $data00 != 13 then +if $data00 != 16 then return -1 endi -if $data01 != 2.30000 then +if $data01 != 2.70000 then return -1 endi -if $data02 != 3.300000000 then +if $data02 != 3.700000000 then return -1 endi @@ -157,13 +153,13 @@ print ===> rows0: $data00 $data01 $data02 $data03 $data04 if $rows != 1 then return -1 endi -if $data00 != 46 then +if $data00 != -2147483556 then return -1 endi -if $data01 != 8.599999905 then +if $data01 != 18.799999952 then return -1 endi -if $data02 != 12.600000000 then +if $data02 != 26.800000000 then return -1 endi @@ -172,42 +168,48 @@ sql select c1, c2, c3 from ct1 print ===> select c1, c2, c3 from ct1 print ===> rows: $rows print ===> rows0: $data00 $data01 $data02 $data03 $data04 -#if $rows != 4 then -# return -1 -#endi -#if $data00 != 10 then -# return -1 -#endi -#if $data01 != 2.00000 then -# return -1 -#endi -#if $data02 != 3.000000000 then -# return -1 -#endi -#if $data10 != 11 then -# return -1 -#endi -#if $data11 != 2.10000 then -# return -1 -#endi -#if $data12 != 3.100000000 then -# return -1 -#endi -#if $data30 != 13 then -# return -1 -#endi -#if $data31 != 2.30000 then -# return -1 -#endi -#if $data32 != 3.300000000 then -# return -1 -#endi +if $rows != 12 then + return -1 +endi +if $data00 != 10 then + return -1 +endi +if $data01 != 2.00000 then + return -1 +endi +if $data02 != 3.000000000 then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data11 != NULL then + return -1 +endi +if $data12 != NULL then + return -1 +endi +if $data30 != 11 then + return -1 +endi +if $data31 != NULL then + return -1 +endi +if $data32 != 3.200000000 then + return -1 +endi +if $data90 != 16 then + return -1 +endi +if $data91 != 2.60000 then + return -1 +endi +if $data92 != 3.600000000 then + return -1 +endi #=================================================================== #=================================================================== - -return - #print =============== query data from stb #sql select * from stb #print ===> @@ -218,19 +220,18 @@ return #endi #print =============== select count(*) from supter table #sql select count(*) from stb +#print $data00 $data01 $data02 #if $rows != 1 then # return -1 #endi -# -#print $data00 $data01 $data02 -#if $data00 != 8 then +#if $data00 != 12 then # return -1 #endi -# + #print =============== select count(column) from supter table #sql select count(ts), count(c1), count(c2), count(c3) from stb #print $data00 $data01 $data02 $data03 -#if $data00 != 8 then +#if $data00 != 12 then # return -1 #endi #if $data01 != 8 then @@ -243,7 +244,6 @@ return # return -1 #endi - #=================================================================== #=================================================================== @@ -251,9 +251,36 @@ print =============== stop and restart taosd, then again do query above system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -sleep 2000 +print ===> waiting dnode ready +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +#=================================================================== +#=================================================================== +print =============== query data from child table sql select * from ct1 -if $rows != 4 then # after fix bug, modify 4 to 7 +print ===> select * from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +if $rows != 12 then return -1 endi if $data01 != 10 then @@ -275,31 +302,33 @@ endi # return -1 #endi - print =============== select count(*) from child table sql select count(*) from ct1 +print ===> select count(*) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 if $rows != 1 then return -1 endi - -print $data00 $data01 $data02 -if $data00 != 4 then +if $data00 != 12 then return -1 endi print =============== select count(column) from child table sql select count(ts), count(c1), count(c2), count(c3) from ct1 -print $data00 $data01 $data02 $data03 -if $data00 != 4 then +print ===> select count(ts), count(c1), count(c2), count(c3) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +if $data00 != 12 then return -1 endi -if $data01 != 4 then +if $data01 != 8 then return -1 endi -if $data02 != 4 then +if $data02 != 8 then return -1 endi -if $data03 != 4 then +if $data03 != 8 then return -1 endi @@ -309,11 +338,13 @@ endi print =============== select min(column) from child table sql select min(c1), min(c2), min(c3) from ct1 -print $data00 $data01 $data02 $data03 +print ===> select min(c1), min(c2), min(c3) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 if $rows != 1 then return -1 endi -if $data00 != 10 then +if $data00 != -2147483647 then return -1 endi if $data01 != 2.00000 then @@ -325,34 +356,119 @@ endi print =============== select max(column) from child table sql select max(c1), max(c2), max(c3) from ct1 -print $data00 $data01 $data02 $data03 +print ===> select max(c1), max(c2), max(c3) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 if $rows != 1 then return -1 endi -if $data00 != 13 then +if $data00 != 16 then return -1 endi -if $data01 != 2.30000 then +if $data01 != 2.70000 then return -1 endi -if $data02 != 3.300000000 then +if $data02 != 3.700000000 then return -1 endi print =============== select sum(column) from child table sql select sum(c1), sum(c2), sum(c3) from ct1 -print $data00 $data01 $data02 $data03 +print ===> select sum(c1), sum(c2), sum(c3) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 if $rows != 1 then return -1 endi -if $data00 != 46 then +if $data00 != -2147483556 then return -1 endi -if $data01 != 8.599999905 then +if $data01 != 18.799999952 then return -1 endi -if $data02 != 12.600000000 then +if $data02 != 26.800000000 then return -1 endi +print =============== select column, from child table +sql select c1, c2, c3 from ct1 +print ===> select c1, c2, c3 from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +if $rows != 12 then + return -1 +endi +if $data00 != 10 then + return -1 +endi +if $data01 != 2.00000 then + return -1 +endi +if $data02 != 3.000000000 then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data11 != NULL then + return -1 +endi +if $data12 != NULL then + return -1 +endi +if $data30 != 11 then + return -1 +endi +if $data31 != NULL then + return -1 +endi +if $data32 != 3.200000000 then + return -1 +endi +if $data90 != 16 then + return -1 +endi +if $data91 != 2.60000 then + return -1 +endi +if $data92 != 3.600000000 then + return -1 +endi +#=================================================================== +#=================================================================== + +#print =============== query data from stb +#sql select * from stb +#print ===> +#print ===> rows: $rows +#print ===> rows0: $data00 $data01 $data02 $data03 $data04 +#if $rows != 4 then +# return -1 +#endi +#print =============== select count(*) from supter table +#sql select count(*) from stb +#print $data00 $data01 $data02 +#if $rows != 1 then +# return -1 +#endi +#if $data00 != 12 then +# return -1 +#endi + +#print =============== select count(column) from supter table +#sql select count(ts), count(c1), count(c2), count(c3) from stb +#print $data00 $data01 $data02 $data03 +#if $data00 != 12 then +# return -1 +#endi +#if $data01 != 8 then +# return -1 +#endi +#if $data02 != 8 then +# return -1 +#endi +#if $data03 != 8 then +# return -1 +#endi + #system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/table/basic1.sim b/tests/script/tsim/table/basic1.sim index a7f45ece08..e7fdf54983 100644 --- a/tests/script/tsim/table/basic1.sim +++ b/tests/script/tsim/table/basic1.sim @@ -3,10 +3,33 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect +#=========== TD-14042 start +sql create database db1; +sql create database db2; +sql create database db3; + +sql use db1; +sql create table st1 (ts timestamp, i int) tags (j int); +sql create table tb1 using st1 tags(1); + +sql use db2; +sql create table st2 (ts timestamp, i int) tags (j int); +sql create table tb2 using st2 tags(1); + +sql use db3; +sql create table st3 (ts timestamp, i int) tags (j int); +sql create table tb3 using st3 tags(1); + +sql show tables +if $rows != 1 then + return -1 +endi +#=========== TD-14042 end + print =============== create database sql create database d1 sql show databases -if $rows != 2 then +if $rows != 5 then return -1 endi @@ -186,10 +209,26 @@ if $rows != 21 then return -1 endi -#system sh/exec.sh -n dnode1 -s stop -x SIGINT -#system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi -sleep 2000 print =============== query data sql select * from c1 if $rows != 3 then diff --git a/tests/script/tsim/testCaseSuite.sim b/tests/script/tsim/testCaseSuite.sim index 88f1911a2b..bca0204e63 100644 --- a/tests/script/tsim/testCaseSuite.sim +++ b/tests/script/tsim/testCaseSuite.sim @@ -1,4 +1,6 @@ +run tsim/user/basic1.sim + run tsim/db/basic1.sim run tsim/db/basic6.sim run tsim/db/basic7.sim @@ -7,12 +9,14 @@ run tsim/db/error1.sim run tsim/dnode/basic1.sim run tsim/insert/basic0.sim -run tsim/insert/basic1.sim -run tsim/insert/null.sim +#run tsim/insert/basic1.sim # TD-14246 +#run tsim/insert/backquote.sim # TD-14261 +#run tsim/insert/null.sim -run tsim/query/interval-offset.sim run tsim/query/interval.sim +#run tsim/query/interval-offset.sim # TD-14266 run tsim/table/basic1.sim -run tsim/user/basic1.sim +run tsim/tmq/basic.sim + diff --git a/tests/script/tsim/tmq/basic.sim b/tests/script/tsim/tmq/basic.sim index 31836f8580..3e42c2cbd7 100644 --- a/tests/script/tsim/tmq/basic.sim +++ b/tests/script/tsim/tmq/basic.sim @@ -47,9 +47,12 @@ sql drop database useless_db # -m startTimestamp, default is 1640966400000 [2022-01-01 00:00:00] # -g showMsgFlag, default is 0 # -#system_content ../../debug/tests/test/c/tmq_demo -c ../../sim/tsim/cfg -system ../../debug/tests/test/c/tmq_demo -c ../../sim/tsim/cfg -print result-> $system_content +print cmd===> system_content ../../debug/tests/test/c/tmq_demo -sim 1 -b 100 -c ../../sim/tsim/cfg -w ../../sim/dnode1/data/vnode/vnode4/wal +system_content ../../debug/tests/test/c/tmq_demo -sim 1 -b 100 -c ../../sim/tsim/cfg -w ../../sim/dnode1/data/vnode/vnode4/wal +print cmd result----> $system_content +if $system_content != @{consume success: 100}@ then + print not match in pos000 +endi sql show databases print ===> $rows $data00 $data01 $data02 $data03 @@ -78,4 +81,5 @@ endi if $data00 != 10000 then return -1 endi + #system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 866bcb2ea0..15022f648a 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -58,6 +58,7 @@ typedef struct { int32_t totalRowsOfPerTbl; int64_t startTimestamp; int32_t showMsgFlag; + int32_t simCase; int32_t totalRowsOfT2; } SConfInfo; @@ -66,7 +67,7 @@ static SConfInfo g_stConfInfo = { "tmqdb", "stb", "./tmqResult.txt", // output_file - "/data2/dnode/data/vnodes/vnode2/wal", + "/data2/dnode/data/vnode/vnode2/wal", 1, // threads 1, // tables 1, // vgroups @@ -77,6 +78,7 @@ static SConfInfo g_stConfInfo = { 10000, // total rows for per table 0, // 2020-01-01 00:00:00.000 0, // show consume msg switch + 0, // if run in sim case 10000, }; @@ -117,6 +119,8 @@ static void printHelp() { printf("%s%s%s%" PRId64 "\n", indent, indent, "startTimestamp, default is ", g_stConfInfo.startTimestamp); printf("%s%s\n", indent, "-g"); printf("%s%s%s%d\n", indent, indent, "showMsgFlag, default is ", g_stConfInfo.showMsgFlag); + printf("%s%s\n", indent, "-sim"); + printf("%s%s%s%d\n", indent, indent, "simCase, default is ", g_stConfInfo.simCase); exit(EXIT_SUCCESS); } @@ -160,14 +164,17 @@ void parseArgument(int32_t argc, char *argv[]) { g_stConfInfo.startTimestamp = atol(argv[++i]); } else if (strcmp(argv[i], "-g") == 0) { g_stConfInfo.showMsgFlag = atol(argv[++i]); + } else if (strcmp(argv[i], "-sim") == 0) { + g_stConfInfo.simCase = atol(argv[++i]); } else { - pPrint("%s unknow para: %s %s", GREEN, argv[++i], NC); + printf("%s unknow para: %s %s", GREEN, argv[++i], NC); exit(-1); } } g_stConfInfo.totalRowsOfT2 = g_stConfInfo.totalRowsOfPerTbl * g_stConfInfo.ratio; +#if 0 pPrint("%s configDir:%s %s", GREEN, configDir, NC); pPrint("%s dbName:%s %s", GREEN, g_stConfInfo.dbName, NC); pPrint("%s stbName:%s %s", GREEN, g_stConfInfo.stbName, NC); @@ -184,6 +191,7 @@ void parseArgument(int32_t argc, char *argv[]) { pPrint("%s totalRowsOfT2:%d %s", GREEN, g_stConfInfo.totalRowsOfT2, NC); pPrint("%s startTimestamp:%" PRId64" %s", GREEN, g_stConfInfo.startTimestamp, NC); pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC); +#endif } static int running = 1; @@ -429,15 +437,21 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLog double consumeTime = (double)(endTime - startTime) / 1000000; if (batchCnt != totalMsgs) { - pPrint("%s inserted msgs: %d and consume msgs: %d mismatch %s", GREEN, totalMsgs, batchCnt, NC); + printf("%s inserted msgs: %d and consume msgs: %d mismatch %s", GREEN, totalMsgs, batchCnt, NC); + exit(-1); } - pPrint("consume result: msgs: %d, skip log cnt: %d, time used:%.3f second\n", batchCnt, skipLogNum, consumeTime); + if (0 == g_stConfInfo.simCase) { + printf("consume result: msgs: %d, skip log cnt: %d, time used:%.3f second\n", batchCnt, skipLogNum, consumeTime); + } else { + printf("{consume success: %d}", totalMsgs); + } taosFprintfFile(g_fp, "|%10d | %10.3f | %8.2f | %10.2f| %10.2f |\n", batchCnt, consumeTime, (double)batchCnt / consumeTime, (double)walLogSize / (1024 * 1024.0) / consumeTime, (double)walLogSize / 1024.0 / batchCnt); err = tmq_consumer_close(tmq); if (err) { fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err)); + exit(-1); } } @@ -679,12 +693,17 @@ int main(int32_t argc, char *argv[]) { walLogSize = getDirectorySize(g_stConfInfo.vnodeWalPath); if (walLogSize <= 0) { - pError("vnode2/wal size incorrect!"); + printf("vnode2/wal size incorrect!"); + exit(-1); } else { - pPrint(".log file size in vnode2/wal: %.3f MBytes\n", (double)walLogSize/(1024 * 1024.0)); + if (0 == g_stConfInfo.simCase) { + pPrint(".log file size in vnode2/wal: %.3f MBytes\n", (double)walLogSize/(1024 * 1024.0)); + } } - - pPrint("insert result: %d rows, %d msgs, time:%.3f sec, speed:%.1f rows/second, %.1f msgs/second\n", totalRows, totalMsgs, seconds, rowsSpeed, msgsSpeed); + + if (0 == g_stConfInfo.simCase) { + pPrint("insert result: %d rows, %d msgs, time:%.3f sec, speed:%.1f rows/second, %.1f msgs/second\n", totalRows, totalMsgs, seconds, rowsSpeed, msgsSpeed); + } taosFprintfFile(g_fp, "|%10d | %10.3f | %8.2f | %10.3f ", totalMsgs, seconds, msgsSpeed, (double)walLogSize/(1024 * 1024.0)); } diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index dae0a1c840..f064833691 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -4,6 +4,7 @@ IF (TD_TAOS_TOOLS) INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include/common) INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include/util) INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include/os) + INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include/libs/transport) ADD_SUBDIRECTORY(taos-tools) ENDIF () diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index b351675e47..284693795e 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -4,9 +4,7 @@ add_executable(shell ${SHELL_SRC}) target_link_libraries( shell PUBLIC taos - PUBLIC util - PUBLIC common - PUBLIC os + PRIVATE os common transport util ) target_include_directories( shell diff --git a/tools/shell/inc/syncMsg.h b/tools/shell/inc/syncMsg.h new file mode 100644 index 0000000000..85ac9c78af --- /dev/null +++ b/tools/shell/inc/syncMsg.h @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_SYNC_MSG_H +#define TDENGINE_SYNC_MSG_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "tsync.h" + +typedef enum { + TAOS_SMSG_START = 0, + TAOS_SMSG_SYNC_DATA = 1, + TAOS_SMSG_SYNC_DATA_RSP = 2, + TAOS_SMSG_SYNC_FWD = 3, + TAOS_SMSG_SYNC_FWD_RSP = 4, + TAOS_SMSG_SYNC_REQ = 5, + TAOS_SMSG_SYNC_REQ_RSP = 6, + TAOS_SMSG_SYNC_MUST = 7, + TAOS_SMSG_SYNC_MUST_RSP = 8, + TAOS_SMSG_STATUS = 9, + TAOS_SMSG_STATUS_RSP = 10, + TAOS_SMSG_SETUP = 11, + TAOS_SMSG_SETUP_RSP = 12, + TAOS_SMSG_SYNC_FILE = 13, + TAOS_SMSG_SYNC_FILE_RSP = 14, + TAOS_SMSG_TEST = 15, + TAOS_SMSG_END = 16 +} ESyncMsgType; + +typedef enum { + SYNC_STATUS_BROADCAST, + SYNC_STATUS_BROADCAST_RSP, + SYNC_STATUS_SETUP_CONN, + SYNC_STATUS_SETUP_CONN_RSP, + SYNC_STATUS_EXCHANGE_DATA, + SYNC_STATUS_EXCHANGE_DATA_RSP, + SYNC_STATUS_CHECK_ROLE, + SYNC_STATUS_CHECK_ROLE_RSP +} ESyncStatusType; + +#pragma pack(push, 1) + +typedef struct { + int8_t type; // msg type + int8_t protocol; // protocol version + uint16_t signature; // fixed value + int32_t code; // + int32_t cId; // cluster Id + int32_t vgId; // vg ID + int32_t len; // content length, does not include head + uint32_t cksum; +} SSyncHead; + +typedef struct { + SSyncHead head; + uint16_t port; + uint16_t tranId; + int32_t sourceId; // only for arbitrator + char fqdn[TSDB_FQDN_LEN]; +} SSyncMsg; + +typedef struct { + SSyncHead head; + int8_t sync; + int8_t reserved; + uint16_t tranId; + int8_t reserverd[4]; +} SSyncRsp; + +typedef struct { + int8_t role; + uint64_t version; +} SPeerStatus; + +typedef struct { + SSyncHead head; + int8_t role; + int8_t ack; + int8_t type; + int8_t reserved[3]; + uint16_t tranId; + uint64_t version; + SPeerStatus peersStatus[TAOS_SYNC_MAX_REPLICA]; +} SPeersStatus; + +typedef struct { + SSyncHead head; + uint64_t fversion; +} SFileVersion; + +typedef struct { + SSyncHead head; + int8_t ack; +} SFileAck; + +typedef struct { + SSyncHead head; + uint64_t version; + int32_t code; +} SFwdRsp; + +#pragma pack(pop) + +#define SYNC_PROTOCOL_VERSION 1 +#define SYNC_SIGNATURE ((uint16_t)(0xCDEF)) + +extern char *statusType[]; + +uint16_t syncGenTranId(); +int32_t syncCheckHead(SSyncHead *pHead); + +void syncBuildSyncFwdMsg(SSyncHead *pHead, int32_t vgId, int32_t len); +void syncBuildSyncFwdRsp(SFwdRsp *pMsg, int32_t vgId, uint64_t version, int32_t code); +void syncBuildSyncReqMsg(SSyncMsg *pMsg, int32_t vgId); +void syncBuildSyncDataMsg(SSyncMsg *pMsg, int32_t vgId); +void syncBuildSyncSetupMsg(SSyncMsg *pMsg, int32_t vgId); +void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId); +void syncBuildSyncTestMsg(SSyncMsg *pMsg, int32_t vgId); + +void syncBuildFileAck(SFileAck *pMsg, int32_t vgId); +void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_VNODEPEER_H diff --git a/tools/shell/inc/tsync.h b/tools/shell/inc/tsync.h new file mode 100644 index 0000000000..d1b68e3f5a --- /dev/null +++ b/tools/shell/inc/tsync.h @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_SYNC_H +#define TDENGINE_SYNC_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define TAOS_SYNC_MAX_REPLICA 5 +#define TAOS_SYNC_MAX_INDEX 0x7FFFFFFF + +typedef enum { + TAOS_SYNC_ROLE_OFFLINE = 0, + TAOS_SYNC_ROLE_UNSYNCED = 1, + TAOS_SYNC_ROLE_SYNCING = 2, + TAOS_SYNC_ROLE_SLAVE = 3, + TAOS_SYNC_ROLE_MASTER = 4 +} ESyncRole; + +typedef enum { + TAOS_SYNC_STATUS_INIT = 0, + TAOS_SYNC_STATUS_START = 1, + TAOS_SYNC_STATUS_FILE = 2, + TAOS_SYNC_STATUS_CACHE = 3 +} ESyncStatus; + +typedef struct { + uint32_t nodeId; // node ID assigned by TDengine + uint16_t nodePort; // node sync Port + char nodeFqdn[TSDB_FQDN_LEN]; // node FQDN +} SNodeInfo; + +typedef struct { + int8_t quorum; // number of confirms required, >=1 + int8_t replica; // number of replications, >=1 + SNodeInfo nodeInfo[TAOS_SYNC_MAX_REPLICA]; +} SSyncCfg; + +typedef struct { + int32_t selfIndex; + uint32_t nodeId[TAOS_SYNC_MAX_REPLICA]; + int32_t role[TAOS_SYNC_MAX_REPLICA]; +} SNodesRole; + +// get the wal file from index or after +// return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file +typedef int32_t (*FGetWalInfo)(int32_t vgId, char *fileName, int64_t *fileId); + +// when a forward pkt is received, call this to handle data +typedef int32_t (*FWriteToCache)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg); + +// when forward is confirmed by peer, master call this API to notify app +typedef void (*FConfirmForward)(int32_t vgId, void *mhandle, int32_t code); + +// when role is changed, call this to notify app +typedef void (*FNotifyRole)(int32_t vgId, int8_t role); + +// if a number of retrieving data failed, call this to start flow control +typedef void (*FNotifyFlowCtrl)(int32_t vgId, int32_t level); + +// when data file is synced successfully, notity app +typedef void (*FStartSyncFile)(int32_t vgId); +typedef void (*FStopSyncFile)(int32_t vgId, uint64_t fversion); + +// get file version +typedef int32_t (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver); + +typedef int32_t (*FSendFile)(void *tsdb, SOCKET socketFd); +typedef int32_t (*FRecvFile)(void *tsdb, SOCKET socketFd); + +typedef struct { + int32_t vgId; // vgroup ID + uint64_t version; // initial version + SSyncCfg syncCfg; // configuration from mgmt + char path[TSDB_FILENAME_LEN]; // path to the file + void * pTsdb; + FGetWalInfo getWalInfoFp; + FWriteToCache writeToCacheFp; + FConfirmForward confirmForward; + FNotifyRole notifyRoleFp; + FNotifyFlowCtrl notifyFlowCtrlFp; + FStartSyncFile startSyncFileFp; + FStopSyncFile stopSyncFileFp; + FGetVersion getVersionFp; + FSendFile sendFileFp; + FRecvFile recvFileFp; +} SSyncInfo; + +typedef void *tsync_h; + +int32_t syncInit(); +void syncCleanUp(); + +int64_t syncStart(const SSyncInfo *); +void syncStop(int64_t rid); +int32_t syncReconfig(int64_t rid, const SSyncCfg *); +int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype, bool force); +void syncConfirmForward(int64_t rid, uint64_t version, int32_t code, bool force); +void syncRecover(int64_t rid); // recover from other nodes: +int32_t syncGetNodesRole(int64_t rid, SNodesRole *); + +extern char *syncRole[]; + +//global configurable parameters +extern int32_t sDebugFlag; +extern char tsArbitrator[]; +extern uint16_t tsSyncPort; + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_SYNC_H diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 78d6f74df1..dd5fa2e2a5 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -32,6 +32,8 @@ int indicator = 1; void insertChar(Command *cmd, char *c, int size); +void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, + int32_t pkgNum, char *pkgType); const char *argp_program_version = version; const char *argp_program_bug_address = ""; static char doc[] = ""; @@ -59,7 +61,8 @@ static struct argp_option options[] = { {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync|speen|fqdn."}, {"pktlen", 'l', "PKTLEN", 0, "Packet length used for net test, default is 1000 bytes."}, {"pktnum", 'N', "PKTNUM", 0, "Packet numbers used for net test, default is 100."}, - {"pkttype", 'S', "PKTTYPE", 0, "Packet type used for net test, default is TCP."}, +// Shuduo: 3.0 does not support UDP any more +// {"pkttype", 'S', "PKTTYPE", 0, "Packet type used for net test, default is TCP."}, {0}}; static error_t parse_opt(int key, char *arg, struct argp_state *state) { @@ -629,16 +632,25 @@ int main(int argc, char *argv[]) { taosDumpGlobalCfg(); exit(0); } +#endif if (args.netTestRole && args.netTestRole[0] != 0) { - if (taos_init()) { + TAOS *con = NULL; + if (args.auth == NULL) { + con = taos_connect(args.host, args.user, args.password, args.database, args.port); + } else { + con = taos_connect_auth(args.host, args.user, args.auth, args.database, args.port); + } + +/* if (taos_init()) { printf("Failed to init taos"); exit(EXIT_FAILURE); } + */ taosNetTest(args.netTestRole, args.host, args.port, args.pktLen, args.pktNum, args.pktType); + taos_close(con); exit(0); } -#endif /* Initialize the shell */ TAOS *con = shellInit(&args); diff --git a/tools/shell/src/backup/tnettest.c b/tools/shell/src/tnettest.c similarity index 85% rename from tools/shell/src/backup/tnettest.c rename to tools/shell/src/tnettest.c index ee32bfb6be..d0b5e5f25c 100644 --- a/tools/shell/src/backup/tnettest.c +++ b/tools/shell/src/tnettest.c @@ -14,18 +14,20 @@ */ #define _DEFAULT_SOURCE +#define ALLOW_FORBID_FUNC #include "os.h" #include "taosdef.h" #include "tmsg.h" #include "taoserror.h" #include "tlog.h" #include "tglobal.h" -#include "tsocket.h" #include "trpc.h" #include "rpcHead.h" #include "tchecksum.h" #include "syncMsg.h" +#include "osSocket.h" + #define MAX_PKG_LEN (64 * 1000) #define MAX_SPEED_PKG_LEN (1024 * 1024 * 1024) #define MIN_SPEED_PKG_LEN 1024 @@ -33,7 +35,7 @@ #define MIN_SPEED_PKG_NUM 1 #define BUFFER_SIZE (MAX_PKG_LEN + 1024) -extern int32_t tsRpcMaxUdpSize; +extern int tsRpcMaxUdpSize; typedef struct { char * hostFqdn; @@ -71,15 +73,23 @@ static void *taosNetBindUdpPort(void *sarg) { return NULL; } - if (taosSetSockOpt(serverSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(serverSocket); + return NULL; + } + pSocket->fd = serverSocket; + pSocket->refId = 0; + + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { uError("failed to set the send buffer size for UDP socket\n"); - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } - if (taosSetSockOpt(serverSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { uError("failed to set the receive buffer size for UDP socket\n"); - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } @@ -98,13 +108,13 @@ static void *taosNetBindUdpPort(void *sarg) { uInfo("UDP: recv:%d bytes from %s at %d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port); if (iDataNum > 0) { - iDataNum = taosSendto(serverSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int32_t)sin_size); + iDataNum = taosSendto(pSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int32_t)sin_size); } uInfo("UDP: send:%d bytes to %s at %d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port); } - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } @@ -132,25 +142,35 @@ static void *taosNetBindTcpPort(void *sarg) { server_addr.sin_addr.s_addr = htonl(INADDR_ANY); int32_t reuse = 1; - if (taosSetSockOpt(serverSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { + TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(serverSocket); + return NULL; + } + pSocket->fd = serverSocket; + pSocket->refId = 0; + + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { uError("failed to bind TCP port:%d since %s", port, strerror(errno)); + taosCloseSocket(&pSocket); return NULL; } - if (taosKeepTcpAlive(serverSocket) < 0) { + if (taosKeepTcpAlive(pSocket) < 0) { uError("failed to set tcp server keep-alive option since %s", strerror(errno)); - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } if (listen(serverSocket, 10) < 0) { uError("failed to listen TCP port:%d since %s", port, strerror(errno)); + taosCloseSocket(&pSocket); return NULL; } @@ -163,26 +183,26 @@ static void *taosNetBindTcpPort(void *sarg) { continue; } - int32_t ret = taosReadMsg(client, buffer, pinfo->pktLen); + int32_t ret = taosReadMsg(pSocket, buffer, pinfo->pktLen); if (ret < 0 || ret != pinfo->pktLen) { uError("TCP: failed to read %d bytes at port:%d since %s", pinfo->pktLen, port, strerror(errno)); - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } uInfo("TCP: read:%d bytes from %s at %d", pinfo->pktLen, taosInetNtoa(clientAddr.sin_addr), port); - ret = taosWriteMsg(client, buffer, pinfo->pktLen); + ret = taosWriteMsg(pSocket, buffer, pinfo->pktLen); if (ret < 0) { uError("TCP: failed to write %d bytes at %d since %s", pinfo->pktLen, port, strerror(errno)); - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } uInfo("TCP: write:%d bytes to %s at %d", pinfo->pktLen, taosInetNtoa(clientAddr.sin_addr), port); } - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } @@ -196,9 +216,17 @@ static int32_t taosNetCheckTcpPort(STestInfo *info) { } int32_t reuse = 1; - if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { + TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(clientSocket); + return -1; + } + pSocket->fd = clientSocket; + pSocket->refId = 0; + + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(clientSocket); + taosCloseSocket(&pSocket); return -1; } @@ -210,27 +238,30 @@ static int32_t taosNetCheckTcpPort(STestInfo *info) { if (connect(clientSocket, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) { uError("TCP: failed to connect port %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } - taosKeepTcpAlive(clientSocket); + taosKeepTcpAlive(pSocket); sprintf(buffer, "client send TCP pkg to %s:%d, content: 1122334455", taosIpStr(info->hostIp), info->port); sprintf(buffer + info->pktLen - 16, "1122334455667788"); - int32_t ret = taosWriteMsg(clientSocket, buffer, info->pktLen); + int32_t ret = taosWriteMsg(pSocket, buffer, info->pktLen); if (ret < 0) { uError("TCP: failed to write msg to %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } - ret = taosReadMsg(clientSocket, buffer, info->pktLen); + ret = taosReadMsg(pSocket, buffer, info->pktLen); if (ret < 0) { uError("TCP: failed to read msg from %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } - taosCloseSocket(clientSocket); + taosCloseSocket(&pSocket); return 0; } @@ -247,13 +278,23 @@ static int32_t taosNetCheckUdpPort(STestInfo *info) { return -1; } - if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(clientSocket); + return -1; + } + pSocket->fd = clientSocket; + pSocket->refId = 0; + + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { uError("failed to set the send buffer size for UDP socket\n"); + taosCloseSocket(&pSocket); return -1; } - if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { uError("failed to set the receive buffer size for UDP socket\n"); + taosCloseSocket(&pSocket); return -1; } @@ -268,9 +309,10 @@ static int32_t taosNetCheckUdpPort(STestInfo *info) { socklen_t sin_size = sizeof(*(struct sockaddr *)&serverAddr); - iDataNum = taosSendto(clientSocket, buffer, info->pktLen, 0, (struct sockaddr *)&serverAddr, (int32_t)sin_size); + iDataNum = taosSendto(pSocket, buffer, info->pktLen, 0, (struct sockaddr *)&serverAddr, (int32_t)sin_size); if (iDataNum < 0 || iDataNum != info->pktLen) { uError("UDP: failed to perform sendto func since %s", strerror(errno)); + taosCloseSocket(&pSocket); return -1; } @@ -280,10 +322,11 @@ static int32_t taosNetCheckUdpPort(STestInfo *info) { if (iDataNum < 0 || iDataNum != info->pktLen) { uError("UDP: received ack:%d bytes(expect:%d) from port:%d since %s", iDataNum, info->pktLen, info->port, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } - taosCloseSocket(clientSocket); + taosCloseSocket(&pSocket); return 0; } @@ -339,7 +382,7 @@ void *taosNetInitRpc(char *secretEncrypt, char spi) { } static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t pktLen, char spi, SStartupReq *pStep) { - SRpcEpSet epSet; + SEpSet epSet; SRpcMsg reqMsg; SRpcMsg rspMsg; void * pRpcConn; @@ -352,11 +395,10 @@ static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t p return TSDB_CODE_RPC_NETWORK_UNAVAIL; } - memset(&epSet, 0, sizeof(SRpcEpSet)); - epSet.inUse = 0; + memset(&epSet, 0, sizeof(SEpSet)); + strcpy(epSet.eps[0].fqdn, serverFqdn); + epSet.eps[0].port = port; epSet.numOfEps = 1; - epSet.port[0] = port; - strcpy(epSet.fqdn[0], serverFqdn); reqMsg.msgType = TDMT_DND_NETWORK_TEST; reqMsg.pCont = rpcMallocCont(pktLen); @@ -425,8 +467,8 @@ static void taosNetCheckSync(char *host, int32_t port) { return; } - SOCKET connFd = taosOpenTcpClientSocket(ip, (uint16_t)port, 0); - if (connFd < 0) { + TdSocketPtr pSocket = taosOpenTcpClientSocket(ip, (uint16_t)port, 0); + if (pSocket == NULL) { uError("failed to create socket while test port:%d since %s", port, strerror(errno)); return; } @@ -443,17 +485,17 @@ static void taosNetCheckSync(char *host, int32_t port) { pHead->len = sizeof(SSyncMsg) - sizeof(SSyncHead); taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SSyncHead)); - if (taosWriteMsg(connFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { + if (taosWriteMsg(pSocket, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { uError("failed to test port:%d while send msg since %s", port, strerror(errno)); return; } - if (taosReadMsg(connFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { + if (taosReadMsg(pSocket, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { uError("failed to test port:%d while recv msg since %s", port, strerror(errno)); } uInfo("successed to test TCP port:%d", port); - taosCloseSocket(connFd); + taosCloseSocket(&pSocket); } static void taosNetTestRpc(char *host, int32_t startPort, int32_t pkgLen) { @@ -494,7 +536,6 @@ static void taosNetTestRpc(char *host, int32_t startPort, int32_t pkgLen) { } taosNetCheckSync(host, startPort + TSDB_PORT_SYNC); - taosNetCheckSync(host, startPort + TSDB_PORT_ARBITRATOR); } static void taosNetTestClient(char *host, int32_t startPort, int32_t pkgLen) { @@ -578,7 +619,7 @@ static void taosNetCheckSpeed(char *host, int32_t port, int32_t pkgLen, } tsCompressMsgSize = -1; - SRpcEpSet epSet; + SEpSet epSet; SRpcMsg reqMsg; SRpcMsg rspMsg; void * pRpcConn; @@ -596,11 +637,10 @@ static void taosNetCheckSpeed(char *host, int32_t port, int32_t pkgLen, for (int32_t i = 1; i <= pkgNum; i++) { uint64_t startTime = taosGetTimestampUs(); - memset(&epSet, 0, sizeof(SRpcEpSet)); - epSet.inUse = 0; + memset(&epSet, 0, sizeof(SEpSet)); + strcpy(epSet.eps[0].fqdn, host); + epSet.eps[0].port = port; epSet.numOfEps = 1; - epSet.port[0] = port; - strcpy(epSet.fqdn[0], host); reqMsg.msgType = TDMT_DND_NETWORK_TEST; reqMsg.pCont = rpcMallocCont(pkgLen); @@ -641,7 +681,7 @@ static void taosNetCheckSpeed(char *host, int32_t port, int32_t pkgLen, void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType) { - tscEmbedded = 1; + tsLogEmbedded = 1; if (host == NULL) host = tsLocalFqdn; if (port == 0) port = tsServerPort; if (0 == strcmp("speed", role)){ @@ -659,14 +699,14 @@ void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, } else if (0 == strcmp("server", role)) { taosNetTestServer(host, port, pkgLen); } else if (0 == strcmp("rpc", role)) { - tscEmbedded = 0; + tsLogEmbedded = 0; taosNetTestRpc(host, port, pkgLen); } else if (0 == strcmp("sync", role)) { taosNetCheckSync(host, port); } else if (0 == strcmp("startup", role)) { taosNetTestStartup(host, port); } else if (0 == strcmp("speed", role)) { - tscEmbedded = 0; + tsLogEmbedded = 0; char type[10] = {0}; taosNetCheckSpeed(host, port, pkgLen, pkgNum, strtolower(type, pkgType)); }else if (0 == strcmp("fqdn", role)) { @@ -675,5 +715,5 @@ void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, taosNetTestStartup(host, port); } - tscEmbedded = 0; + tsLogEmbedded = 0; }