diff --git a/CMakeLists.txt b/CMakeLists.txt index cda71fb3bf..99e48006b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,7 @@ set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib") include(${CMAKE_SUPPORT_DIR}/cmake.options) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma -g3") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma -g3") # contrib add_subdirectory(contrib) diff --git a/include/common/common.h b/include/common/common.h index 1c3a0e22f7..0ae06e5a7f 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -17,7 +17,7 @@ #define TDENGINE_COMMON_H #include "taosdef.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tarray.h" #include "tvariant.h" //typedef struct STimeWindow { diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 9b2bb89793..da58f98e4c 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -20,36 +20,38 @@ extern "C" { #endif -#include "tdef.h" #include "taos.h" +#include "tdef.h" -#define TSWINDOW_INITIALIZER ((STimeWindow) {INT64_MIN, INT64_MAX}) -#define TSWINDOW_DESC_INITIALIZER ((STimeWindow) {INT64_MAX, INT64_MIN}) +typedef uint64_t tb_uid_t; + +#define TSWINDOW_INITIALIZER ((STimeWindow){INT64_MIN, INT64_MAX}) +#define TSWINDOW_DESC_INITIALIZER ((STimeWindow){INT64_MAX, INT64_MIN}) #define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX)) typedef enum { - TAOS_QTYPE_RPC = 1, - TAOS_QTYPE_FWD = 2, - TAOS_QTYPE_WAL = 3, - TAOS_QTYPE_CQ = 4, + TAOS_QTYPE_RPC = 1, + TAOS_QTYPE_FWD = 2, + TAOS_QTYPE_WAL = 3, + TAOS_QTYPE_CQ = 4, TAOS_QTYPE_QUERY = 5 } EQType; typedef enum { - TSDB_SUPER_TABLE = 1, // super table - TSDB_CHILD_TABLE = 2, // table created from super table - TSDB_NORMAL_TABLE = 3, // ordinary table - TSDB_STREAM_TABLE = 4, // table created from stream computing - TSDB_TEMP_TABLE = 5, // temp table created by nest query - TSDB_TABLE_MAX = 6 + TSDB_SUPER_TABLE = 1, // super table + TSDB_CHILD_TABLE = 2, // table created from super table + TSDB_NORMAL_TABLE = 3, // ordinary table + TSDB_STREAM_TABLE = 4, // table created from stream computing + TSDB_TEMP_TABLE = 5, // temp table created by nest query + TSDB_TABLE_MAX = 6 } ETableType; typedef enum { - TSDB_MOD_MNODE = 1, - TSDB_MOD_HTTP = 2, + TSDB_MOD_MNODE = 1, + TSDB_MOD_HTTP = 2, TSDB_MOD_MONITOR = 3, - TSDB_MOD_MQTT = 4, - TSDB_MOD_MAX = 5 + TSDB_MOD_MQTT = 4, + TSDB_MOD_MAX = 5 } EModuleType; typedef enum { @@ -64,11 +66,7 @@ typedef enum { TSDB_CHECK_ITEM_MAX } ECheckItemType; -typedef enum { - TD_ROW_DISCARD_UPDATE = 0, - TD_ROW_OVERWRITE_UPDATE = 1, - TD_ROW_PARTIAL_UPDATE = 2 -} TDUpdateConfig; +typedef enum { TD_ROW_DISCARD_UPDATE = 0, TD_ROW_OVERWRITE_UPDATE = 1, TD_ROW_PARTIAL_UPDATE = 2 } TDUpdateConfig; extern char *qtypeStr[]; diff --git a/include/common/tep.h b/include/common/tep.h index 2addc11e5a..aad5cc81d4 100644 --- a/include/common/tep.h +++ b/include/common/tep.h @@ -2,7 +2,7 @@ #define TDENGINE_TEP_H #include "os.h" -#include "taosmsg.h" +#include "tmsg.h" typedef struct SCorEpSet { int32_t version; diff --git a/include/common/taosmsg.h b/include/common/tmsg.h similarity index 72% rename from include/common/taosmsg.h rename to include/common/tmsg.h index f0b2d0f08b..92b453bc1f 100644 --- a/include/common/taosmsg.h +++ b/include/common/tmsg.h @@ -22,127 +22,36 @@ extern "C" { #include "taosdef.h" #include "taoserror.h" +#include "tcoding.h" #include "tdataformat.h" -// message type +#define TD_MSG_NUMBER_ +#undef TD_MSG_DICT_ +#undef TD_MSG_INFO_ +#undef TD_MSG_SEG_CODE_ +#include "tmsgdef.h" -#ifdef TAOS_MESSAGE_C -#define TAOS_DEFINE_MESSAGE_TYPE( name, msg ) msg, msg "-rsp", -char *taosMsg[] = { - "null", -#else -#define TAOS_DEFINE_MESSAGE_TYPE( name, msg ) name, name##_RSP, -enum { - TSDB_MESSAGE_NULL = 0, -#endif +#undef TD_MSG_NUMBER_ +#undef TD_MSG_DICT_ +#undef TD_MSG_INFO_ +#define TD_MSG_SEG_CODE_ +#include "tmsgdef.h" -// message from client to vnode -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SUBMIT, "submit" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_QUERY, "query" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_FETCH, "fetch" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_TABLE, "create-table" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_TABLE, "drop-table" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_TABLE, "alter-table" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_UPDATE_TAG_VAL, "update-tag-val" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_TABLE_META, "table-meta" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_TABLES_META, "tables-meta" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONSUME, "mq-consume" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_QUERY, "mq-query" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONNECT, "mq-connect" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_DISCONNECT, "mq-disconnect" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_SET_CUR, "mq-set-cur" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RES_READY, "res-ready" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_TASKS_STATUS, "tasks-status" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CANCEL_TASK, "cancel-task" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_TASK, "drop-task" ) +#undef TD_MSG_NUMBER_ +#undef TD_MSG_DICT_ +#undef TD_MSG_INFO_ +#undef TD_MSG_SEG_CODE_ +#include "tmsgdef.h" -// message from client to mnode -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONNECT, "connect" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_ACCT, "create-acct" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_ACCT, "alter-acct" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_ACCT, "drop-acct" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_USER, "create-user" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_USER, "alter-user" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_USER, "drop-user" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_DNODE, "create-dnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONFIG_DNODE, "config-dnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_DNODE, "drop-dnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_MNODE, "create-mnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_MNODE, "drop-mnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_DB, "create-db" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_DB, "drop-db" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_USE_DB, "use-db" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_DB, "alter-db" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SYNC_DB, "sync-db" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_COMPACT_DB, "compact-db" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_FUNCTION, "create-function" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RETRIEVE_FUNCTION, "retrieve-function" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_FUNCTION, "drop-function" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB, "create-stb" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB, "alter-stb" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB, "drop-stb" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_VGROUP_LIST, "vgroup-list" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_QUERY, "kill-query" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_CONN, "kill-conn" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_HEARTBEAT, "heartbeat" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SHOW, "show" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SHOW_RETRIEVE, "retrieve" ) -// message from client to qnode -// message from client to dnode -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "nettest" ) +extern char* tMsgInfo[]; +extern int tMsgDict[]; -// message from vnode to vnode -// message from vnode to mnode -// message from vnode to qnode -// message from vnode to dnode +#define TMSG_SEG_CODE(TYPE) (((TYPE)&0xff00) >> 8) +#define TMSG_SEG_SEQ(TYPE) ((TYPE)&0xff) +#define TMSG_INFO(TYPE) tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)] +#define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)) -// message from mnode to vnode -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB_IN, "create-stb-internal" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB_IN, "alter-stb-internal" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB_IN, "drop-stb-internal" ) -// message from mnode to mnode -// message from mnode to qnode -// message from mnode to dnode -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_VNODE_IN, "create-vnode-internal" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_VNODE_IN, "alter-vnode-internal" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_VNODE_IN, "drop-vnode-internal" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_AUTH_VNODE_IN, "auth-vnode-internal" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SYNC_VNODE_IN, "sync-vnode-internal" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_COMPACT_VNODE_IN, "compact-vnode-internal" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_MNODE_IN, "create-mnode-internal" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_MNODE_IN, "alter-mnode-internal" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_MNODE_IN, "drop-mnode-internal" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONFIG_DNODE_IN, "config-dnode-internal" ) - -// message from qnode to vnode -// message from qnode to mnode -// message from qnode to qnode -// message from qnode to dnode - -// message from dnode to vnode -// message from dnode to mnode -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_STATUS, "status" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_GRANT, "grant" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_AUTH, "auth" ) -// message from dnode to qnode -// message from dnode to dnode - -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY0, "dummy0" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY1, "dummy1" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY2, "dummy2" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY3, "dummy3" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY4, "dummy4" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY5, "dummy5" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY6, "dummy6" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY7, "dummy7" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY8, "dummy8" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY9, "dummy9" ) - -#ifndef TAOS_MESSAGE_C - TSDB_MSG_TYPE_MAX // 147 -#endif - -}; +typedef uint16_t tmsg_t; // IE type #define TSDB_IE_TYPE_SEC 1 @@ -178,48 +87,46 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_MAX, } EShowType; -#define TSDB_ALTER_TABLE_ADD_TAG_COLUMN 1 -#define TSDB_ALTER_TABLE_DROP_TAG_COLUMN 2 +#define TSDB_ALTER_TABLE_ADD_TAG_COLUMN 1 +#define TSDB_ALTER_TABLE_DROP_TAG_COLUMN 2 #define TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN 3 -#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4 +#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4 -#define TSDB_ALTER_TABLE_ADD_COLUMN 5 -#define TSDB_ALTER_TABLE_DROP_COLUMN 6 -#define TSDB_ALTER_TABLE_CHANGE_COLUMN 7 +#define TSDB_ALTER_TABLE_ADD_COLUMN 5 +#define TSDB_ALTER_TABLE_DROP_COLUMN 6 +#define TSDB_ALTER_TABLE_CHANGE_COLUMN 7 #define TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN 8 -#define TSDB_FILL_NONE 0 -#define TSDB_FILL_NULL 1 -#define TSDB_FILL_SET_VALUE 2 -#define TSDB_FILL_LINEAR 3 -#define TSDB_FILL_PREV 4 -#define TSDB_FILL_NEXT 5 +#define TSDB_FILL_NONE 0 +#define TSDB_FILL_NULL 1 +#define TSDB_FILL_SET_VALUE 2 +#define TSDB_FILL_LINEAR 3 +#define TSDB_FILL_PREV 4 +#define TSDB_FILL_NEXT 5 -#define TSDB_ALTER_USER_PASSWD 0x1 +#define TSDB_ALTER_USER_PASSWD 0x1 #define TSDB_ALTER_USER_PRIVILEGES 0x2 -#define TSDB_KILL_MSG_LEN 30 +#define TSDB_KILL_MSG_LEN 30 -#define TSDB_VN_READ_ACCCESS ((char)0x1) -#define TSDB_VN_WRITE_ACCCESS ((char)0x2) +#define TSDB_VN_READ_ACCCESS ((char)0x1) +#define TSDB_VN_WRITE_ACCCESS ((char)0x2) #define TSDB_VN_ALL_ACCCESS (TSDB_VN_READ_ACCCESS | TSDB_VN_WRITE_ACCCESS) -#define TSDB_COL_NORMAL 0x0u // the normal column of the table -#define TSDB_COL_TAG 0x1u // the tag column type -#define TSDB_COL_UDC 0x2u // the user specified normal string column, it is a dummy column -#define TSDB_COL_TMP 0x4u // internal column generated by the previous operators -#define TSDB_COL_NULL 0x8u // the column filter NULL or not +#define TSDB_COL_NORMAL 0x0u // the normal column of the table +#define TSDB_COL_TAG 0x1u // the tag column type +#define TSDB_COL_UDC 0x2u // the user specified normal string column, it is a dummy column +#define TSDB_COL_TMP 0x4u // internal column generated by the previous operators +#define TSDB_COL_NULL 0x8u // the column filter NULL or not -#define TSDB_COL_IS_TAG(f) (((f&(~(TSDB_COL_NULL)))&TSDB_COL_TAG) != 0) -#define TSDB_COL_IS_NORMAL_COL(f) ((f&(~(TSDB_COL_NULL))) == TSDB_COL_NORMAL) -#define TSDB_COL_IS_UD_COL(f) ((f&(~(TSDB_COL_NULL))) == TSDB_COL_UDC) -#define TSDB_COL_REQ_NULL(f) (((f)&TSDB_COL_NULL) != 0) - -extern char *taosMsg[]; +#define TSDB_COL_IS_TAG(f) (((f & (~(TSDB_COL_NULL))) & TSDB_COL_TAG) != 0) +#define TSDB_COL_IS_NORMAL_COL(f) ((f & (~(TSDB_COL_NULL))) == TSDB_COL_NORMAL) +#define TSDB_COL_IS_UD_COL(f) ((f & (~(TSDB_COL_NULL))) == TSDB_COL_UDC) +#define TSDB_COL_REQ_NULL(f) (((f)&TSDB_COL_NULL) != 0) typedef struct SBuildTableMetaInput { int32_t vgId; - char *tableFullName; + char* tableFullName; } SBuildTableMetaInput; typedef struct SBuildUseDBInput { @@ -227,7 +134,6 @@ typedef struct SBuildUseDBInput { int32_t vgVersion; } SBuildUseDBInput; - #pragma pack(push, 1) // null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta @@ -312,7 +218,7 @@ typedef struct { char data[]; } SMDCreateTableMsg; -//typedef struct { +// typedef struct { // int32_t len; // one create table message // char tableName[TSDB_TABLE_FNAME_LEN]; // int16_t numOfColumns; @@ -442,7 +348,7 @@ typedef struct { int8_t type; char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; - int8_t superUser; // denote if it is a super user or not + int8_t superUser; // denote if it is a super user or not int32_t reserve[8]; } SCreateUserMsg, SAlterUserMsg; @@ -493,7 +399,7 @@ typedef struct SColumnFilterList { int16_t numOfFilters; union { int64_t placeholder; - SColumnFilterInfo *filterInfo; + SColumnFilterInfo* filterInfo; }; } SColumnFilterList; /* @@ -578,8 +484,8 @@ typedef struct { int32_t sqlstrLen; // sql query string int32_t prevResultLen; // previous result length int32_t numOfOperator; - int32_t tableScanOperator;// table scan operator. -1 means no scan operator - int32_t udfNum; // number of udf function + int32_t tableScanOperator; // table scan operator. -1 means no scan operator + int32_t udfNum; // number of udf function int32_t udfContentOffset; int32_t udfContentLen; SColumnInfo tableCols[]; @@ -882,7 +788,7 @@ typedef struct SMultiTableMeta { typedef struct { int32_t dataLen; char name[TSDB_TABLE_FNAME_LEN]; - char *data; + char* data; } STagData; typedef struct { @@ -917,7 +823,7 @@ typedef struct SShowRsp { } SShowRsp; typedef struct { - char ep[TSDB_EP_LEN]; // end point, hostname:port + char ep[TSDB_EP_LEN]; // end point, hostname:port } SCreateDnodeMsg; typedef struct { @@ -1019,35 +925,27 @@ typedef struct { // mq related typedef struct { - } SMqConnectReq; typedef struct { - } SMqConnectRsp; typedef struct { - } SMqDisconnectReq; typedef struct { - } SMqDisconnectRsp; typedef struct { - } SMqAckReq; typedef struct { - } SMqAckRsp; typedef struct { - } SMqResetReq; typedef struct { - } SMqResetRsp; // mq related end @@ -1104,17 +1002,17 @@ typedef struct { } SUpdateTagValRsp; typedef struct SSubQueryMsg { - uint64_t schedulerId; - uint64_t queryId; - uint64_t taskId; - uint32_t contentLen; - char msg[]; + uint64_t schedulerId; + uint64_t queryId; + uint64_t taskId; + uint32_t contentLen; + char msg[]; } SSubQueryMsg; typedef struct SResReadyMsg { - uint64_t schedulerId; - uint64_t queryId; - uint64_t taskId; + uint64_t schedulerId; + uint64_t queryId; + uint64_t taskId; } SResReadyMsg; typedef struct SResReadyRsp { @@ -1122,19 +1020,19 @@ typedef struct SResReadyRsp { } SResReadyRsp; typedef struct SResFetchMsg { - uint64_t schedulerId; - uint64_t queryId; - uint64_t taskId; + uint64_t schedulerId; + uint64_t queryId; + uint64_t taskId; } SResFetchMsg; typedef struct SSchTasksStatusMsg { - uint64_t schedulerId; + uint64_t schedulerId; } SSchTasksStatusMsg; typedef struct STaskStatus { - uint64_t queryId; - uint64_t taskId; - int8_t status; + uint64_t queryId; + uint64_t taskId; + int8_t status; } STaskStatus; typedef struct SSchedulerStatusRsp { @@ -1142,11 +1040,10 @@ typedef struct SSchedulerStatusRsp { STaskStatus status[]; } SSchedulerStatusRsp; - typedef struct STaskCancelMsg { - uint64_t schedulerId; - uint64_t queryId; - uint64_t taskId; + uint64_t schedulerId; + uint64_t queryId; + uint64_t taskId; } STaskCancelMsg; typedef struct STaskCancelRsp { @@ -1154,15 +1051,176 @@ typedef struct STaskCancelRsp { } STaskCancelRsp; typedef struct STaskDropMsg { - uint64_t schedulerId; - uint64_t queryId; - uint64_t taskId; + uint64_t schedulerId; + uint64_t queryId; + uint64_t taskId; } STaskDropMsg; typedef struct STaskDropRsp { int32_t code; } STaskDropRsp; +typedef struct { + char name[TSDB_TOPIC_FNAME_LEN]; + int8_t igExists; + int32_t execLen; + void* executor; + int32_t sqlLen; + char* sql; +} SCreateTopicMsg; + +typedef struct { + char name[TSDB_TABLE_FNAME_LEN]; + int8_t igNotExists; +} SDropTopicMsg; + +typedef struct { + char name[TSDB_TABLE_FNAME_LEN]; + int8_t alterType; + SSchema schema; +} SAlterTopicMsg; + +typedef struct { + SMsgHead head; + char name[TSDB_TABLE_FNAME_LEN]; + uint64_t tuid; + int32_t sverson; + int32_t execLen; + char* executor; + int32_t sqlLen; + char* sql; +} SCreateTopicInternalMsg; + +typedef struct { + SMsgHead head; + char name[TSDB_TABLE_FNAME_LEN]; + uint64_t tuid; +} SDropTopicInternalMsg; + +typedef struct SVCreateTbReq { + uint64_t ver; // use a general definition + char* name; + uint32_t ttl; + uint32_t keep; +#define TD_SUPER_TABLE 0 +#define TD_CHILD_TABLE 1 +#define TD_NORMAL_TABLE 2 + uint8_t type; + union { + struct { + tb_uid_t suid; + uint32_t nCols; + SSchema* pSchema; + uint32_t nTagCols; + SSchema* pTagSchema; + } stbCfg; + struct { + tb_uid_t suid; + SKVRow pTag; + } ctbCfg; + struct { + uint32_t nCols; + SSchema* pSchema; + } ntbCfg; + }; +} SVCreateTbReq; + +static FORCE_INLINE int tSerializeSVCreateTbReq(void** buf, const SVCreateTbReq* pReq) { + int tlen = 0; + + tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeString(buf, pReq->name); + tlen += taosEncodeFixedU32(buf, pReq->ttl); + tlen += taosEncodeFixedU32(buf, pReq->keep); + tlen += taosEncodeFixedU8(buf, pReq->type); + + switch (pReq->type) { + case TD_SUPER_TABLE: + tlen += taosEncodeFixedU64(buf, pReq->stbCfg.suid); + tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nCols); + for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { + tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].type); + tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].colId); + tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].bytes); + tlen += taosEncodeString(buf, pReq->stbCfg.pSchema[i].name); + } + tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nTagCols); + for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) { + tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].type); + tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].colId); + tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes); + tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name); + } + break; + case TD_CHILD_TABLE: + tlen += taosEncodeFixedU64(buf, pReq->ctbCfg.suid); + tlen += tdEncodeKVRow(buf, pReq->ctbCfg.pTag); + break; + case TD_NORMAL_TABLE: + tlen += taosEncodeFixedU32(buf, pReq->ntbCfg.nCols); + for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) { + tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].type); + tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].colId); + tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes); + tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name); + } + break; + default: + ASSERT(0); + } + + return tlen; +} + +static FORCE_INLINE void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq) { + buf = taosDecodeFixedU64(buf, &(pReq->ver)); + buf = taosDecodeString(buf, &(pReq->name)); + buf = taosDecodeFixedU32(buf, &(pReq->ttl)); + buf = taosDecodeFixedU32(buf, &(pReq->keep)); + buf = taosDecodeFixedU8(buf, &(pReq->type)); + + switch (pReq->type) { + case TD_SUPER_TABLE: + buf = taosDecodeFixedU64(buf, &(pReq->stbCfg.suid)); + buf = taosDecodeFixedU32(buf, &(pReq->stbCfg.nCols)); + pReq->stbCfg.pSchema = (SSchema*)malloc(pReq->stbCfg.nCols * sizeof(SSchema)); + for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { + buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type)); + buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].colId)); + buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].bytes)); + buf = taosDecodeStringTo(buf, pReq->stbCfg.pSchema[i].name); + } + buf = taosDecodeFixedU32(buf, &pReq->stbCfg.nTagCols); + pReq->stbCfg.pTagSchema = (SSchema*)malloc(pReq->stbCfg.nTagCols * sizeof(SSchema)); + for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) { + buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type)); + buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].colId); + buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes); + buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name); + } + break; + case TD_CHILD_TABLE: + buf = taosDecodeFixedU64(buf, &pReq->ctbCfg.suid); + buf = tdDecodeKVRow(buf, &pReq->ctbCfg.pTag); + break; + case TD_NORMAL_TABLE: + buf = taosDecodeFixedU32(buf, &pReq->ntbCfg.nCols); + pReq->ntbCfg.pSchema = (SSchema*)malloc(pReq->ntbCfg.nCols * sizeof(SSchema)); + for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) { + buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type); + buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].colId); + buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes); + buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name); + } + break; + default: + ASSERT(0); + } + + return buf; +} +typedef struct SVCreateTbRsp { +} SVCreateTbRsp; #pragma pack(pop) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h new file mode 100644 index 0000000000..15b5b9da28 --- /dev/null +++ b/include/common/tmsgdef.h @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#if 0 +#undef TD_MSG_INFO_ +#undef TD_MSG_NUMBER_ +#undef TD_MSG_DICT_ +#undef TD_MSG_SEG_CODE_ +#endif + +#if defined(TD_MSG_INFO_) + +#undef TD_NEW_MSG_SEG +#undef TD_DEF_MSG_TYPE +#define TD_NEW_MSG_SEG(TYPE) "null", +#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) MSG, MSG "-rsp", + +char *tMsgInfo[] = { + +#elif defined(TD_MSG_NUMBER_) + +#undef TD_NEW_MSG_SEG +#undef TD_DEF_MSG_TYPE +#define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM, +#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE##_NUM, TYPE##_RSP_NUM, + +enum { + +#elif defined(TD_MSG_DICT_) + +#undef TD_NEW_MSG_SEG +#undef TD_DEF_MSG_TYPE +#define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM, +#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) + +int tMsgDict[] = { + +#elif defined(TD_MSG_SEG_CODE_) + +#undef TD_NEW_MSG_SEG +#undef TD_DEF_MSG_TYPE +#define TD_NEW_MSG_SEG(TYPE) TYPE##_SEG_CODE, +#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) + +enum { + +#else + +#undef TD_NEW_MSG_SEG +#undef TD_DEF_MSG_TYPE +#define TD_NEW_MSG_SEG(TYPE) TYPE = ((TYPE##_SEG_CODE) << 8), +#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE, TYPE##_RSP, + +enum { +#endif + // Requests handled by DNODE + TD_NEW_MSG_SEG(TDMT_DND_MSG) + TD_DEF_MSG_TYPE(TDMT_DND_NETWORK_TEST, "dnode-nettest", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_CREATE_VNODE, "dnode-create-vnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE, "dnode-alter-vnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_DROP_VNODE, "dnode-drop-vnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_AUTH_VNODE, "dnode-auth-vnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_SYNC_VNODE, "dnode-sync-vnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_COMPACT_VNODE, "dnode-compact-vnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_CREATE_MNODE, "dnode-create-mnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_ALTER_MNODE, "dnode-alter-mnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_DROP_MNODE, "dnode-drop-mnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "dnode-config-dnode", NULL, NULL) + + // Requests handled by MNODE + TD_NEW_MSG_SEG(TDMT_MND_MSG) + TD_DEF_MSG_TYPE(TDMT_MND_CONNECT, "mnode-connect", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TABLE, "mnode-create-table", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_TABLE, "mnode-drop-table", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_ACCT, "mnode-create-acct", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_ACCT, "mnode-alter-acct", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_ACCT, "mnode-drop-acct", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_USER, "mnode-create-user", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_USER, "mnode-alter-user", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_USER, "mnode-drop-user", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_DNODE, "mnode-create-dnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CONFIG_DNODE, "mnode-config-dnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_DNODE, "mnode-drop-dnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_MNODE, "mnode-create-mnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_MNODE, "mnode-drop-mnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_DB, "mnode-create-db", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_DB, "mnode-drop-db", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_USE_DB, "mnode-use-db", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_DB, "mnode-alter-db", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_SYNC_DB, "mnode-sync-db", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_DB, "mnode-compact-db", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_FUNCTION, "mnode-create-function", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_RETRIEVE_FUNCTION, "mnode-retrieve-function", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_FUNCTION, "mnode-drop-function", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STB, "mnode-create-stb", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STB, "mnode-alter-stb", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_STB, "mnode-drop-stb", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_VGROUP_LIST, "mnode-vgroup-list", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_KILL_QUERY, "mnode-kill-query", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_KILL_CONN, "mnode-kill-conn", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_HEARTBEAT, "mnode-heartbeat", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_SHOW, "mnode-show", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_SHOW_RETRIEVE, "mnode-retrieve", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_STATUS, "mnode-status", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_GRANT, "mnode-grant", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_AUTH, "mnode-auth", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "mnode-create-topic", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL) + + // Requests handled by VNODE + TD_NEW_MSG_SEG(TDMT_VND_MSG) + TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "vnode-submit", SSubmitReq, SSubmitRsp) + TD_DEF_MSG_TYPE(TDMT_VND_QUERY, "vnode-query", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_FETCH, "vnode-fetch", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_ALTER_TABLE, "vnode-alter-table", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_UPDATE_TAG_VAL, "vnode-update-tag-val", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_TABLE_META, "vnode-table-meta", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_TABLES_META, "vnode-tables-meta", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONSUME, "vnode-mq-consume", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_MQ_QUERY, "vnode-mq-query", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONNECT, "vnode-mq-connect", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_TASK, "vnode-cancel-task", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_DROP_TASK, "vnode-drop-task", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_CREATE_STB, "vnode-create-super-table", SVCreateTbReq, SVCreateTbRsp) + TD_DEF_MSG_TYPE(TDMT_VND_ALTER_STB, "vnode-alter-stb", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_DROP_STB, "vnode-drop-stb", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_CREATE_TOPIC, "vnode-create-topic", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_ALTER_TOPIC, "vnode-alter-topic", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_DROP_TOPIC, "vnode-drop-topic", NULL, NULL) + + // Requests handled by QNODE + TD_NEW_MSG_SEG(TDMT_QND_MSG) + + // Requests handled by SNODE + TD_NEW_MSG_SEG(TDMT_SND_MSG) + +#if defined(TD_MSG_NUMBER_) + TDMT_MAX +#endif +}; \ No newline at end of file diff --git a/include/common/tname.h b/include/common/tname.h index de9e309b55..3ac7f8b27b 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -16,18 +16,22 @@ #ifndef TDENGINE_TNAME_H #define TDENGINE_TNAME_H +#include "tdef.h" + #define TSDB_DB_NAME_T 1 #define TSDB_TABLE_NAME_T 2 #define T_NAME_ACCT 0x1u #define T_NAME_DB 0x2u #define T_NAME_TABLE 0x4u +#define T_NAME_TOPIC 0x8u typedef struct SName { uint8_t type; //db_name_t, table_name_t int32_t acctId; char dbname[TSDB_DB_NAME_LEN]; char tname[TSDB_TABLE_NAME_LEN]; + char topicName[TSDB_TOPIC_NAME_LEN]; } SName; int32_t tNameExtractFullName(const SName* name, char* dst); diff --git a/include/common/ttime.h b/include/common/ttime.h index ed6496aca6..e22e1d6c4e 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -21,7 +21,7 @@ extern "C" { #endif #include "taosdef.h" -#include "taosmsg.h" +#include "tmsg.h" #define TIME_IS_VAR_DURATION(_t) ((_t) == 'n' || (_t) == 'y' || (_t) == 'N' || (_t) == 'Y') diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index f4a6f7a4c9..09d1f8c013 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -27,7 +27,6 @@ typedef struct SMnodeMsg SMnodeMsg; typedef void (*SendMsgToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); typedef void (*SendMsgToMnodeFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); typedef void (*SendRedirectMsgFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); -typedef int32_t (*PutMsgToMnodeQFp)(SDnode *pDnode, SMnodeMsg *pMsg); typedef struct SMnodeLoad { int64_t numOfDnode; @@ -63,7 +62,6 @@ typedef struct { SReplica replicas[TSDB_MAX_REPLICA]; SMnodeCfg cfg; SDnode *pDnode; - PutMsgToMnodeQFp putMsgToApplyMsgFp; SendMsgToDnodeFp sendMsgToDnodeFp; SendMsgToMnodeFp sendMsgToMnodeFp; SendRedirectMsgFp sendRedirectMsgFp; @@ -172,14 +170,6 @@ void mndProcessWriteMsg(SMnodeMsg *pMsg); */ void mndProcessSyncMsg(SMnodeMsg *pMsg); -/** - * @brief Process the apply request. - * - * @param pMsg The request msg. - * @return int32_t 0 for success, -1 for failure. - */ -void mndProcessApplyMsg(SMnodeMsg *pMsg); - #ifdef __cplusplus } #endif diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index 51d7ca7202..0f648b5150 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -16,6 +16,8 @@ #ifndef _TD_SDB_H_ #define _TD_SDB_H_ +#include "os.h" + #ifdef __cplusplus extern "C" { #endif @@ -159,11 +161,12 @@ typedef enum { SDB_USER = 5, SDB_AUTH = 6, SDB_ACCT = 7, - SDB_VGROUP = 8, - SDB_STB = 9, - SDB_DB = 10, - SDB_FUNC = 11, - SDB_MAX = 12 + SDB_TOPIC = 8, + SDB_VGROUP = 9, + SDB_STB = 10, + SDB_DB = 11, + SDB_FUNC = 12, + SDB_MAX = 13 } ESdbType; typedef struct SSdb SSdb; diff --git a/include/dnode/vnode/meta/meta.h b/include/dnode/vnode/meta/meta.h index 113a970548..d9f5a3ff09 100644 --- a/include/dnode/vnode/meta/meta.h +++ b/include/dnode/vnode/meta/meta.h @@ -19,13 +19,13 @@ #include "mallocator.h" #include "os.h" #include "trow.h" +#include "tmsg.h" #ifdef __cplusplus extern "C" { #endif // Types exported -typedef uint64_t tb_uid_t; typedef struct SMeta SMeta; #define META_SUPER_TABLE 0 @@ -37,39 +37,7 @@ typedef struct SMetaCfg { uint64_t lruSize; } SMetaCfg; -typedef struct STbCfg { - /// name of the table - char *name; - /// time to live of the table - uint32_t ttl; - /// keep time of this table - uint32_t keep; - /// type of table - uint8_t type; - union { - /// super table configurations - struct { - /// super table UID - tb_uid_t suid; - /// row schema - STSchema *pSchema; - /// tag schema - STSchema *pTagSchema; - } stbCfg; - - /// normal table configuration - struct { - /// row schema - STSchema *pSchema; - } ntbCfg; - /// child table configuration - struct { - /// super table UID - tb_uid_t suid; - SKVRow pTag; - } ctbCfg; - }; -} STbCfg; +typedef SVCreateTbReq STbCfg; // SMeta operations SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF); diff --git a/include/dnode/vnode/tq/tq.h b/include/dnode/vnode/tq/tq.h index f5d5cc9a16..b6cb96a57b 100644 --- a/include/dnode/vnode/tq/tq.h +++ b/include/dnode/vnode/tq/tq.h @@ -20,7 +20,7 @@ #include "mallocator.h" #include "os.h" #include "taoserror.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tlist.h" #include "trpc.h" #include "ttimer.h" diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index e708d7b545..812f313e71 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -246,8 +246,8 @@ static FORCE_INLINE void vnodeSetCreateNtbReq(SVnodeReq *pReq, char *name, uint3 pReq->ctReq.ntbCfg.pSchema = pSchema; } -int vnodeBuildReq(void **buf, const SVnodeReq *pReq, uint8_t type); -void *vnodeParseReq(void *buf, SVnodeReq *pReq, uint8_t type); +int vnodeBuildReq(void **buf, const SVnodeReq *pReq, tmsg_t type); +void *vnodeParseReq(void *buf, SVnodeReq *pReq, tmsg_t type); /* ------------------------ FOR COMPILE ------------------------ */ diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 592c5f707c..4d33706303 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -26,7 +26,7 @@ extern "C" { #include "taosdef.h" #include "transport.h" #include "common.h" -#include "taosmsg.h" +#include "tmsg.h" #include "query.h" struct SCatalog; @@ -110,7 +110,7 @@ int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const * @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller) * @return error code */ -int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList); +int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray** pVgroupList); /** * Get a table's vgroup from its name's hash value. @@ -137,7 +137,7 @@ int32_t catalogGetTableHashVgroup(struct SCatalog* pCatalog, void * pTransporter int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp); -int32_t catalogGetQnodeList(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SEpSet* pQnodeEpSet); +int32_t catalogGetQnodeList(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SArray* pQnodeList); diff --git a/include/libs/index/index.h b/include/libs/index/index.h index f93c46da0c..d2b157542f 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -85,6 +85,18 @@ SIndexTerm* indexTermCreate(int64_t suid, int32_t nColVal); void indexTermDestroy(SIndexTerm* p); +/* + * init index + * + */ +int32_t indexInit(); +/* + * destory index + * + */ + +void indexCleanUp(); + #ifdef __cplusplus } #endif diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 7834bc6913..f2f3fcd49b 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -44,7 +44,9 @@ typedef struct SParseContext { */ int32_t qParseQuerySql(SParseContext* pContext, SQueryNode** pQuery); -bool qIsDclQuery(const SQueryNode* pQuery); +bool qIsDdlQuery(const SQueryNode* pQuery); + +void qDestoryQuery(SQueryNode* pQuery); /** * Convert a normal sql statement to only query tags information to enable that the subscribe client can be aware quickly of the true vgroup ids that diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index d4469be5e3..b3262953b5 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -20,7 +20,7 @@ extern "C" { #endif -#include "taosmsg.h" +#include "tmsg.h" #define QUERY_TYPE_MERGE 1 #define QUERY_TYPE_PARTIAL 2 @@ -139,7 +139,7 @@ struct SQueryNode; /** * Create the physical plan for the query, according to the AST. */ -int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag); +int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SQueryDag** pDag); // Set datasource of this subplan, multiple calls may be made to a subplan. // @subplan subplan to be schedule diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 86b51dbb85..1e970fbd66 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -129,8 +129,8 @@ int32_t asyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransp const SSchema* tGetTbnameColumnSchema(); void initQueryModuleMsgHandle(); -extern int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); -extern int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize); +extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); +extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize); #define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0) #define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0) diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 975b103538..fa1483de87 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -59,7 +59,15 @@ int32_t schedulerInit(SSchedulerCfg *cfg); * @param qnodeList Qnode address list, element is SEpAddr * @return */ -int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob); +int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob, uint64_t *numOfRows); + +/** + * Process the query job, generated according to the query physical plan. + * This is a asynchronized API, and is also thread-safety. + * @param qnodeList Qnode address list, element is SEpAddr + * @return + */ +int32_t scheduleAsyncExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob); int32_t scheduleFetchRows(void *pJob, void **data); @@ -79,4 +87,4 @@ void schedulerDestroy(void); } #endif -#endif /*_TD_SCHEDULER_H_*/ \ No newline at end of file +#endif /*_TD_SCHEDULER_H_*/ diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 2afd9d48b7..b88d45fbd6 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -22,43 +22,43 @@ extern "C" { #include #include #include "taosdef.h" -#include "taosmsg.h" +#include "tmsg.h" -#define TAOS_CONN_SERVER 0 -#define TAOS_CONN_CLIENT 1 +#define TAOS_CONN_SERVER 0 +#define TAOS_CONN_CLIENT 1 extern int tsRpcHeadSize; typedef struct SRpcConnInfo { - uint32_t clientIp; - uint16_t clientPort; - uint32_t serverIp; - char user[TSDB_USER_LEN]; + uint32_t clientIp; + uint16_t clientPort; + uint32_t serverIp; + char user[TSDB_USER_LEN]; } SRpcConnInfo; typedef struct SRpcMsg { - uint8_t msgType; - void *pCont; + tmsg_t msgType; + void * pCont; int contLen; int32_t code; - void *handle; // rpc handle returned to app - void *ahandle; // app handle set by client + void * handle; // rpc handle returned to app + void * ahandle; // app handle set by client } SRpcMsg; typedef struct SRpcInit { - uint16_t localPort; // local port - char *label; // for debug purpose - int numOfThreads; // number of threads to handle connections - int sessions; // number of sessions allowed - int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS - int idleTime; // milliseconds, 0 means idle timer is disabled + uint16_t localPort; // local port + char * label; // for debug purpose + int numOfThreads; // number of threads to handle connections + int sessions; // number of sessions allowed + int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS + int idleTime; // milliseconds, 0 means idle timer is disabled // the following is for client app ecurity only - char *user; // user name - char spi; // security parameter index - char encrypt; // encrypt algorithm - char *secret; // key for authentication - char *ckey; // ciphering key + char *user; // user name + char spi; // security parameter index + char encrypt; // encrypt algorithm + char *secret; // key for authentication + char *ckey; // ciphering key // call back to process incoming msg, code shall be ignored by server app void (*cfp)(void *parent, SRpcMsg *, SEpSet *); @@ -70,19 +70,19 @@ typedef struct SRpcInit { } SRpcInit; int32_t rpcInit(); -void rpcCleanup(); -void *rpcOpen(const SRpcInit *pRpc); -void rpcClose(void *); -void *rpcMallocCont(int contLen); -void rpcFreeCont(void *pCont); -void *rpcReallocCont(void *ptr, int contLen); -void rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid); -void rpcSendResponse(const SRpcMsg *pMsg); -void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet); -int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); -void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); -int rpcReportProgress(void *pConn, char *pCont, int contLen); -void rpcCancelRequest(int64_t rid); +void rpcCleanup(); +void * rpcOpen(const SRpcInit *pRpc); +void rpcClose(void *); +void * rpcMallocCont(int contLen); +void rpcFreeCont(void *pCont); +void * rpcReallocCont(void *ptr, int contLen); +void rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid); +void rpcSendResponse(const SRpcMsg *pMsg); +void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet); +int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); +void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); +int rpcReportProgress(void *pConn, char *pCont, int contLen); +void rpcCancelRequest(int64_t rid); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 4362950844..b6e8964b98 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -220,6 +220,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_COLUMN_ALREAY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AA) #define TSDB_CODE_MND_COLUMN_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AB) #define TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x03AC) +#define TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03AD) // mnode-func #define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03C0) @@ -234,6 +235,15 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TRANS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D0) #define TSDB_CODE_MND_TRANS_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D1) +// mnode-topic +#define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E0) +#define TSDB_CODE_MND_TOPIC_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E1) +#define TSDB_CODE_MND_TOO_MANY_TOPICS TAOS_DEF_ERROR_CODE(0, 0x03E2) +#define TSDB_CODE_MND_INVALID_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E3) +#define TSDB_CODE_MND_INVALID_TOPIC_OPTION TAOS_DEF_ERROR_CODE(0, 0x03E4) +#define TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED TAOS_DEF_ERROR_CODE(0, 0x03E5) +#define TSDB_CODE_MND_NAME_CONFLICT_WITH_STB TAOS_DEF_ERROR_CODE(0, 0x03E6) + // dnode #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) #define TSDB_CODE_DND_EXITING TAOS_DEF_ERROR_CODE(0, 0x0401) diff --git a/include/util/tcoding.h b/include/util/tcoding.h index a2c91c5dbe..6e6a91130c 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -357,6 +357,17 @@ static FORCE_INLINE void *taosDecodeString(void *buf, char **value) { return POINTER_SHIFT(buf, size); } +static FORCE_INLINE void *taosDecodeStringTo(void *buf, char *value) { + uint64_t size = 0; + + buf = taosDecodeVariantU64(buf, &size); + memcpy(value, buf, (size_t)size); + + value[size] = '\0'; + + return POINTER_SHIFT(buf, size); +} + #ifdef __cplusplus } #endif diff --git a/include/util/tdef.h b/include/util/tdef.h index f3f3643268..64a169b4f1 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -163,6 +163,7 @@ do { \ #define TSDB_NODE_NAME_LEN 64 #define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string +#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string #define TSDB_DB_NAME_LEN 65 #define TSDB_FULL_DB_NAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN) @@ -175,6 +176,7 @@ do { \ #define TSDB_TYPE_STR_MAX_LEN 32 #define TSDB_TABLE_FNAME_LEN (TSDB_FULL_DB_NAME_LEN + TSDB_TABLE_NAME_LEN) +#define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN #define TSDB_COL_NAME_LEN 65 #define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index c78bf02cbd..ad6a93415c 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( taos INTERFACE api - PRIVATE os util common transport parser catalog function qcom + PRIVATE os util common transport parser planner catalog scheduler function qcom ) ADD_SUBDIRECTORY(test) \ No newline at end of file diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index eba904c0b8..a57597a60a 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -22,7 +22,7 @@ extern "C" { #include "taos.h" #include "common.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tdef.h" #include "tep.h" #include "thash.h" @@ -116,6 +116,7 @@ typedef struct SRequestObj { int32_t type; // request type STscObj *pTscObj; char *sqlstr; // sql string + int32_t sqlLen; int64_t self; char *msgBuf; void *pInfo; // sql parse info, generated by parser module @@ -128,9 +129,9 @@ extern SAppInfo appInfo; extern int32_t clientReqRefPool; extern int32_t clientConnRefPool; -SMsgSendInfo* buildSendMsgInfoImpl(SRequestObj *pRequest); +extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code); -extern int (*handleRequestRspFp[TSDB_MSG_TYPE_MAX])(void* param, const SDataBuf* pMsg, int32_t code); +SMsgSendInfo* buildSendMsgInfoImpl(SRequestObj*); int taos_init(); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 7fcdafaf44..917ee1dee1 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -18,7 +18,7 @@ #include "clientInt.h" #include "clientLog.h" #include "query.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tcache.h" #include "tconfig.h" #include "tglobal.h" diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index ed211e142c..9fcfb684b4 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -9,6 +9,17 @@ #include "tpagedfile.h" #include "tref.h" #include "parser.h" +#include "planner.h" +#include "scheduler.h" + +#define CHECK_CODE_GOTO(expr, lable) \ + do { \ + int32_t code = expr; \ + if (TSDB_CODE_SUCCESS != code) { \ + terrno = code; \ + goto lable; \ + } \ + } while (0) static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest); @@ -111,11 +122,94 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, return taosConnectImpl(ip, user, &secretEncrypt[0], db, port, NULL, NULL, *pInst); } -static bool supportedQueryType(int32_t type) { - return (type == TSDB_MSG_TYPE_CREATE_USER || type == TSDB_MSG_TYPE_SHOW || type == TSDB_MSG_TYPE_DROP_USER || - type == TSDB_MSG_TYPE_DROP_ACCT || type == TSDB_MSG_TYPE_CREATE_DB || type == TSDB_MSG_TYPE_CREATE_ACCT || - type == TSDB_MSG_TYPE_CREATE_TABLE || type == TSDB_MSG_TYPE_CREATE_STB || type == TSDB_MSG_TYPE_USE_DB || - type == TSDB_MSG_TYPE_DROP_DB || type == TSDB_MSG_TYPE_DROP_STB); +int32_t buildRequest(STscObj *pTscObj, const char *sql, int sqlLen, SRequestObj** pRequest) { + *pRequest = createRequest(pTscObj, NULL, NULL, TSDB_SQL_SELECT); + if (*pRequest == NULL) { + tscError("failed to malloc sqlObj"); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + (*pRequest)->sqlstr = malloc(sqlLen + 1); + if ((*pRequest)->sqlstr == NULL) { + tscError("0x%"PRIx64" failed to prepare sql string buffer", (*pRequest)->self); + (*pRequest)->msgBuf = strdup("failed to prepare sql string buffer"); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + strntolower((*pRequest)->sqlstr, sql, (int32_t)sqlLen); + (*pRequest)->sqlstr[sqlLen] = 0; + (*pRequest)->sqlLen = sqlLen; + + tscDebugL("0x%"PRIx64" SQL: %s", (*pRequest)->requestId, (*pRequest)->sqlstr); + return TSDB_CODE_SUCCESS; +} + +int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery) { + SParseContext cxt = { + .ctx = {.requestId = pRequest->requestId, .acctId = pRequest->pTscObj->acctId, .db = getConnectionDB(pRequest->pTscObj)}, + .pSql = pRequest->sqlstr, + .sqlLen = pRequest->sqlLen, + .pMsg = pRequest->msgBuf, + .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE + }; + int32_t code = qParseQuerySql(&cxt, pQuery); + tfree(cxt.ctx.db); + return code; +} + +int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) { + SDclStmtInfo* pDcl = (SDclStmtInfo*)pQuery; + pRequest->type = pDcl->msgType; + pRequest->body.requestMsg = (SDataBuf){.pData = pDcl->pMsg, .len = pDcl->msgLen}; + + STscObj* pTscObj = pRequest->pTscObj; + + SMsgSendInfo* body = buildSendMsgInfoImpl(pRequest); + SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet; + + if (pDcl->msgType == TDMT_MND_CREATE_TABLE) { + struct SCatalog* pCatalog = NULL; + + char buf[12] = {0}; + sprintf(buf, "%d", pRequest->pTscObj->pAppInfo->clusterId); + int32_t code = catalogGetHandle(buf, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + SCreateTableMsg* pMsg = body->msgInfo.pData; + + SName t = {0}; + tNameFromString(&t, pMsg->name, T_NAME_ACCT|T_NAME_DB|T_NAME_TABLE); + + char db[TSDB_DB_NAME_LEN + TS_PATH_DELIMITER_LEN + TSDB_ACCT_ID_LEN] = {0}; + tNameGetFullDbName(&t, db); + + SVgroupInfo info = {0}; + catalogGetTableHashVgroup(pCatalog, pRequest->pTscObj->pTransporter, pEpSet, db, tNameGetTableName(&t), &info); + + int64_t transporterId = 0; + SEpSet ep = {0}; + ep.inUse = info.inUse; + ep.numOfEps = info.numOfEps; + for(int32_t i = 0; i < ep.numOfEps; ++i) { + ep.port[i] = info.epAddr[i].port; + tstrncpy(ep.fqdn[i], info.epAddr[i].fqdn, tListLen(ep.fqdn[i])); + } + + asyncSendMsgToServer(pTscObj->pTransporter, &ep, &transporterId, body); + } else { + int64_t transporterId = 0; + asyncSendMsgToServer(pTscObj->pTransporter, pEpSet, &transporterId, body); + } + + tsem_wait(&pRequest->body.rspSem); + destroySendMsgInfo(body); + return TSDB_CODE_SUCCESS; +} + +int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) { + return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL/*todo appInfo.xxx*/, pDag, pJob); } TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { @@ -128,94 +222,27 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { nPrintTsc("%s", sql) - SRequestObj* pRequest = createRequest(pTscObj, NULL, NULL, TSDB_SQL_SELECT); - if (pRequest == NULL) { - tscError("failed to malloc sqlObj"); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; - } - - pRequest->sqlstr = malloc(sqlLen + 1); - if (pRequest->sqlstr == NULL) { - tscError("0x%"PRIx64" failed to prepare sql string buffer", pRequest->self); - - pRequest->msgBuf = strdup("failed to prepare sql string buffer"); - terrno = pRequest->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return pRequest; - } - - strntolower(pRequest->sqlstr, sql, (int32_t)sqlLen); - pRequest->sqlstr[sqlLen] = 0; - - tscDebugL("0x%"PRIx64" SQL: %s", pRequest->requestId, pRequest->sqlstr); - - SParseContext cxt = { - .ctx = {.requestId = pRequest->requestId, .acctId = pTscObj->acctId, .db = getConnectionDB(pTscObj)}, - .pSql = pRequest->sqlstr, - .sqlLen = sqlLen, - .pMsg = pRequest->msgBuf, - .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE - }; - + SRequestObj* pRequest = NULL; SQueryNode* pQuery = NULL; - int32_t code = qParseQuerySql(&cxt, &pQuery); - if (qIsDclQuery(pQuery)) { - SDclStmtInfo* pDcl = (SDclStmtInfo*) pQuery; + SQueryDag* pDag = NULL; + void* pJob = NULL; - pRequest->type = pDcl->msgType; - pRequest->body.requestMsg = (SDataBuf){.pData = pDcl->pMsg, .len = pDcl->msgLen}; - - SMsgSendInfo* body = buildSendMsgInfoImpl(pRequest); - SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet; - - if (pDcl->msgType == TSDB_MSG_TYPE_CREATE_TABLE) { - struct SCatalog* pCatalog = NULL; - - char buf[12] = {0}; - sprintf(buf, "%d", pTscObj->pAppInfo->clusterId); - code = catalogGetHandle(buf, &pCatalog); - if (code != 0) { - pRequest->code = code; - return pRequest; - } - - SCreateTableMsg* pMsg = body->msgInfo.pData; - - SName t = {0}; - tNameFromString(&t, pMsg->name, T_NAME_ACCT|T_NAME_DB|T_NAME_TABLE); - - char db[TSDB_DB_NAME_LEN + TS_PATH_DELIMITER_LEN + TSDB_ACCT_ID_LEN] = {0}; - tNameGetFullDbName(&t, db); - - SVgroupInfo info = {0}; - catalogGetTableHashVgroup(pCatalog, pTscObj->pTransporter, pEpSet, db, tNameGetTableName(&t), &info); - - int64_t transporterId = 0; - SEpSet ep = {0}; - ep.inUse = info.inUse; - ep.numOfEps = info.numOfEps; - for(int32_t i = 0; i < ep.numOfEps; ++i) { - ep.port[i] = info.epAddr[i].port; - tstrncpy(ep.fqdn[i], info.epAddr[i].fqdn, tListLen(ep.fqdn[i])); - } - - asyncSendMsgToServer(pTscObj->pTransporter, &ep, &transporterId, body); - } else { - int64_t transporterId = 0; - asyncSendMsgToServer(pTscObj->pTransporter, pEpSet, &transporterId, body); - } - - tsem_wait(&pRequest->body.rspSem); - destroySendMsgInfo(body); + terrno = TSDB_CODE_SUCCESS; + CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); + CHECK_CODE_GOTO(parseSql(pRequest, &pQuery), _return); + if (qIsDdlQuery(pQuery)) { + CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return); + goto _return; } + CHECK_CODE_GOTO(qCreateQueryDag(pQuery, &pDag), _return); + CHECK_CODE_GOTO(scheduleQuery(pRequest, pDag, &pJob), _return); - tfree(cxt.ctx.db); - - if (code != TSDB_CODE_SUCCESS) { - pRequest->code = code; - return pRequest; +_return: + qDestoryQuery(pQuery); + qDestroyQueryDag(pDag); + if (NULL != pRequest && TSDB_CODE_SUCCESS != terrno) { + pRequest->code = terrno; } - return pRequest; } @@ -262,7 +289,7 @@ STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, con return pTscObj; } - SRequestObj *pRequest = createRequest(pTscObj, fp, param, TSDB_MSG_TYPE_CONNECT); + SRequestObj *pRequest = createRequest(pTscObj, fp, param, TDMT_MND_CONNECT); if (pRequest == NULL) { destroyTscObj(pTscObj); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -299,7 +326,7 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest) { return NULL; } - pMsgSendInfo->msgType = TSDB_MSG_TYPE_CONNECT; + pMsgSendInfo->msgType = TDMT_MND_CONNECT; pMsgSendInfo->msgInfo.len = sizeof(SConnectMsg); pMsgSendInfo->requestObjRefId = pRequest->self; pMsgSendInfo->requestId = pRequest->requestId; @@ -336,7 +363,7 @@ static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { int32_t asyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo) { char *pMsg = rpcMallocCont(pInfo->msgInfo.len); if (NULL == pMsg) { - tscError("0x%"PRIx64" msg:%s malloc failed", pInfo->requestId, taosMsg[pInfo->msgType]); + tscError("0x%"PRIx64" msg:%s malloc failed", pInfo->requestId, TMSG_INFO(pInfo->msgType)); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return -1; } @@ -381,11 +408,11 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { */ if (pMsg->code == TSDB_CODE_SUCCESS) { tscDebug("0x%" PRIx64 " message:%s, code:%s rspLen:%d, elapsed:%" PRId64 " ms", pRequest->requestId, - taosMsg[pMsg->msgType], tstrerror(pMsg->code), pMsg->contLen, + TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), pMsg->contLen, pRequest->metric.rsp - pRequest->metric.start); } else { tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d, elapsed time:%" PRId64 " ms", pRequest->requestId, - taosMsg[pMsg->msgType], tstrerror(pMsg->code), pMsg->contLen, + TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), pMsg->contLen, pRequest->metric.rsp - pRequest->metric.start); } @@ -429,7 +456,7 @@ void* doFetchRow(SRequestObj* pRequest) { SReqResultInfo* pResultInfo = &pRequest->body.resInfo; if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { - pRequest->type = TSDB_MSG_TYPE_SHOW_RETRIEVE; + pRequest->type = TDMT_MND_SHOW_RETRIEVE; SMsgSendInfo* body = buildSendMsgInfoImpl(pRequest); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index af85f1b310..894b810d7b 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -2,7 +2,7 @@ #include "clientInt.h" #include "clientLog.h" #include "query.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tglobal.h" #include "tref.h" #include "trpc.h" diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 700a7a20cf..42e1f2941b 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -20,7 +20,7 @@ #include "clientLog.h" #include "trpc.h" -int (*handleRequestRspFp[TSDB_MSG_TYPE_MAX])(void*, const SDataBuf* pMsg, int32_t code); +int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; @@ -67,7 +67,7 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { } static int32_t buildRetrieveMnodeMsg(SRequestObj *pRequest, SMsgSendInfo* pMsgSendInfo) { - pMsgSendInfo->msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + pMsgSendInfo->msgType = TDMT_MND_SHOW_RETRIEVE; pMsgSendInfo->msgInfo.len = sizeof(SRetrieveTableMsg); pMsgSendInfo->requestObjRefId = pRequest->self; pMsgSendInfo->param = pRequest; @@ -86,7 +86,7 @@ static int32_t buildRetrieveMnodeMsg(SRequestObj *pRequest, SMsgSendInfo* pMsgSe SMsgSendInfo* buildSendMsgInfoImpl(SRequestObj *pRequest) { SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); - if (pRequest->type == TSDB_MSG_TYPE_SHOW_RETRIEVE) { + if (pRequest->type == TDMT_MND_SHOW_RETRIEVE) { buildRetrieveMnodeMsg(pRequest, pMsgSendInfo); } else { assert(pRequest != NULL); @@ -279,11 +279,11 @@ void initMsgHandleFp() { tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp; #endif - handleRequestRspFp[TSDB_MSG_TYPE_CONNECT] = processConnectRsp; - handleRequestRspFp[TSDB_MSG_TYPE_SHOW] = processShowRsp; - handleRequestRspFp[TSDB_MSG_TYPE_SHOW_RETRIEVE] = processRetrieveMnodeRsp; - handleRequestRspFp[TSDB_MSG_TYPE_CREATE_DB] = processCreateDbRsp; - handleRequestRspFp[TSDB_MSG_TYPE_USE_DB] = processUseDbRsp; - handleRequestRspFp[TSDB_MSG_TYPE_CREATE_TABLE] = processCreateTableRsp; - handleRequestRspFp[TSDB_MSG_TYPE_DROP_DB] = processDropDbRsp; + handleRequestRspFp[TDMT_MND_CONNECT] = processConnectRsp; + handleRequestRspFp[TDMT_MND_SHOW] = processShowRsp; + handleRequestRspFp[TDMT_MND_SHOW_RETRIEVE] = processRetrieveMnodeRsp; + handleRequestRspFp[TDMT_MND_CREATE_DB] = processCreateDbRsp; + handleRequestRspFp[TDMT_MND_USE_DB] = processUseDbRsp; + handleRequestRspFp[TDMT_MND_CREATE_TABLE] = processCreateTableRsp; + handleRequestRspFp[TDMT_MND_DROP_DB] = processDropDbRsp; } \ No newline at end of file diff --git a/source/common/src/taosmsg.c b/source/common/src/tmsg.c similarity index 73% rename from source/common/src/taosmsg.c rename to source/common/src/tmsg.c index b35e3f1478..a7f285046c 100644 --- a/source/common/src/taosmsg.c +++ b/source/common/src/tmsg.c @@ -13,8 +13,16 @@ * along with this program. If not, see . */ -#define TAOS_MESSAGE_C - -#include "taosmsg.h" +#include "tmsg.h" +#undef TD_MSG_NUMBER_ +#undef TD_MSG_DICT_ +#define TD_MSG_INFO_ +#undef TD_MSG_SEG_CODE_ +#include "tmsgdef.h" +#undef TD_MSG_NUMBER_ +#undef TD_MSG_INFO_ +#define TD_MSG_DICT_ +#undef TD_MSG_SEG_CODE_ +#include "tmsgdef.h" \ No newline at end of file diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 9661016483..e24c8f7a81 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -2,7 +2,7 @@ #include "tutil.h" #include "tname.h" -#include "taosmsg.h" +#include "tmsg.h" #define VALID_NAME_TYPE(x) ((x) == TSDB_DB_NAME_T || (x) == TSDB_TABLE_NAME_T) diff --git a/source/common/test/CMakeLists.txt b/source/common/test/CMakeLists.txt index f4c849780d..03ca95ff64 100644 --- a/source/common/test/CMakeLists.txt +++ b/source/common/test/CMakeLists.txt @@ -16,3 +16,13 @@ TARGET_INCLUDE_DIRECTORIES( PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/common/" PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/common/inc" ) + +# tmsg test +add_executable(tmsgTest "") +target_sources(tmsgTest + PRIVATE + "tmsgTest.cpp" + "../src/tmsg.c" +) +target_include_directories(tmsgTest PUBLIC "${CMAKE_SOURCE_DIR}/include/common/") +target_link_libraries(tmsgTest PUBLIC os util gtest gtest_main) \ No newline at end of file diff --git a/source/common/test/tmsgTest.cpp b/source/common/test/tmsgTest.cpp new file mode 100644 index 0000000000..ca33d24a8c --- /dev/null +++ b/source/common/test/tmsgTest.cpp @@ -0,0 +1,11 @@ +#include + +#include "gtest/gtest.h" + +#include "tmsg.h" + +TEST(td_msg_test, simple_msg_test) { + // std::cout << TMSG_INFO(TDMT_VND_DROP_TABLE) << std::endl; + // std::cout << TMSG_INFO(TDMT_MND_DROP_SUPER_TABLE) << std::endl; + // std::cout << TMSG_INFO(TDMT_MND_CREATE_SUPER_TABLE) << std::endl; +} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index d8acda530d..6f1357e9c1 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -22,7 +22,7 @@ extern "C" { #include "cJSON.h" #include "os.h" -#include "taosmsg.h" +#include "tmsg.h" #include "thash.h" #include "tlockfree.h" #include "tlog.h" @@ -80,7 +80,6 @@ typedef struct { SRWLatch latch; taos_queue pReadQ; taos_queue pWriteQ; - taos_queue pApplyQ; taos_queue pSyncQ; taos_queue pMgmtQ; SWorkerPool mgmtPool; @@ -105,7 +104,7 @@ typedef struct { typedef struct { void *serverRpc; void *clientRpc; - DndMsgFp msgFp[TSDB_MSG_TYPE_MAX]; + DndMsgFp msgFp[TDMT_MAX]; } STransMgmt; typedef struct SDnode { diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index 9e2d4d4c9b..af86e59518 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -63,12 +63,12 @@ void dndGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { } void dndSendRedirectMsg(SDnode *pDnode, SRpcMsg *pMsg) { - int32_t msgType = pMsg->msgType; + tmsg_t msgType = pMsg->msgType; SEpSet epSet = {0}; dndGetMnodeEpSet(pDnode, &epSet); - dDebug("RPC %p, msg:%s is redirected, num:%d inUse:%d", pMsg->handle, taosMsg[msgType], epSet.numOfEps, epSet.inUse); + dDebug("RPC %p, msg:%s is redirected, num:%d inUse:%d", pMsg->handle, TMSG_INFO(msgType), epSet.numOfEps, epSet.inUse); for (int32_t i = 0; i < epSet.numOfEps; ++i) { dDebug("mnode index:%d %s:%u", i, epSet.fqdn[i], epSet.port[i]); if (strcmp(epSet.fqdn[i], pDnode->opt.localFqdn) == 0 && epSet.port[i] == pDnode->opt.serverPort) { @@ -369,7 +369,7 @@ void dndSendStatusMsg(SDnode *pDnode) { dndGetVnodeLoads(pDnode, &pStatus->vnodeLoads); contLen = sizeof(SStatusMsg) + pStatus->vnodeLoads.num * sizeof(SVnodeLoad); - SRpcMsg rpcMsg = {.pCont = pStatus, .contLen = contLen, .msgType = TSDB_MSG_TYPE_STATUS}; + SRpcMsg rpcMsg = {.pCont = pStatus, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)9527}; pMgmt->statusSent = 1; dTrace("pDnode:%p, send status msg to mnode", pDnode); @@ -556,14 +556,14 @@ void dndCleanupDnode(SDnode *pDnode) { void dndProcessDnodeReq(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { switch (pMsg->msgType) { - case TSDB_MSG_TYPE_NETWORK_TEST: + case TDMT_DND_NETWORK_TEST: dndProcessStartupReq(pDnode, pMsg); break; - case TSDB_MSG_TYPE_CONFIG_DNODE_IN: + case TDMT_DND_CONFIG_DNODE: dndProcessConfigDnodeReq(pDnode, pMsg); break; default: - dError("RPC %p, dnode req:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]); + dError("RPC %p, dnode req:%s not processed", pMsg->handle, TMSG_INFO(pMsg->msgType)); SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED}; rpcSendResponse(&rspMsg); } @@ -574,17 +574,17 @@ void dndProcessDnodeReq(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { void dndProcessDnodeRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { switch (pMsg->msgType) { - case TSDB_MSG_TYPE_STATUS_RSP: + case TDMT_MND_STATUS_RSP: dndProcessStatusRsp(pDnode, pMsg, pEpSet); break; - case TSDB_MSG_TYPE_AUTH_RSP: + case TDMT_MND_AUTH_RSP: dndProcessAuthRsp(pDnode, pMsg, pEpSet); break; - case TSDB_MSG_TYPE_GRANT_RSP: + case TDMT_MND_GRANT_RSP: dndProcessGrantRsp(pDnode, pMsg, pEpSet); break; default: - dError("RPC %p, dnode rsp:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]); + dError("RPC %p, dnode rsp:%s not processed", pMsg->handle, TMSG_INFO(pMsg->msgType)); } rpcFreeCont(pMsg->pCont); diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index c4d69c4626..8fbb473af1 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -28,18 +28,15 @@ static void dndCleanupMnodeSyncWorker(SDnode *pDnode); static void dndCleanupMnodeMgmtWorker(SDnode *pDnode); static int32_t dndAllocMnodeReadQueue(SDnode *pDnode); static int32_t dndAllocMnodeWriteQueue(SDnode *pDnode); -static int32_t dndAllocMnodeApplyQueue(SDnode *pDnode); static int32_t dndAllocMnodeSyncQueue(SDnode *pDnode); static int32_t dndAllocMnodeMgmtQueue(SDnode *pDnode); static void dndFreeMnodeReadQueue(SDnode *pDnode); static void dndFreeMnodeWriteQueue(SDnode *pDnode); -static void dndFreeMnodeApplyQueue(SDnode *pDnode); static void dndFreeMnodeSyncQueue(SDnode *pDnode); static void dndFreeMnodeMgmtQueue(SDnode *pDnode); static void dndProcessMnodeReadQueue(SDnode *pDnode, SMnodeMsg *pMsg); static void dndProcessMnodeWriteQueue(SDnode *pDnode, SMnodeMsg *pMsg); -static void dndProcessMnodeApplyQueue(SDnode *pDnode, SMnodeMsg *pMsg); static void dndProcessMnodeSyncQueue(SDnode *pDnode, SMnodeMsg *pMsg); static void dndProcessMnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg); static int32_t dndWriteMnodeMsgToQueue(SMnode *pMnode, taos_queue pQueue, SRpcMsg *pRpcMsg); @@ -47,7 +44,6 @@ void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEp void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); void dndProcessMnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -static int32_t dndPutMsgIntoMnodeApplyQueue(SDnode *pDnode, SMnodeMsg *pMsg); static int32_t dndStartMnodeWorker(SDnode *pDnode); static void dndStopMnodeWorker(SDnode *pDnode); @@ -271,11 +267,6 @@ static int32_t dndStartMnodeWorker(SDnode *pDnode) { return -1; } - if (dndAllocMnodeApplyQueue(pDnode) != 0) { - dError("failed to alloc mnode apply queue since %s", terrstr()); - return -1; - } - if (dndAllocMnodeSyncQueue(pDnode) != 0) { dError("failed to alloc mnode sync queue since %s", terrstr()); return -1; @@ -293,7 +284,6 @@ static void dndStopMnodeWorker(SDnode *pDnode) { while (pMgmt->refCount > 1) taosMsleep(10); while (!taosQueueEmpty(pMgmt->pReadQ)) taosMsleep(10); - while (!taosQueueEmpty(pMgmt->pApplyQ)) taosMsleep(10); while (!taosQueueEmpty(pMgmt->pWriteQ)) taosMsleep(10); while (!taosQueueEmpty(pMgmt->pSyncQ)) taosMsleep(10); @@ -303,7 +293,6 @@ static void dndStopMnodeWorker(SDnode *pDnode) { dndFreeMnodeReadQueue(pDnode); dndFreeMnodeWriteQueue(pDnode); - dndFreeMnodeApplyQueue(pDnode); dndFreeMnodeSyncQueue(pDnode); } @@ -328,7 +317,6 @@ static void dndInitMnodeOption(SDnode *pDnode, SMnodeOpt *pOption) { pOption->sendMsgToDnodeFp = dndSendMsgToDnode; pOption->sendMsgToMnodeFp = dndSendMsgToMnode; pOption->sendRedirectMsgFp = dndSendRedirectMsg; - pOption->putMsgToApplyMsgFp = dndPutMsgIntoMnodeApplyQueue; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); pOption->cfg.sver = pDnode->opt.sver; @@ -552,13 +540,13 @@ static void dndProcessMnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) { int32_t code = 0; switch (pMsg->msgType) { - case TSDB_MSG_TYPE_CREATE_MNODE_IN: + case TDMT_DND_CREATE_MNODE: code = dndProcessCreateMnodeReq(pDnode, pMsg); break; - case TSDB_MSG_TYPE_ALTER_MNODE_IN: + case TDMT_DND_ALTER_MNODE: code = dndProcessAlterMnodeReq(pDnode, pMsg); break; - case TSDB_MSG_TYPE_DROP_MNODE_IN: + case TDMT_DND_DROP_MNODE: code = dndProcessDropMnodeReq(pDnode, pMsg); break; default: @@ -604,20 +592,6 @@ static void dndProcessMnodeWriteQueue(SDnode *pDnode, SMnodeMsg *pMsg) { mndCleanupMsg(pMsg); } -static void dndProcessMnodeApplyQueue(SDnode *pDnode, SMnodeMsg *pMsg) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode != NULL) { - mndProcessApplyMsg(pMsg); - dndReleaseMnode(pDnode, pMnode); - } else { - mndSendRsp(pMsg, terrno); - } - - mndCleanupMsg(pMsg); -} - static void dndProcessMnodeSyncQueue(SDnode *pDnode, SMnodeMsg *pMsg) { SMnodeMgmt *pMgmt = &pDnode->mmgmt; @@ -712,19 +686,6 @@ void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { dndReleaseMnode(pDnode, pMnode); } -static int32_t dndPutMsgIntoMnodeApplyQueue(SDnode *pDnode, SMnodeMsg *pMsg) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL) { - return -1; - } - - int32_t code = taosWriteQitem(pMgmt->pApplyQ, pMsg); - dndReleaseMnode(pDnode, pMnode); - return code; -} - static int32_t dndAllocMnodeMgmtQueue(SDnode *pDnode) { SMnodeMgmt *pMgmt = &pDnode->mmgmt; pMgmt->pMgmtQ = tWorkerAllocQueue(&pMgmt->mgmtPool, pDnode, (FProcessItem)dndProcessMnodeMgmtQueue); @@ -817,23 +778,6 @@ static void dndFreeMnodeWriteQueue(SDnode *pDnode) { pMgmt->pWriteQ = NULL; } -static int32_t dndAllocMnodeApplyQueue(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - pMgmt->pApplyQ = tWorkerAllocQueue(&pMgmt->writePool, pDnode, (FProcessItem)dndProcessMnodeApplyQueue); - if (pMgmt->pApplyQ == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static void dndFreeMnodeApplyQueue(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - tWorkerFreeQueue(&pMgmt->writePool, pMgmt->pApplyQ); - pMgmt->pApplyQ = NULL; -} - static int32_t dndInitMnodeWriteWorker(SDnode *pDnode) { SMnodeMgmt *pMgmt = &pDnode->mmgmt; SWorkerPool *pPool = &pMgmt->writePool; diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 3d797eba8f..b8ebe3f884 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -31,118 +31,118 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { // msg from client to dnode - pMgmt->msgFp[TSDB_MSG_TYPE_SUBMIT] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_QUERY] = dndProcessVnodeQueryMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_FETCH] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_TABLE] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_TABLE] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_TABLE] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_TABLE_META] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_TABLES_META] = dndProcessVnodeQueryMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_MQ_QUERY] = dndProcessVnodeQueryMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONSUME] = dndProcessVnodeQueryMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONNECT] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_MQ_DISCONNECT] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_MQ_SET_CUR] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_RES_READY] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_TASKS_STATUS] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_CANCEL_TASK] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_TASK] = dndProcessVnodeFetchMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBMIT)] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY)] = dndProcessVnodeQueryMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH)] = dndProcessVnodeFetchMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_TABLE)] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_TABLE)] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_UPDATE_TAG_VAL)] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLE_META)] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLES_META)] = dndProcessVnodeQueryMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_QUERY)] = dndProcessVnodeQueryMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_CONSUME)] = dndProcessVnodeQueryMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_CONNECT)] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_DISCONNECT)] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_RES_READY)] = dndProcessVnodeFetchMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TASKS_STATUS)] = dndProcessVnodeFetchMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CANCEL_TASK)] = dndProcessVnodeFetchMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_TASK)] = dndProcessVnodeFetchMsg; // msg from client to mnode - pMgmt->msgFp[TSDB_MSG_TYPE_CONNECT] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_ACCT] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_ACCT] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_ACCT] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_USER] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_USER] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_USER] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_DNODE] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_CONFIG_DNODE] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_DNODE] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_MNODE] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_MNODE] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_DB] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_DB] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_USE_DB] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_DB] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_SYNC_DB] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_COMPACT_DB] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_FUNCTION] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_RETRIEVE_FUNCTION] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_FUNCTION] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STB] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STB] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STB] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_VGROUP_LIST] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_KILL_QUERY] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_KILL_CONN] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_HEARTBEAT] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_SHOW] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_SHOW_RETRIEVE] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CONNECT)] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_ACCT)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_ACCT)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_ACCT)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_USER)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_USER)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_USER)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_DNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CONFIG_DNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_DNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_MNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_MNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_DB)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_USE_DB)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_DB)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SYNC_DB)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_COMPACT_DB)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_FUNCTION)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNCTION)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_FUNCTION)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_STB)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_STB)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_VGROUP_LIST)] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_QUERY)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_CONN)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_HEARTBEAT)] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW)] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW_RETRIEVE)] = dndProcessMnodeReadMsg; // message from client to dnode - pMgmt->msgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dndProcessDnodeReq; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_NETWORK_TEST)] = dndProcessDnodeReq; // message from mnode to vnode - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STB_IN] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STB_IN_RSP] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STB_IN] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STB_IN_RSP] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STB_IN] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STB_IN_RSP] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CREATE_STB)] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CREATE_STB_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_STB)] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_STB_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_STB)] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_STB_RSP)] = dndProcessMnodeWriteMsg; // message from mnode to dnode - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_VNODE_IN] = dndProcessVnodeMgmtMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_VNODE_IN_RSP] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_VNODE_IN] = dndProcessVnodeMgmtMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_VNODE_IN_RSP] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_VNODE_IN] = dndProcessVnodeMgmtMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_VNODE_IN_RSP] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_SYNC_VNODE_IN] = dndProcessVnodeMgmtMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_SYNC_VNODE_IN_RSP] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_AUTH_VNODE_IN] = dndProcessVnodeMgmtMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_AUTH_VNODE_IN_RSP] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_COMPACT_VNODE_IN] = dndProcessVnodeMgmtMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_COMPACT_VNODE_IN_RSP] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_MNODE_IN] = dndProcessMnodeMgmtMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_MNODE_IN_RSP] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_MNODE_IN] = dndProcessMnodeMgmtMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_MNODE_IN_RSP] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_MNODE_IN] = dndProcessMnodeMgmtMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_MNODE_IN_RSP] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_CONFIG_DNODE_IN] = dndProcessDnodeReq; - pMgmt->msgFp[TSDB_MSG_TYPE_CONFIG_DNODE_IN_RSP] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_VNODE)] = dndProcessVnodeMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_VNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_VNODE)] = dndProcessVnodeMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_VNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_VNODE)] = dndProcessVnodeMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_VNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_SYNC_VNODE)] = dndProcessVnodeMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_SYNC_VNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_AUTH_VNODE)] = dndProcessVnodeMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_AUTH_VNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_COMPACT_VNODE)] = dndProcessVnodeMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_COMPACT_VNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_MNODE)] = dndProcessMnodeMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_MNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_MNODE)] = dndProcessMnodeMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_MNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_MNODE)] = dndProcessMnodeMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_MNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CONFIG_DNODE)] = dndProcessDnodeReq; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CONFIG_DNODE_RSP)] = dndProcessMnodeWriteMsg; // message from dnode to mnode - pMgmt->msgFp[TSDB_MSG_TYPE_GRANT] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_GRANT_RSP] = dndProcessDnodeRsp; - pMgmt->msgFp[TSDB_MSG_TYPE_STATUS] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_STATUS_RSP] = dndProcessDnodeRsp; - pMgmt->msgFp[TSDB_MSG_TYPE_AUTH] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_AUTH_RSP] = dndProcessDnodeRsp; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GRANT)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GRANT_RSP)] = dndProcessDnodeRsp; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS_RSP)] = dndProcessDnodeRsp; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_AUTH)] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_AUTH_RSP)] = dndProcessDnodeRsp; } static void dndProcessResponse(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { SDnode *pDnode = parent; STransMgmt *pMgmt = &pDnode->tmgmt; - int32_t msgType = pMsg->msgType; + tmsg_t msgType = pMsg->msgType; if (dndGetStat(pDnode) == DND_STAT_STOPPED) { if (pMsg == NULL || pMsg->pCont == NULL) return; - dTrace("RPC %p, rsp:%s is ignored since dnode is stopping", pMsg->handle, taosMsg[msgType]); + dTrace("RPC %p, rsp:%s is ignored since dnode is stopping", pMsg->handle, TMSG_INFO(msgType)); rpcFreeCont(pMsg->pCont); return; } - DndMsgFp fp = pMgmt->msgFp[msgType]; + DndMsgFp fp = pMgmt->msgFp[TMSG_INDEX(msgType)]; if (fp != NULL) { (*fp)(pDnode, pMsg, pEpSet); - dTrace("RPC %p, rsp:%s is processed, code:0x%x", pMsg->handle, taosMsg[msgType], pMsg->code & 0XFFFF); + dTrace("RPC %p, rsp:%s is processed, code:0x%x", pMsg->handle, TMSG_INFO(msgType), pMsg->code & 0XFFFF); } else { - dError("RPC %p, rsp:%s not processed", pMsg->handle, taosMsg[msgType]); + dError("RPC %p, rsp:%s not processed", pMsg->handle, TMSG_INFO(msgType)); rpcFreeCont(pMsg->pCont); } } @@ -186,21 +186,21 @@ static void dndProcessRequest(void *param, SRpcMsg *pMsg, SEpSet *pEpSet) { SDnode *pDnode = param; STransMgmt *pMgmt = &pDnode->tmgmt; - int32_t msgType = pMsg->msgType; - if (msgType == TSDB_MSG_TYPE_NETWORK_TEST) { + tmsg_t msgType = pMsg->msgType; + if (msgType == TDMT_DND_NETWORK_TEST) { dTrace("RPC %p, network test req, app:%p will be processed, code:0x%x", pMsg->handle, pMsg->ahandle, pMsg->code); dndProcessDnodeReq(pDnode, pMsg, pEpSet); return; } if (dndGetStat(pDnode) == DND_STAT_STOPPED) { - dError("RPC %p, req:%s app:%p is ignored since dnode exiting", pMsg->handle, taosMsg[msgType], pMsg->ahandle); + dError("RPC %p, req:%s app:%p is ignored since dnode exiting", pMsg->handle, TMSG_INFO(msgType), pMsg->ahandle); SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_DND_EXITING}; rpcSendResponse(&rspMsg); rpcFreeCont(pMsg->pCont); return; } else if (dndGetStat(pDnode) != DND_STAT_RUNNING) { - dError("RPC %p, req:%s app:%p is ignored since dnode not running", pMsg->handle, taosMsg[msgType], pMsg->ahandle); + dError("RPC %p, req:%s app:%p is ignored since dnode not running", pMsg->handle, TMSG_INFO(msgType), pMsg->ahandle); SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_APP_NOT_READY}; rpcSendResponse(&rspMsg); rpcFreeCont(pMsg->pCont); @@ -208,18 +208,18 @@ static void dndProcessRequest(void *param, SRpcMsg *pMsg, SEpSet *pEpSet) { } if (pMsg->pCont == NULL) { - dTrace("RPC %p, req:%s app:%p not processed since content is null", pMsg->handle, taosMsg[msgType], pMsg->ahandle); + dTrace("RPC %p, req:%s app:%p not processed since content is null", pMsg->handle, TMSG_INFO(msgType), pMsg->ahandle); SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN}; rpcSendResponse(&rspMsg); return; } - DndMsgFp fp = pMgmt->msgFp[msgType]; + DndMsgFp fp = pMgmt->msgFp[TMSG_INDEX(msgType)]; if (fp != NULL) { - dTrace("RPC %p, req:%s app:%p will be processed", pMsg->handle, taosMsg[msgType], pMsg->ahandle); + dTrace("RPC %p, req:%s app:%p will be processed", pMsg->handle, TMSG_INFO(msgType), pMsg->ahandle); (*fp)(pDnode, pMsg, pEpSet); } else { - dError("RPC %p, req:%s app:%p is not processed since no handle", pMsg->handle, taosMsg[msgType], pMsg->ahandle); + dError("RPC %p, req:%s app:%p is not processed since no handle", pMsg->handle, TMSG_INFO(msgType), pMsg->ahandle); SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED}; rpcSendResponse(&rspMsg); rpcFreeCont(pMsg->pCont); @@ -281,7 +281,7 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char SAuthMsg *pMsg = rpcMallocCont(sizeof(SAuthMsg)); tstrncpy(pMsg->user, user, TSDB_USER_LEN); - SRpcMsg rpcMsg = {.pCont = pMsg, .contLen = sizeof(SAuthMsg), .msgType = TSDB_MSG_TYPE_AUTH}; + SRpcMsg rpcMsg = {.pCont = pMsg, .contLen = sizeof(SAuthMsg), .msgType = TDMT_MND_AUTH}; SRpcMsg rpcRsp = {0}; dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp); diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index 598d6e47be..351fc20784 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -773,22 +773,22 @@ static void dndProcessVnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) { int32_t code = 0; switch (pMsg->msgType) { - case TSDB_MSG_TYPE_CREATE_VNODE_IN: + case TDMT_DND_CREATE_VNODE: code = dndProcessCreateVnodeReq(pDnode, pMsg); break; - case TSDB_MSG_TYPE_ALTER_VNODE_IN: + case TDMT_DND_ALTER_VNODE: code = dndProcessAlterVnodeReq(pDnode, pMsg); break; - case TSDB_MSG_TYPE_DROP_VNODE_IN: + case TDMT_DND_DROP_VNODE: code = dndProcessDropVnodeReq(pDnode, pMsg); break; - case TSDB_MSG_TYPE_AUTH_VNODE_IN: + case TDMT_DND_AUTH_VNODE: code = dndProcessAuthVnodeReq(pDnode, pMsg); break; - case TSDB_MSG_TYPE_SYNC_VNODE_IN: + case TDMT_DND_SYNC_VNODE: code = dndProcessSyncVnodeReq(pDnode, pMsg); break; - case TSDB_MSG_TYPE_COMPACT_VNODE_IN: + case TDMT_DND_COMPACT_VNODE: code = dndProcessCompactVnodeReq(pDnode, pMsg); break; default: @@ -829,6 +829,7 @@ static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); int32_t code = vnodeApplyWMsg(pVnode->pImpl, pMsg, &pRsp); if (pRsp != NULL) { + pRsp->ahandle = pMsg->ahandle; rpcSendResponse(pRsp); free(pRsp); } else { diff --git a/source/dnode/mgmt/impl/test/acct/acct.cpp b/source/dnode/mgmt/impl/test/acct/acct.cpp index 69ddae675d..be0ad4ab0f 100644 --- a/source/dnode/mgmt/impl/test/acct/acct.cpp +++ b/source/dnode/mgmt/impl/test/acct/acct.cpp @@ -30,7 +30,7 @@ TEST_F(DndTestAcct, 01_CreateAcct) { SCreateAcctMsg* pReq = (SCreateAcctMsg*)rpcMallocCont(contLen); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_ACCT, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_ACCT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); } @@ -40,7 +40,7 @@ TEST_F(DndTestAcct, 02_AlterAcct) { SAlterAcctMsg* pReq = (SAlterAcctMsg*)rpcMallocCont(contLen); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_ALTER_ACCT, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_ALTER_ACCT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); } @@ -50,7 +50,7 @@ TEST_F(DndTestAcct, 03_DropAcct) { SDropAcctMsg* pReq = (SDropAcctMsg*)rpcMallocCont(contLen); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_ACCT, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_ACCT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); } @@ -61,7 +61,7 @@ TEST_F(DndTestAcct, 04_ShowAcct) { SShowMsg* pReq = (SShowMsg*)rpcMallocCont(contLen); pReq->type = TSDB_MGMT_TABLE_ACCT; - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_SHOW, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_SHOW, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_MSG_TYPE); } \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/db/db.cpp b/source/dnode/mgmt/impl/test/db/db.cpp index 823f58e3a8..5c67c10817 100644 --- a/source/dnode/mgmt/impl/test/db/db.cpp +++ b/source/dnode/mgmt/impl/test/db/db.cpp @@ -27,24 +27,25 @@ Testbase DndTestDb::test; TEST_F(DndTestDb, 01_ShowDb) { test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); - CHECK_META("show databases", 17); + CHECK_META("show databases", 18); CHECK_SCHEMA(0, TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN - 1 + VARSTR_HEADER_SIZE, "name"); CHECK_SCHEMA(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); CHECK_SCHEMA(2, TSDB_DATA_TYPE_SMALLINT, 2, "vgroups"); - CHECK_SCHEMA(3, TSDB_DATA_TYPE_SMALLINT, 2, "replica"); - CHECK_SCHEMA(4, TSDB_DATA_TYPE_SMALLINT, 2, "quorum"); - CHECK_SCHEMA(5, TSDB_DATA_TYPE_SMALLINT, 2, "days"); - CHECK_SCHEMA(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "keep0,keep1,keep2"); - CHECK_SCHEMA(7, TSDB_DATA_TYPE_INT, 4, "cache"); - CHECK_SCHEMA(8, TSDB_DATA_TYPE_INT, 4, "blocks"); - CHECK_SCHEMA(9, TSDB_DATA_TYPE_INT, 4, "minrows"); - CHECK_SCHEMA(10, TSDB_DATA_TYPE_INT, 4, "maxrows"); - CHECK_SCHEMA(11, TSDB_DATA_TYPE_TINYINT, 1, "wallevel"); - CHECK_SCHEMA(12, TSDB_DATA_TYPE_INT, 4, "fsync"); - CHECK_SCHEMA(13, TSDB_DATA_TYPE_TINYINT, 1, "comp"); - CHECK_SCHEMA(14, TSDB_DATA_TYPE_TINYINT, 1, "cachelast"); - CHECK_SCHEMA(15, TSDB_DATA_TYPE_BINARY, 3 + VARSTR_HEADER_SIZE, "precision"); - CHECK_SCHEMA(16, TSDB_DATA_TYPE_TINYINT, 1, "update"); + CHECK_SCHEMA(3, TSDB_DATA_TYPE_INT, 4, "ntables"); + CHECK_SCHEMA(4, TSDB_DATA_TYPE_SMALLINT, 2, "replica"); + CHECK_SCHEMA(5, TSDB_DATA_TYPE_SMALLINT, 2, "quorum"); + CHECK_SCHEMA(6, TSDB_DATA_TYPE_SMALLINT, 2, "days"); + CHECK_SCHEMA(7, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "keep0,keep1,keep2"); + CHECK_SCHEMA(8, TSDB_DATA_TYPE_INT, 4, "cache"); + CHECK_SCHEMA(9, TSDB_DATA_TYPE_INT, 4, "blocks"); + CHECK_SCHEMA(10, TSDB_DATA_TYPE_INT, 4, "minrows"); + CHECK_SCHEMA(11, TSDB_DATA_TYPE_INT, 4, "maxrows"); + CHECK_SCHEMA(12, TSDB_DATA_TYPE_TINYINT, 1, "wallevel"); + CHECK_SCHEMA(13, TSDB_DATA_TYPE_INT, 4, "fsync"); + CHECK_SCHEMA(14, TSDB_DATA_TYPE_TINYINT, 1, "comp"); + CHECK_SCHEMA(15, TSDB_DATA_TYPE_TINYINT, 1, "cachelast"); + CHECK_SCHEMA(16, TSDB_DATA_TYPE_BINARY, 3 + VARSTR_HEADER_SIZE, "precision"); + CHECK_SCHEMA(17, TSDB_DATA_TYPE_TINYINT, 1, "update"); test.SendShowRetrieveMsg(); EXPECT_EQ(test.GetShowRows(), 0); @@ -76,19 +77,20 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { pReq->cacheLastRow = 0; pReq->ignoreExist = 1; - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DB, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); - CHECK_META("show databases", 17); + CHECK_META("show databases", 18); test.SendShowRetrieveMsg(); EXPECT_EQ(test.GetShowRows(), 1); CheckBinary("d1", TSDB_DB_NAME_LEN - 1); CheckTimestamp(); CheckInt16(2); // vgroups + CheckInt32(0); // ntables CheckInt16(1); // replica CheckInt16(1); // quorum CheckInt16(10); // days @@ -136,7 +138,7 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { pReq->quorum = 2; pReq->cacheLastRow = 1; - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_ALTER_DB, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_ALTER_DB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } @@ -147,6 +149,7 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { CheckBinary("d1", TSDB_DB_NAME_LEN - 1); CheckTimestamp(); CheckInt16(2); // vgroups + CheckInt32(0); CheckInt16(1); // replica CheckInt16(2); // quorum CheckInt16(10); // days @@ -162,52 +165,52 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { CheckBinary("ms", 3); // precision CheckInt8(0); // update - // // restart - // test.Restart(); + // restart + test.Restart(); - // test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); - // CHECK_META("show databases", 17); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); + CHECK_META("show databases", 18); - // test.SendShowRetrieveMsg(); - // EXPECT_EQ(test.GetShowRows(), 1); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); - // CheckBinary("d1", TSDB_DB_NAME_LEN - 1); - // CheckTimestamp(); - // CheckInt16(2); // vgroups - // CheckInt16(1); // replica - // CheckInt16(2); // quorum - // CheckInt16(10); // days - // CheckBinary("300,400,500", 24); // days - // CheckInt32(16); // cache - // CheckInt32(12); // blocks - // CheckInt32(100); // minrows - // CheckInt32(4096); // maxrows - // CheckInt8(2); // wallevel - // CheckInt32(4000); // fsync - // CheckInt8(2); // comp - // CheckInt8(1); // cachelast - // CheckBinary("ms", 3); // precision - // CheckInt8(0); // update + CheckBinary("d1", TSDB_DB_NAME_LEN - 1); + CheckTimestamp(); + CheckInt16(2); // vgroups + CheckInt32(0); + CheckInt16(1); // replica + CheckInt16(2); // quorum + CheckInt16(10); // days + CheckBinary("300,400,500", 24); // days + CheckInt32(16); // cache + CheckInt32(12); // blocks + CheckInt32(100); // minrows + CheckInt32(4096); // maxrows + CheckInt8(2); // wallevel + CheckInt32(4000); // fsync + CheckInt8(2); // comp + CheckInt8(1); // cachelast + CheckBinary("ms", 3); // precision + CheckInt8(0); // update - // { - // int32_t contLen = sizeof(SDropDbMsg); + { + int32_t contLen = sizeof(SDropDbMsg); - // SDropDbMsg* pReq = (SDropDbMsg*)rpcMallocCont(contLen); - // strcpy(pReq->db, "1.d1"); + SDropDbMsg* pReq = (SDropDbMsg*)rpcMallocCont(contLen); + strcpy(pReq->db, "1.d1"); - // SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_DB, pReq, contLen); - // ASSERT_NE(pMsg, nullptr); - // ASSERT_EQ(pMsg->code, 0); - // } + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_DB, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } - // test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); - // CHECK_META("show databases", 17); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); + CHECK_META("show databases", 18); - // test.SendShowRetrieveMsg(); - // EXPECT_EQ(test.GetShowRows(), 0); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 0); } -#if 0 TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) { { int32_t contLen = sizeof(SCreateDbMsg); @@ -234,13 +237,13 @@ TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) { pReq->cacheLastRow = 0; pReq->ignoreExist = 1; - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DB, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); - CHECK_META("show databases", 17); + CHECK_META("show databases", 18); test.SendShowRetrieveMsg(); EXPECT_EQ(test.GetShowRows(), 1); @@ -253,7 +256,7 @@ TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) { strcpy(pReq->db, "1.d2"); pReq->vgVersion = htonl(-1); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_USE_DB, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_USE_DB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); @@ -299,5 +302,3 @@ TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) { } } } - -#endif \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index c9708ec1cf..4009a83cd0 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -81,7 +81,7 @@ TEST_F(DndTestDnode, 02_ConfigDnode) { pReq->dnodeId = htonl(1); strcpy(pReq->config, "ddebugflag 131"); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CONFIG_DNODE, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CONFIG_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } @@ -93,7 +93,7 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); strcpy(pReq->ep, "localhost:9042"); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DNODE, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } @@ -126,7 +126,7 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_DNODE, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } @@ -150,7 +150,7 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); strcpy(pReq->ep, "localhost:9043"); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DNODE, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } @@ -161,7 +161,7 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); strcpy(pReq->ep, "localhost:9044"); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DNODE, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } @@ -172,7 +172,7 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); strcpy(pReq->ep, "localhost:9045"); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DNODE, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } diff --git a/source/dnode/mgmt/impl/test/mnode/mnode.cpp b/source/dnode/mgmt/impl/test/mnode/mnode.cpp index 6724c85500..0d89542288 100644 --- a/source/dnode/mgmt/impl/test/mnode/mnode.cpp +++ b/source/dnode/mgmt/impl/test/mnode/mnode.cpp @@ -77,7 +77,7 @@ TEST_F(DndTestMnode, 02_Create_Mnode_Invalid_Id) { SCreateMnodeMsg* pReq = (SCreateMnodeMsg*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_MNODE, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MNODE_ALREADY_EXIST); } @@ -90,7 +90,7 @@ TEST_F(DndTestMnode, 03_Create_Mnode_Invalid_Id) { SCreateMnodeMsg* pReq = (SCreateMnodeMsg*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_MNODE, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST); } @@ -104,7 +104,7 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); strcpy(pReq->ep, "localhost:9062"); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DNODE, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); @@ -121,7 +121,7 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { SCreateMnodeMsg* pReq = (SCreateMnodeMsg*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_MNODE, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); @@ -148,7 +148,7 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { SDropMnodeMsg* pReq = (SDropMnodeMsg*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_MNODE, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_MNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); @@ -169,7 +169,7 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { // SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(contLen); // pReq->dnodeId = htonl(2); -// SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_DNODE, pReq, contLen); +// SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_DNODE, pReq, contLen); // ASSERT_NE(pMsg, nullptr); // ASSERT_EQ(pMsg->code, 0); // } @@ -193,7 +193,7 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { // SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); // strcpy(pReq->ep, "localhost:9063"); -// SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DNODE, pReq, contLen); +// SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); // ASSERT_NE(pMsg, nullptr); // ASSERT_EQ(pMsg->code, 0); // } @@ -204,7 +204,7 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { // SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); // strcpy(pReq->ep, "localhost:9064"); -// SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DNODE, pReq, contLen); +// SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); // ASSERT_NE(pMsg, nullptr); // ASSERT_EQ(pMsg->code, 0); // } @@ -215,7 +215,7 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { // SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); // strcpy(pReq->ep, "localhost:9065"); -// SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DNODE, pReq, contLen); +// SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); // ASSERT_NE(pMsg, nullptr); // ASSERT_EQ(pMsg->code, 0); // } diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp index 2e82d4d1aa..081f9e7ef5 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -35,7 +35,7 @@ TEST_F(DndTestProfile, 01_ConnectMsg) { strcpy(pReq->app, "dnode_test_profile"); strcpy(pReq->db, ""); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CONNECT, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CONNECT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); @@ -67,7 +67,7 @@ TEST_F(DndTestProfile, 02_ConnectMsg_InvalidDB) { strcpy(pReq->app, "dnode_test_profile"); strcpy(pReq->db, "invalid_db"); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CONNECT, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CONNECT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_DB); ASSERT_EQ(pMsg->contLen, 0); @@ -105,7 +105,7 @@ TEST_F(DndTestProfile, 04_HeartBeatMsg) { pReq->numOfStreams = htonl(0); strcpy(pReq->app, "dnode_test_profile"); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_HEARTBEAT, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_HEARTBEAT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); @@ -138,7 +138,7 @@ TEST_F(DndTestProfile, 05_KillConnMsg) { SKillConnMsg* pReq = (SKillConnMsg*)rpcMallocCont(contLen); pReq->connId = htonl(connId); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_KILL_CONN, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_KILL_CONN, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } @@ -153,7 +153,7 @@ TEST_F(DndTestProfile, 05_KillConnMsg) { pReq->numOfStreams = htonl(0); strcpy(pReq->app, "dnode_test_profile"); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_HEARTBEAT, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_HEARTBEAT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_CONNECTION); ASSERT_EQ(pMsg->contLen, 0); @@ -167,7 +167,7 @@ TEST_F(DndTestProfile, 05_KillConnMsg) { strcpy(pReq->app, "dnode_test_profile"); strcpy(pReq->db, ""); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CONNECT, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CONNECT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); @@ -198,7 +198,7 @@ TEST_F(DndTestProfile, 06_KillConnMsg_InvalidConn) { SKillConnMsg* pReq = (SKillConnMsg*)rpcMallocCont(contLen); pReq->connId = htonl(2345); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_KILL_CONN, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_KILL_CONN, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_CONN_ID); } @@ -211,7 +211,7 @@ TEST_F(DndTestProfile, 07_KillQueryMsg) { pReq->connId = htonl(connId); pReq->queryId = htonl(1234); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_KILL_QUERY, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_KILL_QUERY, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); ASSERT_EQ(pMsg->contLen, 0); @@ -227,7 +227,7 @@ TEST_F(DndTestProfile, 07_KillQueryMsg) { pReq->numOfStreams = htonl(0); strcpy(pReq->app, "dnode_test_profile"); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_HEARTBEAT, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_HEARTBEAT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); @@ -261,7 +261,7 @@ TEST_F(DndTestProfile, 08_KillQueryMsg_InvalidConn) { pReq->connId = htonl(2345); pReq->queryId = htonl(1234); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_KILL_QUERY, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_KILL_QUERY, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_CONN_ID); } diff --git a/source/dnode/mgmt/impl/test/show/show.cpp b/source/dnode/mgmt/impl/test/show/show.cpp index bec681cb8f..a0df0f2921 100644 --- a/source/dnode/mgmt/impl/test/show/show.cpp +++ b/source/dnode/mgmt/impl/test/show/show.cpp @@ -32,7 +32,7 @@ TEST_F(DndTestShow, 01_ShowMsg_InvalidMsgMax) { pReq->type = TSDB_MGMT_TABLE_MAX; strcpy(pReq->db, ""); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_SHOW, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_SHOW, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_MSG_TYPE); } @@ -44,7 +44,7 @@ TEST_F(DndTestShow, 02_ShowMsg_InvalidMsgStart) { pReq->type = TSDB_MGMT_TABLE_START; strcpy(pReq->db, ""); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_SHOW, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_SHOW, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_MSG_TYPE); } @@ -57,7 +57,7 @@ TEST_F(DndTestShow, 02_ShowMsg_Conn) { strcpy(pReq->app, "dnode_test_show"); strcpy(pReq->db, ""); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CONNECT, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CONNECT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); diff --git a/source/dnode/mgmt/impl/test/stb/stb.cpp b/source/dnode/mgmt/impl/test/stb/stb.cpp index 992ebaf902..e67a3ea8cf 100644 --- a/source/dnode/mgmt/impl/test/stb/stb.cpp +++ b/source/dnode/mgmt/impl/test/stb/stb.cpp @@ -51,7 +51,7 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { pReq->cacheLastRow = 0; pReq->ignoreExist = 1; - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DB, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } @@ -106,7 +106,7 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { strcpy(pSchema->name, "tag3"); } - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_STB, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_STB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } @@ -133,7 +133,7 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { STableInfoMsg* pReq = (STableInfoMsg*)rpcMallocCont(contLen); strcpy(pReq->tableFname, "1.d1.stb"); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_TABLE_META, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_VND_TABLE_META, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); @@ -192,7 +192,7 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { SDropStbMsg* pReq = (SDropStbMsg*)rpcMallocCont(contLen); strcpy(pReq->name, "1.d1.stb"); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_STB, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_STB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } diff --git a/source/dnode/mgmt/impl/test/sut/inc/base.h b/source/dnode/mgmt/impl/test/sut/inc/base.h index b0de6f5c0c..fef7ad9d43 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/base.h +++ b/source/dnode/mgmt/impl/test/sut/inc/base.h @@ -20,7 +20,7 @@ #include "os.h" #include "dnode.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tconfig.h" #include "tdataformat.h" #include "tglobal.h" @@ -37,7 +37,7 @@ class Testbase { void Init(const char* path, int16_t port); void Cleanup(); void Restart(); - SRpcMsg* SendMsg(int8_t msgType, void* pCont, int32_t contLen); + SRpcMsg* SendMsg(tmsg_t msgType, void* pCont, int32_t contLen); private: void InitLog(const char* path); diff --git a/source/dnode/mgmt/impl/test/sut/src/base.cpp b/source/dnode/mgmt/impl/test/sut/src/base.cpp index 59e99b1e1d..f3fe8cfa86 100644 --- a/source/dnode/mgmt/impl/test/sut/src/base.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/base.cpp @@ -60,7 +60,7 @@ void Testbase::Cleanup() { void Testbase::Restart() { server.Restart(); } -SRpcMsg* Testbase::SendMsg(int8_t msgType, void* pCont, int32_t contLen) { +SRpcMsg* Testbase::SendMsg(tmsg_t msgType, void* pCont, int32_t contLen) { SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pCont; rpcMsg.contLen = contLen; @@ -75,7 +75,7 @@ void Testbase::SendShowMetaMsg(int8_t showType, const char* db) { pShow->type = showType; strcpy(pShow->db, db); - SRpcMsg* pMsg = SendMsg(TSDB_MSG_TYPE_SHOW, pShow, contLen); + SRpcMsg* pMsg = SendMsg(TDMT_MND_SHOW, pShow, contLen); SShowRsp* pShowRsp = (SShowRsp*)pMsg->pCont; ASSERT(pShowRsp != nullptr); @@ -124,7 +124,7 @@ void Testbase::SendShowRetrieveMsg() { pRetrieve->showId = htonl(showId); pRetrieve->free = 0; - SRpcMsg* pMsg = SendMsg(TSDB_MSG_TYPE_SHOW_RETRIEVE, pRetrieve, contLen); + SRpcMsg* pMsg = SendMsg(TDMT_MND_SHOW_RETRIEVE, pRetrieve, contLen); pRetrieveRsp = (SRetrieveTableRsp*)pMsg->pCont; pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); diff --git a/source/dnode/mgmt/impl/test/sut/src/client.cpp b/source/dnode/mgmt/impl/test/sut/src/client.cpp index fce3da1507..13429cec28 100644 --- a/source/dnode/mgmt/impl/test/sut/src/client.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/client.cpp @@ -18,7 +18,7 @@ static void processClientRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { TestClient* client = (TestClient*)parent; client->SetRpcRsp(pMsg); - uInfo("response:%s from dnode, code:0x%x", taosMsg[pMsg->msgType], pMsg->code); + uInfo("response:%s from dnode, code:0x%x", TMSG_INFO(pMsg->msgType), pMsg->code); tsem_post(client->GetSem()); } diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp index a1810e7ada..aa160304c8 100644 --- a/source/dnode/mgmt/impl/test/user/user.cpp +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -51,7 +51,7 @@ TEST_F(DndTestUser, 02_Create_Drop_Alter_User) { strcpy(pReq->user, "u1"); strcpy(pReq->pass, "p1"); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_USER, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_USER, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } @@ -63,7 +63,7 @@ TEST_F(DndTestUser, 02_Create_Drop_Alter_User) { strcpy(pReq->user, "u2"); strcpy(pReq->pass, "p2"); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_USER, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_USER, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } @@ -94,7 +94,7 @@ TEST_F(DndTestUser, 02_Create_Drop_Alter_User) { strcpy(pReq->user, "u1"); strcpy(pReq->pass, "p2"); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_ALTER_USER, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_ALTER_USER, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } @@ -124,7 +124,7 @@ TEST_F(DndTestUser, 02_Create_Drop_Alter_User) { SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(contLen); strcpy(pReq->user, "u1"); - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_USER, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_USER, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } diff --git a/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp b/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp index c534fcbabe..718fbea50d 100644 --- a/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp +++ b/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp @@ -60,7 +60,7 @@ TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { pReplica->port = htons(9150); } - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_VNODE_IN, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_DND_CREATE_VNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } @@ -100,7 +100,7 @@ TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { pReplica->port = htons(9150); } - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_ALTER_VNODE_IN, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_DND_ALTER_VNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } @@ -119,9 +119,9 @@ TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; rpcMsg.contLen = sizeof(SDropVnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_DROP_VNODE_IN; + rpcMsg.msgType = TDMT_DND_DROP_VNODE; - SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_VNODE_IN, pReq, contLen); + SRpcMsg* pMsg = test.SendMsg(TDMT_DND_DROP_VNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index a0404d5bcc..c6fb0cce1d 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -20,7 +20,7 @@ #include "cJSON.h" #include "sync.h" -#include "taosmsg.h" +#include "tmsg.h" #include "thash.h" #include "tlog.h" #include "trpc.h" @@ -95,7 +95,9 @@ typedef struct { int32_t id; ETrnStage stage; ETrnPolicy policy; + int32_t retryTimes; void *rpcHandle; + void *rpcAHandle; SArray *redoLogs; SArray *undoLogs; SArray *commitLogs; @@ -285,6 +287,21 @@ typedef struct { char payload[]; } SShowObj; +typedef struct { + char name[TSDB_TOPIC_FNAME_LEN]; + char db[TSDB_FULL_DB_NAME_LEN]; + int64_t createTime; + int64_t updateTime; + uint64_t uid; + uint64_t dbUid; + int32_t version; + SRWLatch lock; + int32_t execLen; + void* executor; + int32_t sqlLen; + char* sql; +} STopicObj; + typedef struct SMnodeMsg { char user[TSDB_USER_LEN]; char db[TSDB_FULL_DB_NAME_LEN]; diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index ba72d95537..a9932ce048 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -61,6 +61,13 @@ typedef struct { char email[TSDB_FQDN_LEN]; } STelemMgmt; +typedef struct { + int32_t errCode; + sem_t syncSem; + SSyncNode *pSyncNode; + ESyncState state; +} SSyncMgmt; + typedef struct SMnode { int32_t dnodeId; int32_t clusterId; @@ -77,17 +84,17 @@ typedef struct SMnode { SShowMgmt showMgmt; SProfileMgmt profileMgmt; STelemMgmt telemMgmt; - MndMsgFp msgFp[TSDB_MSG_TYPE_MAX]; + SSyncMgmt syncMgmt; + MndMsgFp msgFp[TDMT_MAX]; SendMsgToDnodeFp sendMsgToDnodeFp; SendMsgToMnodeFp sendMsgToMnodeFp; SendRedirectMsgFp sendRedirectMsgFp; - PutMsgToMnodeQFp putMsgToApplyMsgFp; } SMnode; void mndSendMsgToDnode(SMnode *pMnode, SEpSet *pEpSet, SRpcMsg *rpcMsg); void mndSendMsgToMnode(SMnode *pMnode, SRpcMsg *pMsg); void mndSendRedirectMsg(SMnode *pMnode, SRpcMsg *pMsg); -void mndSetMsgHandle(SMnode *pMnode, int32_t msgType, MndMsgFp fp); +void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); uint64_t mndGenerateUid(char *name, int32_t len) ; diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index 58cae73c7f..48847dc6a3 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -25,6 +25,9 @@ extern "C" { int32_t mndInitStb(SMnode *pMnode); void mndCleanupStb(SMnode *pMnode); +SStbObj *mndAcquireStb(SMnode *pMnode, char *stbName); +void mndReleaseStb(SMnode *pMnode, SStbObj *pStb); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndTopic.h b/source/dnode/mnode/impl/inc/mndTopic.h new file mode 100644 index 0000000000..d092f47d4b --- /dev/null +++ b/source/dnode/mnode/impl/inc/mndTopic.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MND_TOPIC_H_ +#define _TD_MND_TOPIC_H_ + +#include "mndInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t mndInitTopic(SMnode *pMnode); +void mndCleanupTopic(SMnode *pMnode); + +STopicObj *mndAcquireTopic(SMnode *pMnode, char *topicName); +void mndReleaseTopic(SMnode *pMnode, STopicObj *pTopic); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_TOPIC_H_*/ diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index a8d37ba655..2d57179f1c 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -24,7 +24,7 @@ extern "C" { typedef struct { SEpSet epSet; - int8_t msgType; + tmsg_t msgType; int8_t msgSent; int8_t msgReceived; int32_t errCode; @@ -35,7 +35,7 @@ typedef struct { int32_t mndInitTrans(SMnode *pMnode); void mndCleanupTrans(SMnode *pMnode); -STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle); +STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, SRpcMsg *pMsg); void mndTransDrop(STrans *pTrans); int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw); diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index 4c2706f481..767f628306 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -40,9 +40,9 @@ int32_t mndInitAcct(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndAcctActionUpdate, .deleteFp = (SdbDeleteFp)mndAcctActionDelete}; - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_ACCT, mndProcessCreateAcctMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_ACCT, mndProcessAlterAcctMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_ACCT, mndProcessDropAcctMsg); + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_ACCT, mndProcessCreateAcctMsg); + mndSetMsgHandle(pMnode, TDMT_MND_ALTER_ACCT, mndProcessAlterAcctMsg); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_ACCT, mndProcessDropAcctMsg); return sdbSetTable(pMnode->pSdb, table); } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index dd474d85f3..6eb94eccf2 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -48,12 +48,12 @@ int32_t mndInitDb(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndDbActionUpdate, .deleteFp = (SdbDeleteFp)mndDbActionDelete}; - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_DB, mndProcessCreateDbMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_DB, mndProcessAlterDbMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_DB, mndProcessDropDbMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_USE_DB, mndProcessUseDbMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_SYNC_DB, mndProcessSyncDbMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_COMPACT_DB, mndProcessCompactDbMsg); + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_DB, mndProcessCreateDbMsg); + mndSetMsgHandle(pMnode, TDMT_MND_ALTER_DB, mndProcessAlterDbMsg); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_DB, mndProcessDropDbMsg); + mndSetMsgHandle(pMnode, TDMT_MND_USE_DB, mndProcessUseDbMsg); + mndSetMsgHandle(pMnode, TDMT_MND_SYNC_DB, mndProcessSyncDbMsg); + mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbMsg); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_DB, mndGetDbMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs); @@ -311,7 +311,7 @@ static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.pCont = pMsg; action.contLen = sizeof(SCreateVnodeMsg); - action.msgType = TSDB_MSG_TYPE_CREATE_VNODE_IN; + action.msgType = TDMT_DND_CREATE_VNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); return -1; @@ -340,7 +340,7 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.pCont = pMsg; action.contLen = sizeof(SDropVnodeMsg); - action.msgType = TSDB_MSG_TYPE_DROP_VNODE_IN; + action.msgType = TDMT_DND_DROP_VNODE; if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pMsg); return -1; @@ -400,7 +400,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat } int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); if (pTrans == NULL) { mError("db:%s, failed to create since %s", pCreate->db, terrstr()); goto CREATE_DB_OVER; @@ -541,32 +541,74 @@ static int32_t mndSetDbCfgFromAlterDbMsg(SDbObj *pDb, SAlterDbMsg *pAlter) { } static int32_t mndSetUpdateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { - SSdbRaw *pRedoRaw = mndDbActionEncode(pNewDb); + SSdbRaw *pRedoRaw = mndDbActionEncode(pOldDb); if (pRedoRaw == NULL) return -1; if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_UPDATING) != 0) return -1; return 0; } -static int32_t mndSetUpdateDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { - SSdbRaw *pUndoRaw = mndDbActionEncode(pOldDb); - if (pUndoRaw == NULL) return -1; - if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; - if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) return -1; +static int32_t mndSetUpdateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { + SSdbRaw *pCommitRaw = mndDbActionEncode(pNewDb); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; return 0; } -static int32_t mndSetUpdateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { return 0; } +static int32_t mndBuildUpdateVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { + for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { + STransAction action = {0}; + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; -static int32_t mndSetUpdateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { return 0; } + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); -static int32_t mndSetUpdateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { return 0; } + SAlterVnodeMsg *pMsg = (SAlterVnodeMsg *)mndBuildCreateVnodeMsg(pMnode, pDnode, pDb, pVgroup); + if (pMsg == NULL) return -1; + + action.pCont = pMsg; + action.contLen = sizeof(SAlterVnodeMsg); + action.msgType = TDMT_DND_ALTER_VNODE; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + } + + return 0; +} + +static int32_t mndSetUpdateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + + while (1) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + if (pVgroup->dbUid == pNewDb->uid) { + if (mndBuildUpdateVgroupAction(pMnode, pTrans, pNewDb, pVgroup) != 0) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + } + + sdbRelease(pSdb, pVgroup); + } + + return 0; +} static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pOldDb, SDbObj *pNewDb) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); if (pTrans == NULL) { mError("db:%s, failed to update since %s", pOldDb->name, terrstr()); return terrno; @@ -579,11 +621,6 @@ static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pOldDb, SDbO goto UPDATE_DB_OVER; } - if (mndSetUpdateDbUndoLogs(pMnode, pTrans, pOldDb, pNewDb) != 0) { - mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); - goto UPDATE_DB_OVER; - } - if (mndSetUpdateDbCommitLogs(pMnode, pTrans, pOldDb, pNewDb) != 0) { mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); goto UPDATE_DB_OVER; @@ -594,11 +631,6 @@ static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pOldDb, SDbO goto UPDATE_DB_OVER; } - if (mndSetUpdateDbUndoActions(pMnode, pTrans, pOldDb, pNewDb) != 0) { - mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); - goto UPDATE_DB_OVER; - } - if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); goto UPDATE_DB_OVER; @@ -660,31 +692,87 @@ static int32_t mndSetDropDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) return 0; } -static int32_t mndSetDropDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { - SSdbRaw *pUndoRaw = mndDbActionEncode(pDb); - if (pUndoRaw == NULL) return -1; - if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; - if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) return -1; - - return 0; -} - static int32_t mndSetDropDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { SSdbRaw *pCommitRaw = mndDbActionEncode(pDb); if (pCommitRaw == NULL) return -1; if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + + while (1) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + if (pVgroup->dbUid == pDb->uid) { + SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); + if (pVgRaw == NULL || mndTransAppendCommitlog(pTrans, pVgRaw) != 0) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPED); + } + + sdbRelease(pSdb, pVgroup); + } + return 0; } -static int32_t mndSetDropDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { return 0; } +static int32_t mndBuildDropVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { + for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { + STransAction action = {0}; + SVnodeGid * pVgid = pVgroup->vnodeGid + vn; -static int32_t mndSetDropDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { return 0; } + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + SDropVnodeMsg *pMsg = mndBuildDropVnodeMsg(pMnode, pDnode, pDb, pVgroup); + if (pMsg == NULL) return -1; + + action.pCont = pMsg; + action.contLen = sizeof(SCreateVnodeMsg); + action.msgType = TDMT_DND_DROP_VNODE; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + } + + return 0; +} + +static int32_t mndSetDropDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + + while (1) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + if (pVgroup->dbUid == pDb->uid) { + if (mndBuildDropVgroupAction(pMnode, pTrans, pDb, pVgroup) != 0) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + } + + sdbRelease(pSdb, pVgroup); + } + + return 0; +} static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pDb) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); if (pTrans == NULL) { mError("db:%s, failed to drop since %s", pDb->name, terrstr()); return -1; @@ -697,11 +785,6 @@ static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pDb) { goto DROP_DB_OVER; } - if (mndSetDropDbUndoLogs(pMnode, pTrans, pDb) != 0) { - mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); - goto DROP_DB_OVER; - } - if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) { mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); goto DROP_DB_OVER; @@ -712,11 +795,6 @@ static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pDb) { goto DROP_DB_OVER; } - if (mndSetDropDbUndoActions(pMnode, pTrans, pDb) != 0) { - mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); - goto DROP_DB_OVER; - } - if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); goto DROP_DB_OVER; @@ -876,6 +954,12 @@ static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMe pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "ntables"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "replica"); @@ -1017,6 +1101,10 @@ static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3 *(int16_t *)pWrite = pDb->cfg.numOfVgroups; cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = 0; // todo + cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; *(int16_t *)pWrite = pDb->cfg.replications; cols++; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 9751771773..80c9f9544e 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -73,11 +73,11 @@ int32_t mndInitDnode(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndDnodeActionUpdate, .deleteFp = (SdbDeleteFp)mndDnodeActionDelete}; - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_DNODE, mndProcessCreateDnodeMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_DNODE, mndProcessDropDnodeMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CONFIG_DNODE, mndProcessConfigDnodeMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CONFIG_DNODE_IN_RSP, mndProcessConfigDnodeRsp); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_STATUS, mndProcessStatusMsg); + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_DNODE, mndProcessCreateDnodeMsg); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_DNODE, mndProcessDropDnodeMsg); + mndSetMsgHandle(pMnode, TDMT_MND_CONFIG_DNODE, mndProcessConfigDnodeMsg); + mndSetMsgHandle(pMnode, TDMT_DND_CONFIG_DNODE_RSP, mndProcessConfigDnodeRsp); + mndSetMsgHandle(pMnode, TDMT_MND_STATUS, mndProcessStatusMsg); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_VARIABLES, mndGetConfigMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VARIABLES, mndRetrieveConfigs); @@ -396,7 +396,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg * return terrno; } - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); if (pTrans == NULL) { mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr()); return -1; @@ -452,7 +452,7 @@ static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) { } static int32_t mndDropDnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); if (pTrans == NULL) { mError("dnode:%d, failed to drop since %s", pDnode->id, terrstr()); return -1; @@ -527,7 +527,7 @@ static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg) { pCfgDnode->dnodeId = htonl(pCfg->dnodeId); memcpy(pCfgDnode->config, pCfg->config, TSDB_DNODE_CONFIG_LEN); - SRpcMsg rpcMsg = {.msgType = TSDB_MSG_TYPE_CONFIG_DNODE_IN, + SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pCfgDnode, .contLen = sizeof(SCfgDnodeMsg), .ahandle = pMsg->rpcMsg.ahandle}; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index a513c9d57d..638d984c69 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -44,9 +44,9 @@ int32_t mndInitFunc(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndFuncActionUpdate, .deleteFp = (SdbDeleteFp)mndFuncActionDelete}; - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_FUNCTION, mndProcessCreateFuncMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_FUNCTION, mndProcessDropFuncMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_RETRIEVE_FUNCTION, mndProcessRetrieveFuncMsg); + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_FUNCTION, mndProcessCreateFuncMsg); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_FUNCTION, mndProcessDropFuncMsg); + mndSetMsgHandle(pMnode, TDMT_MND_RETRIEVE_FUNCTION, mndProcessRetrieveFuncMsg); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndGetFuncMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndRetrieveFuncs); @@ -147,7 +147,7 @@ static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncMsg *pC pFunc->pCode = pFunc->pData + pCreate->commentSize; memcpy(pFunc->pCode, pCreate->pCont + pCreate->commentSize, pFunc->codeSize); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); if (pTrans == NULL) { free(pFunc); mError("func:%s, failed to create since %s", pCreate->name, terrstr()); @@ -195,7 +195,7 @@ static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncMsg *pC } static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pMsg, SFuncObj *pFunc) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); if (pTrans == NULL) { mError("func:%s, failed to drop since %s", pFunc->name, terrstr()); return -1; @@ -441,7 +441,7 @@ static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int16_t le } if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { - int32_t bytes = len > 0 ? (int)(len - VARSTR_HEADER_SIZE) : len; + int32_t bytes = len > 0 ? (int32_t)(len - VARSTR_HEADER_SIZE) : len; snprintf(buf, buflen - 1, "%s(%d)", tDataTypes[type].name, type == TSDB_DATA_TYPE_NCHAR ? bytes / 4 : bytes); buf[buflen - 1] = 0; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index a019c0dc55..8f57a18a1d 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -47,11 +47,11 @@ int32_t mndInitMnode(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndMnodeActionUpdate, .deleteFp = (SdbDeleteFp)mndMnodeActionDelete}; - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_MNODE, mndProcessCreateMnodeReq); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_MNODE, mndProcessDropMnodeReq); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_MNODE_IN_RSP, mndProcessCreateMnodeRsp); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_MNODE_IN_RSP, mndProcessAlterMnodeRsp); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_MNODE_IN_RSP, mndProcessDropMnodeRsp); + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_MNODE, mndProcessCreateMnodeReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_MNODE, mndProcessDropMnodeReq); + mndSetMsgHandle(pMnode, TDMT_DND_CREATE_MNODE_RSP, mndProcessCreateMnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_ALTER_MNODE_RSP, mndProcessAlterMnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_DROP_MNODE_RSP, mndProcessDropMnodeRsp); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_MNODE, mndGetMnodeMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_MNODE, mndRetrieveMnodes); @@ -293,7 +293,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno action.epSet = mndGetDnodeEpset(pMObj->pDnode); action.pCont = pMsg; action.contLen = sizeof(SAlterMnodeInMsg); - action.msgType = TSDB_MSG_TYPE_ALTER_MNODE_IN; + action.msgType = TDMT_DND_ALTER_MNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); @@ -317,7 +317,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno action.epSet = mndGetDnodeEpset(pDnode); action.pCont = pMsg; action.contLen = sizeof(SCreateMnodeInMsg); - action.msgType = TSDB_MSG_TYPE_CREATE_MNODE_IN; + action.msgType = TDMT_DND_CREATE_MNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); return -1; @@ -334,7 +334,7 @@ static int32_t mndCreateMnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode mnodeObj.updateTime = mnodeObj.createdTime; int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, pMsg->rpcMsg.handle); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); if (pTrans == NULL) { mError("mnode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); goto CREATE_MNODE_OVER; @@ -461,7 +461,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode action.epSet = mndGetDnodeEpset(pMObj->pDnode); action.pCont = pMsg; action.contLen = sizeof(SAlterMnodeInMsg); - action.msgType = TSDB_MSG_TYPE_ALTER_MNODE_IN; + action.msgType = TDMT_DND_ALTER_MNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); @@ -488,7 +488,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode action.epSet = mndGetDnodeEpset(pDnode); action.pCont = pMsg; action.contLen = sizeof(SDropMnodeInMsg); - action.msgType = TSDB_MSG_TYPE_DROP_MNODE_IN; + action.msgType = TDMT_DND_DROP_MNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); return -1; @@ -500,7 +500,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode static int32_t mndDropMnode(SMnode *pMnode, SMnodeMsg *pMsg, SMnodeObj *pObj) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, pMsg->rpcMsg.handle); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); if (pTrans == NULL) { mError("mnode:%d, failed to drop since %s", pObj->id, terrstr()); goto DROP_MNODE_OVER; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index a4b73e0f3f..833be3884c 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -68,10 +68,10 @@ int32_t mndInitProfile(SMnode *pMnode) { return -1; } - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_HEARTBEAT, mndProcessHeartBeatMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CONNECT, mndProcessConnectMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_KILL_QUERY, mndProcessKillQueryMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_KILL_CONN, mndProcessKillConnectionMsg); + mndSetMsgHandle(pMnode, TDMT_MND_HEARTBEAT, mndProcessHeartBeatMsg); + mndSetMsgHandle(pMnode, TDMT_MND_CONNECT, mndProcessConnectMsg); + mndSetMsgHandle(pMnode, TDMT_MND_KILL_QUERY, mndProcessKillQueryMsg); + mndSetMsgHandle(pMnode, TDMT_MND_KILL_CONN, mndProcessKillConnectionMsg); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndGetConnsMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndRetrieveConns); diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 41f09e85a9..4e22a07460 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -36,8 +36,8 @@ int32_t mndInitShow(SMnode *pMnode) { return -1; } - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_SHOW, mndProcessShowMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_SHOW_RETRIEVE, mndProcessRetrieveMsg); + mndSetMsgHandle(pMnode, TDMT_MND_SHOW, mndProcessShowMsg); + mndSetMsgHandle(pMnode, TDMT_MND_SHOW_RETRIEVE, mndProcessRetrieveMsg); return 0; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index a51312a2a9..24d88068bf 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -52,13 +52,13 @@ int32_t mndInitStb(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndStbActionUpdate, .deleteFp = (SdbDeleteFp)mndStbActionDelete}; - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_STB, mndProcessCreateStbMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_STB, mndProcessAlterStbMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_STB, mndProcessDropStbMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_STB_IN_RSP, mndProcessCreateStbInRsp); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_STB_IN_RSP, mndProcessAlterStbInRsp); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_STB_IN_RSP, mndProcessDropStbInRsp); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_TABLE_META, mndProcessStbMetaMsg); + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STB, mndProcessCreateStbMsg); + mndSetMsgHandle(pMnode, TDMT_MND_ALTER_STB, mndProcessAlterStbMsg); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_STB, mndProcessDropStbMsg); + mndSetMsgHandle(pMnode, TDMT_VND_CREATE_STB_RSP, mndProcessCreateStbInRsp); + mndSetMsgHandle(pMnode, TDMT_VND_ALTER_STB_RSP, mndProcessAlterStbInRsp); + mndSetMsgHandle(pMnode, TDMT_VND_DROP_STB_RSP, mndProcessDropStbInRsp); + mndSetMsgHandle(pMnode, TDMT_VND_TABLE_META, mndProcessStbMetaMsg); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_STB, mndGetStbMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STB, mndRetrieveStb); @@ -177,7 +177,7 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOldStb, SStbObj *pNewStb } SStbObj *mndAcquireStb(SMnode *pMnode, char *stbName) { - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; SStbObj *pStb = sdbAcquire(pSdb, SDB_STB, stbName); if (pStb == NULL) { terrno = TSDB_CODE_MND_STB_NOT_EXIST; @@ -200,7 +200,43 @@ static SDbObj *mndAcquireDbByStb(SMnode *pMnode, char *stbName) { return mndAcquireDb(pMnode, db); } -static SCreateStbInternalMsg *mndBuildCreateStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb) { +static void *mndBuildCreateStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int *pContLen) { +#if 1 + SVCreateTbReq req; + void * buf; + int bsize; + SMsgHead * pMsgHead; + + req.ver = 0; + req.name = pStb->name; + req.ttl = 0; + req.keep = 0; + req.type = TD_SUPER_TABLE; + req.stbCfg.suid = pStb->uid; + req.stbCfg.nCols = pStb->numOfColumns; + req.stbCfg.pSchema = pStb->pSchema; + req.stbCfg.nTagCols = pStb->numOfTags; + req.stbCfg.pTagSchema = pStb->pSchema + pStb->numOfColumns; + + bsize = tSerializeSVCreateTbReq(NULL, &req); + buf = malloc(sizeof(SMsgHead) + bsize); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pMsgHead = (SMsgHead *)buf; + + pMsgHead->contLen = htonl(sizeof(SMsgHead) + bsize); + pMsgHead->vgId = htonl(pVgroup->vgId); + + void *pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tSerializeSVCreateTbReq(&pBuf, &req); + + *pContLen = sizeof(SMsgHead) + bsize; + return buf; + +#else int32_t totalCols = pStb->numOfTags + pStb->numOfColumns; int32_t contLen = totalCols * sizeof(SSchema) + sizeof(SCreateStbInternalMsg); @@ -227,7 +263,9 @@ static SCreateStbInternalMsg *mndBuildCreateStbMsg(SMnode *pMnode, SVgObj *pVgro pSchema->colId = htonl(pSchema->colId); } + *pContLen = contLen; return pCreate; +#endif } static SDropStbInternalMsg *mndBuildDropStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb) { @@ -324,16 +362,17 @@ static int32_t mndSetCreateStbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj } static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; - void *pIter = NULL; + void * pIter = NULL; + int contLen; while (1) { pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; if (pVgroup->dbUid != pDb->uid) continue; - SCreateStbInternalMsg *pMsg = mndBuildCreateStbMsg(pMnode, pVgroup, pStb); + void *pMsg = mndBuildCreateStbMsg(pMnode, pVgroup, pStb, &contLen); if (pMsg == NULL) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); @@ -344,8 +383,8 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); action.pCont = pMsg; - action.contLen = htonl(pMsg->head.contLen); - action.msgType = TSDB_MSG_TYPE_CREATE_STB_IN; + action.contLen = contLen; + action.msgType = TDMT_VND_CREATE_STB; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); sdbCancelFetch(pSdb, pIter); @@ -359,9 +398,9 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj } static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; - void *pIter = NULL; + void * pIter = NULL; while (1) { pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); @@ -380,7 +419,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.epSet = mndGetVgroupEpset(pMnode, pVgroup); action.pCont = pMsg; action.contLen = sizeof(SDropStbInternalMsg); - action.msgType = TSDB_MSG_TYPE_DROP_STB_IN; + action.msgType = TDMT_VND_DROP_STB; if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pMsg); sdbCancelFetch(pSdb, pIter); @@ -415,7 +454,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCre memcpy(stbObj.pSchema, pCreate->pSchema, totalSize); int32_t code = 0; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); if (pTrans == NULL) { mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); return -1; @@ -461,7 +500,7 @@ CREATE_STB_OVER: } static int32_t mndProcessCreateStbMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; + SMnode * pMnode = pMsg->pMnode; SCreateStbMsg *pCreate = pMsg->rpcMsg.pCont; mDebug("stb:%s, start to create", pCreate->name); @@ -484,6 +523,15 @@ static int32_t mndProcessCreateStbMsg(SMnodeMsg *pMsg) { } } + // topic should have different name with stb + SStbObj *pTopic = mndAcquireStb(pMnode, pCreate->name); + if (pTopic != NULL) { + sdbRelease(pMnode->pSdb, pTopic); + terrno = TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC; + mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + SDbObj *pDb = mndAcquireDbByStb(pMnode, pCreate->name); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; @@ -536,7 +584,7 @@ static int32_t mndCheckAlterStbMsg(SAlterStbMsg *pAlter) { static int32_t mndUpdateStb(SMnode *pMnode, SMnodeMsg *pMsg, SStbObj *pOldStb, SStbObj *pNewStb) { return 0; } static int32_t mndProcessAlterStbMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; + SMnode * pMnode = pMsg->pMnode; SAlterStbMsg *pAlter = pMsg->rpcMsg.pCont; mDebug("stb:%s, start to alter", pAlter->name); @@ -605,7 +653,7 @@ static int32_t mndSetDropStbUndoActions(SMnode *pMnode, STrans *pTrans, SStbObj static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pMsg, SStbObj *pStb) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); if (pTrans == NULL) { mError("stb:%s, failed to drop since %s", pStb->name, terrstr()); return -1; @@ -650,7 +698,7 @@ DROP_STB_OVER: } static int32_t mndProcessDropStbMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; + SMnode * pMnode = pMsg->pMnode; SDropStbMsg *pDrop = pMsg->rpcMsg.pCont; mDebug("stb:%s, start to drop", pDrop->name); @@ -685,7 +733,7 @@ static int32_t mndProcessDropStbInRsp(SMnodeMsg *pMsg) { } static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; + SMnode * pMnode = pMsg->pMnode; STableInfoMsg *pInfo = pMsg->rpcMsg.pCont; mDebug("stb:%s, start to retrieve meta", pInfo->tableFname); @@ -757,7 +805,7 @@ static int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs } int32_t numOfStbs = 0; - void *pIter = NULL; + void * pIter = NULL; while (1) { SStbObj *pStb = NULL; pIter = sdbFetch(pSdb, SDB_STB, pIter, (void **)&pStb); @@ -776,7 +824,7 @@ static int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { SMnode *pMnode = pMsg->pMnode; - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; if (mndGetNumOfStbs(pMnode, pShow->db, &pShow->numOfRows) != 0) { return -1; @@ -825,8 +873,8 @@ static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pM } static void mndExtractTableName(char *tableId, char *name) { - int pos = -1; - int num = 0; + int32_t pos = -1; + int32_t num = 0; for (pos = 0; tableId[pos] != 0; ++pos) { if (tableId[pos] == '.') num++; if (num == 2) break; @@ -838,12 +886,12 @@ static void mndExtractTableName(char *tableId, char *name) { } static int32_t mndRetrieveStb(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; - SSdb *pSdb = pMnode->pSdb; + SMnode * pMnode = pMsg->pMnode; + SSdb * pSdb = pMnode->pSdb; int32_t numOfRows = 0; SStbObj *pStb = NULL; int32_t cols = 0; - char *pWrite; + char * pWrite; char prefix[64] = {0}; tstrncpy(prefix, pShow->db, 64); @@ -891,4 +939,4 @@ static int32_t mndRetrieveStb(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3 static void mndCancelGetNextStb(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 5e9165f898..6a2fca836f 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -14,26 +14,54 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "mndInt.h" -#include "mndTrans.h" +#include "mndSync.h" -int32_t mndInitSync(SMnode *pMnode) { return 0; } -void mndCleanupSync(SMnode *pMnode) {} +int32_t mndInitSync(SMnode *pMnode) { + SSyncMgmt *pMgmt = &pMnode->syncMgmt; + tsem_init(&pMgmt->syncSem, 0, 0); + + pMgmt->state = TAOS_SYNC_STATE_LEADER; + pMgmt->pSyncNode = NULL; + return 0; +} + +void mndCleanupSync(SMnode *pMnode) { + SSyncMgmt *pMgmt = &pMnode->syncMgmt; + tsem_destroy(&pMgmt->syncSem); +} + +static int32_t mndSyncApplyCb(struct SSyncFSM *fsm, SyncIndex index, const SSyncBuffer *buf, void *pData) { + SMnode *pMnode = pData; + SSyncMgmt *pMgmt = &pMnode->syncMgmt; + + pMgmt->errCode = 0; + tsem_post(&pMgmt->syncSem); + + return 0; +} int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw) { - int32_t code = 0; +#if 1 + return 0; +#else + if (pMnode->replica == 1) return 0; - // int32_t len = sdbGetRawTotalSize(pRaw); - // SSdbRaw *pReceived = calloc(1, len); - // memcpy(pReceived, pRaw, len); - // mDebug("trans:%d, data:%p recv from sync, code:0x%x pMsg:%p", pMsg->id, pReceived, code & 0xFFFF, pMsg); + SSyncMgmt *pMgmt = &pMnode->syncMgmt; + pMgmt->errCode = 0; - // mndTransApply(pMnode, pReceived, code); - return code; + SSyncBuffer buf = {.data = pRaw, .len = sdbGetRawTotalSize(pRaw)}; + + bool isWeak = false; + int32_t code = syncPropose(pMgmt->pSyncNode, &buf, pMnode, isWeak); + + if (code != 0) return code; + + tsem_wait(&pMgmt->syncSem); + return pMgmt->errCode; +#endif } bool mndIsMaster(SMnode *pMnode) { - // pMnode->role = TAOS_SYNC_STATE_LEADER; - return true; + SSyncMgmt *pMgmt = &pMnode->syncMgmt; + return pMgmt->state == TAOS_SYNC_STATE_LEADER; } \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c new file mode 100644 index 0000000000..04f6907918 --- /dev/null +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -0,0 +1,857 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndDb.h" +#include "mndDnode.h" +#include "mndMnode.h" +#include "mndShow.h" +#include "mndStb.h" +#include "mndTrans.h" +#include "mndUser.h" +#include "mndVgroup.h" +#include "tname.h" + +#define MND_TOPIC_VER_NUMBER 1 +#define MND_TOPIC_RESERVE_SIZE 64 + +static SSdbRaw *mndTopicActionEncode(STopicObj *pTopic); +static SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw); +static int32_t mndTopicActionInsert(SSdb *pSdb, STopicObj *pTopic); +static int32_t mndTopicActionDelete(SSdb *pSdb, STopicObj *pTopic); +static int32_t mndTopicActionUpdate(SSdb *pSdb, STopicObj *pTopic, STopicObj *pNewTopic); +static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg); +static int32_t mndProcessAlterTopicMsg(SMnodeMsg *pMsg); +static int32_t mndProcessDropTopicMsg(SMnodeMsg *pMsg); +static int32_t mndProcessCreateTopicInRsp(SMnodeMsg *pMsg); +static int32_t mndProcessAlterTopicInRsp(SMnodeMsg *pMsg); +static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pMsg); +static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg); +static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveTopic(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter); + +int32_t mndInitTopic(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_TOPIC, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndTopicActionEncode, + .decodeFp = (SdbDecodeFp)mndTopicActionDecode, + .insertFp = (SdbInsertFp)mndTopicActionInsert, + .updateFp = (SdbUpdateFp)mndTopicActionUpdate, + .deleteFp = (SdbDeleteFp)mndTopicActionDelete}; + + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_TOPIC, mndProcessCreateTopicMsg); + mndSetMsgHandle(pMnode, TDMT_MND_ALTER_TOPIC, mndProcessAlterTopicMsg); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_TOPIC, mndProcessDropTopicMsg); + mndSetMsgHandle(pMnode, TDMT_VND_CREATE_TOPIC_RSP, mndProcessCreateTopicInRsp); + mndSetMsgHandle(pMnode, TDMT_VND_ALTER_TOPIC_RSP, mndProcessAlterTopicInRsp); + mndSetMsgHandle(pMnode, TDMT_VND_DROP_TOPIC_RSP, mndProcessDropTopicInRsp); + mndSetMsgHandle(pMnode, TDMT_VND_TABLE_META, mndProcessTopicMetaMsg); + + /*mndAddShowMetaHandle(pMnode, TSDB_MGMT_TOPIC, mndGetTopicMeta);*/ + /*mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TOPIC, mndRetrieveTopic);*/ + /*mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TOPIC, mndCancelGetNextTopic);*/ + + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_TOPIC, mndProcessCreateTopicMsg); + mndSetMsgHandle(pMnode, TDMT_VND_CREATE_TOPIC_RSP, mndProcessCreateTopicInRsp); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupTopic(SMnode *pMnode) {} + +static SSdbRaw *mndTopicActionEncode(STopicObj *pTopic) { + int32_t size = sizeof(STopicObj) + MND_TOPIC_RESERVE_SIZE; + SSdbRaw *pRaw = sdbAllocRaw(SDB_TOPIC, MND_TOPIC_VER_NUMBER, size); + if (pRaw == NULL) return NULL; + + int32_t dataPos = 0; + SDB_SET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TABLE_FNAME_LEN); + SDB_SET_BINARY(pRaw, dataPos, pTopic->db, TSDB_FULL_DB_NAME_LEN); + SDB_SET_INT64(pRaw, dataPos, pTopic->createTime); + SDB_SET_INT64(pRaw, dataPos, pTopic->updateTime); + SDB_SET_INT64(pRaw, dataPos, pTopic->uid); + SDB_SET_INT64(pRaw, dataPos, pTopic->dbUid); + SDB_SET_INT32(pRaw, dataPos, pTopic->version); + SDB_SET_INT32(pRaw, dataPos, pTopic->execLen); + SDB_SET_BINARY(pRaw, dataPos, pTopic->executor, pTopic->execLen); + SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen); + SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen); + + SDB_SET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE); + SDB_SET_DATALEN(pRaw, dataPos); + + return pRaw; +} + +static SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + + if (sver != MND_TOPIC_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + mError("failed to decode topic since %s", terrstr()); + return NULL; + } + + int32_t size = sizeof(STopicObj) + TSDB_MAX_COLUMNS * sizeof(SSchema); + SSdbRow *pRow = sdbAllocRow(size); + STopicObj *pTopic = sdbGetRowObj(pRow); + if (pTopic == NULL) return NULL; + + int32_t dataPos = 0; + SDB_GET_BINARY(pRaw, pRow, dataPos, pTopic->name, TSDB_TABLE_FNAME_LEN); + SDB_GET_BINARY(pRaw, pRow, dataPos, pTopic->db, TSDB_FULL_DB_NAME_LEN); + SDB_GET_INT64(pRaw, pRow, dataPos, &pTopic->createTime); + SDB_GET_INT64(pRaw, pRow, dataPos, &pTopic->updateTime); + SDB_GET_INT64(pRaw, pRow, dataPos, &pTopic->uid); + SDB_GET_INT64(pRaw, pRow, dataPos, &pTopic->dbUid); + SDB_GET_INT32(pRaw, pRow, dataPos, &pTopic->version); + SDB_GET_INT32(pRaw, pRow, dataPos, &pTopic->execLen); + SDB_GET_BINARY(pRaw, pRow, dataPos, pTopic->executor, pTopic->execLen); + SDB_GET_INT32(pRaw, pRow, dataPos, &pTopic->sqlLen); + SDB_GET_BINARY(pRaw, pRow, dataPos, pTopic->sql, pTopic->sqlLen); + + SDB_GET_RESERVE(pRaw, pRow, dataPos, MND_TOPIC_RESERVE_SIZE); + + return pRow; +} + +static int32_t mndTopicActionInsert(SSdb *pSdb, STopicObj *pTopic) { + mTrace("topic:%s, perform insert action", pTopic->name); + return 0; +} + +static int32_t mndTopicActionDelete(SSdb *pSdb, STopicObj *pTopic) { + mTrace("topic:%s, perform delete action", pTopic->name); + return 0; +} + +static int32_t mndTopicActionUpdate(SSdb *pSdb, STopicObj *pOldTopic, STopicObj *pNewTopic) { + mTrace("topic:%s, perform update action", pOldTopic->name); + atomic_exchange_32(&pOldTopic->updateTime, pNewTopic->updateTime); + atomic_exchange_32(&pOldTopic->version, pNewTopic->version); + + taosWLockLatch(&pOldTopic->lock); +#if 0 + + pOldTopic->numOfColumns = pNewTopic->numOfColumns; + pOldTopic->numOfTags = pNewTopic->numOfTags; + int32_t totalCols = pNewTopic->numOfTags + pNewTopic->numOfColumns; + int32_t totalSize = totalCols * sizeof(SSchema); + + if (pOldTopic->numOfTags + pOldTopic->numOfColumns < totalCols) { + void *pSchema = malloc(totalSize); + if (pSchema != NULL) { + free(pOldTopic->pSchema); + pOldTopic->pSchema = pSchema; + } + } + + memcpy(pOldTopic->pSchema, pNewTopic->pSchema, totalSize); + +#endif + taosWUnLockLatch(&pOldTopic->lock); + return 0; +} + +STopicObj *mndAcquireTopic(SMnode *pMnode, char *topicName) { + SSdb *pSdb = pMnode->pSdb; + STopicObj *pTopic = sdbAcquire(pSdb, SDB_TOPIC, topicName); + if (pTopic == NULL) { + terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; + } + return pTopic; +} + +void mndReleaseTopic(SMnode *pMnode, STopicObj *pTopic) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pTopic); +} + +static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) { + SName name = {0}; + tNameFromString(&name, topicName, T_NAME_ACCT | T_NAME_DB | T_NAME_TOPIC); + + char db[TSDB_TABLE_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, db); + + return mndAcquireDb(pMnode, db); +} + +static SCreateTopicInternalMsg *mndBuildCreateTopicMsg(SMnode *pMnode, SVgObj *pVgroup, STopicObj *pTopic) { + int32_t totalCols = 0; + int32_t contLen = sizeof(SCreateTopicInternalMsg) + pTopic->execLen + pTopic->sqlLen; + + SCreateTopicInternalMsg *pCreate = calloc(1, contLen); + if (pCreate == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pCreate->head.contLen = htonl(contLen); + pCreate->head.vgId = htonl(pVgroup->vgId); + memcpy(pCreate->name, pTopic->name, TSDB_TABLE_FNAME_LEN); + pCreate->tuid = htobe64(pTopic->uid); + pCreate->sverson = htonl(pTopic->version); + + pCreate->sql = malloc(pTopic->sqlLen); + if (pCreate->sql == NULL) { + free(pCreate); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + memcpy(pCreate->sql, pTopic->sql, pTopic->sqlLen); + + pCreate->executor = malloc(pTopic->execLen); + if (pCreate->executor == NULL) { + free(pCreate); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + memcpy(pCreate->executor, pTopic->executor, pTopic->execLen); + + return pCreate; +} + +static SDropTopicInternalMsg *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, STopicObj *pTopic) { + int32_t contLen = sizeof(SDropTopicInternalMsg); + + SDropTopicInternalMsg *pDrop = calloc(1, contLen); + if (pDrop == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pDrop->head.contLen = htonl(contLen); + pDrop->head.vgId = htonl(pVgroup->vgId); + memcpy(pDrop->name, pTopic->name, TSDB_TABLE_FNAME_LEN); + pDrop->tuid = htobe64(pTopic->uid); + + return pDrop; +} + +static int32_t mndCheckCreateTopicMsg(SCreateTopicMsg *pCreate) { + // deserialize and other stuff + return 0; +} + +static int32_t mndSetCreateTopicRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, STopicObj *pTopic) { + SSdbRaw *pRedoRaw = mndTopicActionEncode(pTopic); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; + + return 0; +} + +static int32_t mndSetCreateTopicUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, STopicObj *pTopic) { + SSdbRaw *pUndoRaw = mndTopicActionEncode(pTopic); + if (pUndoRaw == NULL) return -1; + if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; + + return 0; +} + +static int32_t mndSetCreateTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, STopicObj *pTopic) { + SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +static int32_t mndSetCreateTopicRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, STopicObj *pTopic) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pDb->uid) continue; + + SCreateTopicInternalMsg *pMsg = mndBuildCreateTopicMsg(pMnode, pVgroup, pTopic); + if (pMsg == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pMsg; + action.contLen = htonl(pMsg->head.contLen); + action.msgType = TDMT_VND_CREATE_TOPIC; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + +static int32_t mndSetCreateTopicUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, STopicObj *pTopic) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pDb->uid) continue; + + SDropTopicInternalMsg *pMsg = mndBuildDropTopicMsg(pMnode, pVgroup, pTopic); + if (pMsg == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pMsg; + action.contLen = sizeof(SDropTopicInternalMsg); + action.msgType = TDMT_VND_DROP_TOPIC; + if (mndTransAppendUndoAction(pTrans, &action) != 0) { + free(pMsg); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + +static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCreateTopicMsg *pCreate, SDbObj *pDb) { + STopicObj topicObj = {0}; + tstrncpy(topicObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); + tstrncpy(topicObj.db, pDb->name, TSDB_FULL_DB_NAME_LEN); + topicObj.createTime = taosGetTimestampMs(); + topicObj.updateTime = topicObj.createTime; + topicObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); + topicObj.dbUid = pDb->uid; + topicObj.version = 1; + +#if 0 + int32_t totalCols = topicObj.numOfColumns + topicObj.numOfTags; + int32_t totalSize = totalCols * sizeof(SSchema); + topicObj.sql = malloc(totalSize); + if (topicObj.sql == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + memcpy(topicObj.sql, pCreate->sql, totalSize); +#endif + + int32_t code = 0; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); + if (pTrans == NULL) { + mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + mDebug("trans:%d, used to create topic:%s", pTrans->id, pCreate->name); + + if (mndSetCreateTopicRedoLogs(pMnode, pTrans, pDb, &topicObj) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto CREATE_TOPIC_OVER; + } + + if (mndSetCreateTopicUndoLogs(pMnode, pTrans, pDb, &topicObj) != 0) { + mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); + goto CREATE_TOPIC_OVER; + } + + if (mndSetCreateTopicCommitLogs(pMnode, pTrans, pDb, &topicObj) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto CREATE_TOPIC_OVER; + } + + if (mndSetCreateTopicRedoActions(pMnode, pTrans, pDb, &topicObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto CREATE_TOPIC_OVER; + } + + if (mndSetCreateTopicUndoActions(pMnode, pTrans, pDb, &topicObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto CREATE_TOPIC_OVER; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + code = 0; + +CREATE_TOPIC_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SCreateTopicMsg *pCreate = pMsg->rpcMsg.pCont; + + mDebug("topic:%s, start to create", pCreate->name); + + if (mndCheckCreateTopicMsg(pCreate) != 0) { + mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + STopicObj *pTopic = mndAcquireTopic(pMnode, pCreate->name); + if (pTopic != NULL) { + sdbRelease(pMnode->pSdb, pTopic); + if (pCreate->igExists) { + mDebug("topic:%s, already exist, ignore exist is set", pCreate->name); + return 0; + } else { + terrno = TSDB_CODE_MND_TOPIC_ALREADY_EXIST; + mError("db:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + } + + // topic should have different name with stb + SStbObj *pStb = mndAcquireStb(pMnode, pCreate->name); + if (pStb != NULL) { + sdbRelease(pMnode->pSdb, pStb); + terrno = TSDB_CODE_MND_NAME_CONFLICT_WITH_STB; + mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + SDbObj *pDb = mndAcquireDbByTopic(pMnode, pCreate->name); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + int32_t code = mndCreateTopic(pMnode, pMsg, pCreate, pDb); + mndReleaseDb(pMnode, pDb); + + if (code != 0) { + terrno = code; + mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndCheckAlterTopicMsg(SAlterTopicMsg *pAlter) { + SSchema *pSchema = &pAlter->schema; + pSchema->colId = htonl(pSchema->colId); + pSchema->bytes = htonl(pSchema->bytes); + + if (pSchema->type <= 0) { + terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; + return -1; + } + if (pSchema->colId < 0 || pSchema->colId >= (TSDB_MAX_COLUMNS + TSDB_MAX_TAGS)) { + terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; + return -1; + } + if (pSchema->bytes <= 0) { + terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; + return -1; + } + if (pSchema->name[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; + return -1; + } + + return 0; +} + +static int32_t mndUpdateTopic(SMnode *pMnode, SMnodeMsg *pMsg, STopicObj *pOldTopic, STopicObj *pNewTopic) { return 0; } + +static int32_t mndProcessAlterTopicMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SAlterTopicMsg *pAlter = pMsg->rpcMsg.pCont; + + mDebug("topic:%s, start to alter", pAlter->name); + + if (mndCheckAlterTopicMsg(pAlter) != 0) { + mError("topic:%s, failed to alter since %s", pAlter->name, terrstr()); + return -1; + } + + STopicObj *pTopic = mndAcquireTopic(pMnode, pAlter->name); + if (pTopic == NULL) { + terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; + mError("topic:%s, failed to alter since %s", pAlter->name, terrstr()); + return -1; + } + + STopicObj topicObj = {0}; + memcpy(&topicObj, pTopic, sizeof(STopicObj)); + + int32_t code = mndUpdateTopic(pMnode, pMsg, pTopic, &topicObj); + mndReleaseTopic(pMnode, pTopic); + + if (code != 0) { + mError("topic:%s, failed to alter since %s", pAlter->name, tstrerror(code)); + return code; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessAlterTopicInRsp(SMnodeMsg *pMsg) { + mndTransHandleActionRsp(pMsg); + return 0; +} + +static int32_t mndSetDropTopicRedoLogs(SMnode *pMnode, STrans *pTrans, STopicObj *pTopic) { + SSdbRaw *pRedoRaw = mndTopicActionEncode(pTopic); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropTopicUndoLogs(SMnode *pMnode, STrans *pTrans, STopicObj *pTopic) { + SSdbRaw *pUndoRaw = mndTopicActionEncode(pTopic); + if (pUndoRaw == NULL) return -1; + if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropTopicCommitLogs(SMnode *pMnode, STrans *pTrans, STopicObj *pTopic) { + SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropTopicRedoActions(SMnode *pMnode, STrans *pTrans, STopicObj *pTopic) { return 0; } + +static int32_t mndSetDropTopicUndoActions(SMnode *pMnode, STrans *pTrans, STopicObj *pTopic) { return 0; } + +static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pMsg, STopicObj *pTopic) { + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); + if (pTrans == NULL) { + mError("topic:%s, failed to drop since %s", pTopic->name, terrstr()); + return -1; + } + mDebug("trans:%d, used to drop topic:%s", pTrans->id, pTopic->name); + + if (mndSetDropTopicRedoLogs(pMnode, pTrans, pTopic) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto DROP_TOPIC_OVER; + } + + if (mndSetDropTopicUndoLogs(pMnode, pTrans, pTopic) != 0) { + mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); + goto DROP_TOPIC_OVER; + } + + if (mndSetDropTopicCommitLogs(pMnode, pTrans, pTopic) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto DROP_TOPIC_OVER; + } + + if (mndSetDropTopicRedoActions(pMnode, pTrans, pTopic) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto DROP_TOPIC_OVER; + } + + if (mndSetDropTopicUndoActions(pMnode, pTrans, pTopic) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto DROP_TOPIC_OVER; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + goto DROP_TOPIC_OVER; + } + + code = 0; + +DROP_TOPIC_OVER: + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessDropTopicMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SDropTopicMsg *pDrop = pMsg->rpcMsg.pCont; + + mDebug("topic:%s, start to drop", pDrop->name); + + STopicObj *pTopic = mndAcquireTopic(pMnode, pDrop->name); + if (pTopic == NULL) { + if (pDrop->igNotExists) { + mDebug("topic:%s, not exist, ignore not exist is set", pDrop->name); + return 0; + } else { + terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; + mError("topic:%s, failed to drop since %s", pDrop->name, terrstr()); + return -1; + } + } + + int32_t code = mndDropTopic(pMnode, pMsg, pTopic); + mndReleaseTopic(pMnode, pTopic); + + if (code != 0) { + terrno = code; + mError("topic:%s, failed to drop since %s", pDrop->name, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pMsg) { + mndTransHandleActionRsp(pMsg); + return 0; +} + +static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + STableInfoMsg *pInfo = pMsg->rpcMsg.pCont; + + mDebug("topic:%s, start to retrieve meta", pInfo->tableFname); + +#if 0 + SDbObj *pDb = mndAcquireDbByTopic(pMnode, pInfo->tableFname); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + mError("topic:%s, failed to retrieve meta since %s", pInfo->tableFname, terrstr()); + return -1; + } + + STopicObj *pTopic = mndAcquireTopic(pMnode, pInfo->tableFname); + if (pTopic == NULL) { + mndReleaseDb(pMnode, pDb); + terrno = TSDB_CODE_MND_INVALID_TOPIC; + mError("topic:%s, failed to get meta since %s", pInfo->tableFname, terrstr()); + return -1; + } + + taosRLockLatch(&pTopic->lock); + int32_t totalCols = pTopic->numOfColumns + pTopic->numOfTags; + int32_t contLen = sizeof(STableMetaMsg) + totalCols * sizeof(SSchema); + + STableMetaMsg *pMeta = rpcMallocCont(contLen); + if (pMeta == NULL) { + taosRUnLockLatch(&pTopic->lock); + mndReleaseDb(pMnode, pDb); + mndReleaseTopic(pMnode, pTopic); + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("topic:%s, failed to get meta since %s", pInfo->tableFname, terrstr()); + return -1; + } + + memcpy(pMeta->topicFname, pTopic->name, TSDB_TABLE_FNAME_LEN); + pMeta->numOfTags = htonl(pTopic->numOfTags); + pMeta->numOfColumns = htonl(pTopic->numOfColumns); + pMeta->precision = pDb->cfg.precision; + pMeta->tableType = TSDB_SUPER_TABLE; + pMeta->update = pDb->cfg.update; + pMeta->sversion = htonl(pTopic->version); + pMeta->tuid = htonl(pTopic->uid); + + for (int32_t i = 0; i < totalCols; ++i) { + SSchema *pSchema = &pMeta->pSchema[i]; + SSchema *pSrcSchema = &pTopic->pSchema[i]; + memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); + pSchema->type = pSrcSchema->type; + pSchema->colId = htonl(pSrcSchema->colId); + pSchema->bytes = htonl(pSrcSchema->bytes); + } + taosRUnLockLatch(&pTopic->lock); + mndReleaseDb(pMnode, pDb); + mndReleaseTopic(pMnode, pTopic); + + pMsg->pCont = pMeta; + pMsg->contLen = contLen; + + mDebug("topic:%s, meta is retrieved, cols:%d tags:%d", pInfo->tableFname, pTopic->numOfColumns, pTopic->numOfTags); +#endif + return 0; +} + +static int32_t mndProcessCreateTopicInRsp(SMnodeMsg *pMsg) { + mndTransHandleActionRsp(pMsg); + return 0; +} + +static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) { + SSdb *pSdb = pMnode->pSdb; + + SDbObj *pDb = mndAcquireDb(pMnode, dbName); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + return -1; + } + + int32_t numOfTopics = 0; + void *pIter = NULL; + while (1) { + STopicObj *pTopic = NULL; + pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic); + if (pIter == NULL) break; + + if (strcmp(pTopic->db, dbName) == 0) { + numOfTopics++; + } + + sdbRelease(pSdb, pTopic); + } + + *pNumOfTopics = numOfTopics; + return 0; +} + +static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + if (mndGetNumOfTopics(pMnode, pShow->db, &pShow->numOfRows) != 0) { + return -1; + } + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "name"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "columns"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "tags"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htonl(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_TOPIC); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + + return 0; +} + +static void mndExtractTableName(char *tableId, char *name) { + int32_t pos = -1; + int32_t num = 0; + for (pos = 0; tableId[pos] != 0; ++pos) { + if (tableId[pos] == '.') num++; + if (num == 2) break; + } + + if (num == 2) { + strcpy(name, tableId + pos + 1); + } +} + +static int32_t mndRetrieveTopic(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + STopicObj *pTopic = NULL; + int32_t cols = 0; + char *pWrite; + char prefix[64] = {0}; + + tstrncpy(prefix, pShow->db, 64); + strcat(prefix, TS_PATH_DELIMITER); + int32_t prefixLen = (int32_t)strlen(prefix); + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_TOPIC, pShow->pIter, (void **)&pTopic); + if (pShow->pIter == NULL) break; + + if (strncmp(pTopic->name, prefix, prefixLen) != 0) { + sdbRelease(pSdb, pTopic); + continue; + } + + cols = 0; + + char topicName[TSDB_TABLE_NAME_LEN] = {0}; + tstrncpy(topicName, pTopic->name + prefixLen, TSDB_TABLE_NAME_LEN); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, topicName); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pTopic->createTime; + cols++; + + /*pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;*/ + /**(int32_t *)pWrite = pTopic->numOfColumns;*/ + /*cols++;*/ + + /*pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;*/ + /**(int32_t *)pWrite = pTopic->numOfTags;*/ + /*cols++;*/ + + numOfRows++; + sdbRelease(pSdb, pTopic); + } + + pShow->numOfReads += numOfRows; + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + return numOfRows; +} + +static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 53e3bc5203..0b714c34ae 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -17,9 +17,9 @@ #include "mndTrans.h" #include "mndSync.h" -#define TSDB_TRANS_VER_NUMBER 1 -#define TSDB_TRN_ARRAY_SIZE 8 -#define TSDB_TRN_RESERVE_SIZE 64 +#define MND_TRANS_VER_NUMBER 1 +#define MND_TRANS_ARRAY_SIZE 8 +#define MND_TRANS_RESERVE_SIZE 64 static SSdbRaw *mndTransActionEncode(STrans *pTrans); static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); @@ -27,7 +27,6 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans); static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOldTrans); static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans); -static void mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle); static void mndTransSendRpcRsp(STrans *pTrans, int32_t code); static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw); static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction); @@ -40,7 +39,7 @@ static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans); -static int32_t mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans); +static void mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans); static int32_t mndTransPerformExecuteStage(SMnode *pMnode, STrans *pTrans); static int32_t mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans); static int32_t mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans); @@ -61,7 +60,7 @@ int32_t mndInitTrans(SMnode *pMnode) { void mndCleanupTrans(SMnode *pMnode) {} static SSdbRaw *mndTransActionEncode(STrans *pTrans) { - int32_t rawDataLen = sizeof(STrans) + TSDB_TRN_RESERVE_SIZE; + int32_t rawDataLen = sizeof(STrans) + MND_TRANS_RESERVE_SIZE; int32_t redoLogNum = taosArrayGetSize(pTrans->redoLogs); int32_t undoLogNum = taosArrayGetSize(pTrans->undoLogs); int32_t commitLogNum = taosArrayGetSize(pTrans->commitLogs); @@ -93,7 +92,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { rawDataLen += (sizeof(STransAction) + pAction->contLen); } - SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TSDB_TRANS_VER_NUMBER, rawDataLen); + SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, MND_TRANS_VER_NUMBER, rawDataLen); if (pRaw == NULL) { mError("trans:%d, failed to alloc raw since %s", pTrans->id, terrstr()); return NULL; @@ -132,7 +131,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { for (int32_t i = 0; i < redoActionNum; ++i) { STransAction *pAction = taosArrayGet(pTrans->redoActions, i); SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet)); - SDB_SET_INT8(pRaw, dataPos, pAction->msgType) + SDB_SET_INT16(pRaw, dataPos, pAction->msgType) SDB_SET_INT32(pRaw, dataPos, pAction->contLen) SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen); } @@ -140,12 +139,12 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { for (int32_t i = 0; i < undoActionNum; ++i) { STransAction *pAction = taosArrayGet(pTrans->undoActions, i); SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet)); - SDB_SET_INT8(pRaw, dataPos, pAction->msgType) + SDB_SET_INT16(pRaw, dataPos, pAction->msgType) SDB_SET_INT32(pRaw, dataPos, pAction->contLen) SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pCont, pAction->contLen); } - SDB_SET_RESERVE(pRaw, dataPos, TSDB_TRN_RESERVE_SIZE) + SDB_SET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE) SDB_SET_DATALEN(pRaw, dataPos); mTrace("trans:%d, encode to raw:%p, len:%d", pTrans->id, pRaw, dataPos); return pRaw; @@ -155,12 +154,9 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { int32_t code = 0; int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) { - mError("failed to get soft ver from raw:%p since %s", pRaw, terrstr()); - return NULL; - } + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != TSDB_TRANS_VER_NUMBER) { + if (sver != MND_TRANS_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; mError("failed to get check soft ver from raw:%p since %s", pRaw, terrstr()); return NULL; @@ -173,11 +169,11 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { return NULL; } - pTrans->redoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); - pTrans->undoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); - pTrans->commitLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); - pTrans->redoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(STransAction)); - pTrans->undoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(STransAction)); + pTrans->redoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); + pTrans->undoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); + pTrans->commitLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); + pTrans->redoActions = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(STransAction)); + pTrans->undoActions = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(STransAction)); if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL || pTrans->redoActions == NULL || pTrans->undoActions == NULL) { @@ -243,7 +239,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < redoActionNum; ++i) { STransAction action = {0}; SDB_GET_BINARY(pRaw, pRow, dataPos, (void *)&action.epSet, sizeof(SEpSet)); - SDB_GET_INT8(pRaw, pRow, dataPos, &action.msgType) + SDB_GET_INT16(pRaw, pRow, dataPos, &action.msgType) SDB_GET_INT32(pRaw, pRow, dataPos, &action.contLen) action.pCont = malloc(action.contLen); if (action.pCont == NULL) { @@ -262,7 +258,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < undoActionNum; ++i) { STransAction action = {0}; SDB_GET_BINARY(pRaw, pRow, dataPos, (void *)&action.epSet, sizeof(SEpSet)); - SDB_GET_INT8(pRaw, pRow, dataPos, &action.msgType) + SDB_GET_INT16(pRaw, pRow, dataPos, &action.msgType) SDB_GET_INT32(pRaw, pRow, dataPos, &action.contLen) action.pCont = malloc(action.contLen); if (action.pCont == NULL) { @@ -278,7 +274,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { } } - SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_TRN_RESERVE_SIZE) + SDB_GET_RESERVE(pRaw, pRow, dataPos, MND_TRANS_RESERVE_SIZE) TRANS_DECODE_OVER: if (code != 0) { @@ -358,7 +354,7 @@ char *mndTransPolicyStr(ETrnPolicy policy) { } } -STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) { +STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, SRpcMsg *pMsg) { STrans *pTrans = calloc(1, sizeof(STrans)); if (pTrans == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -369,12 +365,13 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) { pTrans->id = sdbGetMaxId(pMnode->pSdb, SDB_TRANS); pTrans->stage = TRN_STAGE_PREPARE; pTrans->policy = policy; - pTrans->rpcHandle = rpcHandle; - pTrans->redoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); - pTrans->undoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); - pTrans->commitLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); - pTrans->redoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(STransAction)); - pTrans->undoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(STransAction)); + pTrans->rpcHandle = pMsg->handle; + pTrans->rpcAHandle = pMsg->ahandle; + pTrans->redoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); + pTrans->undoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); + pTrans->commitLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); + pTrans->redoActions = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(STransAction)); + pTrans->undoActions = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(STransAction)); if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL || pTrans->redoActions == NULL || pTrans->undoActions == NULL) { @@ -416,11 +413,6 @@ void mndTransDrop(STrans *pTrans) { tfree(pTrans); } -static void mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle) { - pTrans->rpcHandle = rpcHandle; - mTrace("trans:%d, set rpc handle:%p", pTrans->id, rpcHandle); -} - static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw) { if (pArray == NULL || pRaw == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -466,13 +458,13 @@ static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction) { int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction) { int32_t code = mndTransAppendAction(pTrans->redoActions, pAction); - mTrace("trans:%d, msg:%s append to redo actions, code:0x%x", pTrans->id, taosMsg[pAction->msgType], code); + mTrace("trans:%d, msg:%s append to redo actions, code:0x%x", pTrans->id, TMSG_INFO(pAction->msgType), code); return code; } int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction) { int32_t code = mndTransAppendAction(pTrans->undoActions, pAction); - mTrace("trans:%d, msg:%s append to undo actions, code:0x%x", pTrans->id, taosMsg[pAction->msgType], code); + mTrace("trans:%d, msg:%s append to undo actions, code:0x%x", pTrans->id, TMSG_INFO(pAction->msgType), code); return code; } @@ -504,11 +496,12 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { STrans *pNewTrans = mndAcquireTrans(pMnode, pTrans->id); if (pNewTrans == NULL) { - mError("trans:%d, failed to ready from sdb since %s", pTrans->id, terrstr()); + mError("trans:%d, failed to read from sdb since %s", pTrans->id, terrstr()); return -1; } pNewTrans->rpcHandle = pTrans->rpcHandle; + pNewTrans->rpcAHandle = pTrans->rpcAHandle; mndTransExecute(pMnode, pNewTrans); mndReleaseTrans(pMnode, pNewTrans); return 0; @@ -526,19 +519,17 @@ int32_t mndTransCommit(SMnode *pMnode, STrans *pTrans) { if (taosArrayGetSize(pTrans->commitLogs) != 0) { mTrace("trans:%d, sync to other nodes", pTrans->id); - int32_t code = mndSyncPropose(pMnode, pRaw); - if (code != 0) { + if (mndSyncPropose(pMnode, pRaw) != 0) { mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); sdbFreeRaw(pRaw); return -1; } - mTrace("trans:%d, sync finished", pTrans->id); - code = sdbWrite(pMnode->pSdb, pRaw); - if (code != 0) { - mError("trans:%d, failed to write sdb since %s", pTrans->id, terrstr()); - return -1; - } + } + + if (sdbWrite(pMnode->pSdb, pRaw) != 0) { + mError("trans:%d, failed to write sdb since %s", pTrans->id, terrstr()); + return -1; } mDebug("trans:%d, commit finished", pTrans->id); @@ -576,10 +567,11 @@ int32_t mndTransRollback(SMnode *pMnode, STrans *pTrans) { static void mndTransSendRpcRsp(STrans *pTrans, int32_t code) { if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return; - mDebug("trans:%d, send rpc rsp, RPC:%p code:0x%x", pTrans->id, pTrans->rpcHandle, code & 0xFFFF); + mDebug("trans:%d, send rpc rsp, RPC:%p ahandle:%p code:0x%x", pTrans->id, pTrans->rpcHandle, pTrans->rpcAHandle, + code & 0xFFFF); if (pTrans->rpcHandle != NULL) { - SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, .code = code}; + SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, .code = code, .ahandle = pTrans->rpcAHandle}; rpcSendResponse(&rspMsg); } } @@ -614,7 +606,7 @@ void mndTransHandleActionRsp(SMnodeMsg *pMsg) { } int32_t actionNum = taosArrayGetSize(pTrans->redoActions); - if (action < 0 || action > actionNum) { + if (action < 0 || action >= actionNum) { mError("trans:%d, invalid action:%d", transId, action); goto HANDLE_ACTION_RSP_OVER; } @@ -636,6 +628,8 @@ static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) { SSdb *pSdb = pMnode->pSdb; int32_t arraySize = taosArrayGetSize(pArray); + if (arraySize == 0) return 0; + for (int32_t i = 0; i < arraySize; ++i) { SSdbRaw *pRaw = taosArrayGetP(pArray, i); int32_t code = sdbWriteNotFree(pSdb, pRaw); @@ -648,45 +642,15 @@ static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) { } static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans) { - int32_t code = 0; - if (taosArrayGetSize(pTrans->redoLogs) != 0) { - code = mndTransExecuteLogs(pMnode, pTrans->redoLogs); - if (code != 0) { - mError("trans:%d, failed to execute redo logs since %s", pTrans->id, terrstr()) - } else { - mDebug("trans:%d, execute redo logs finished", pTrans->id) - } - } - - return code; + return mndTransExecuteLogs(pMnode, pTrans->redoLogs); } static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans) { - int32_t code = 0; - if (taosArrayGetSize(pTrans->undoLogs) != 0) { - code = mndTransExecuteLogs(pMnode, pTrans->undoLogs); - if (code != 0) { - mError("trans:%d, failed to execute undo logs since %s", pTrans->id, terrstr()) - } else { - mDebug("trans:%d, execute undo logs finished", pTrans->id) - } - } - - return code; + return mndTransExecuteLogs(pMnode, pTrans->undoLogs); } static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans) { - int32_t code = 0; - if (taosArrayGetSize(pTrans->commitLogs) != 0) { - code = mndTransExecuteLogs(pMnode, pTrans->commitLogs); - if (code != 0) { - mError("trans:%d, failed to execute commit logs since %s", pTrans->id, terrstr()) - } else { - mDebug("trans:%d, execute commit logs finished", pTrans->id) - } - } - - return code; + return mndTransExecuteLogs(pMnode, pTrans->commitLogs); } static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) { @@ -696,7 +660,9 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA for (int32_t action = 0; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pArray, action); if (pAction == NULL) continue; - if (pAction->msgReceived && pAction->errCode == 0) continue; + // if (pAction->msgSent && !pAction->msgReceived) continue; + // if (pAction->msgSent && pAction->msgReceived && pAction->errCode == 0) continue; + if (pAction->msgSent) continue; int64_t signature = pTrans->id; signature = (signature << 32); @@ -718,25 +684,25 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA mndSendMsgToDnode(pMnode, &pAction->epSet, &rpcMsg); } - int32_t numOfReceivedMsgs = 0; - int32_t errorCode = 0; + int32_t numOfReceived = 0; + int32_t errCode = 0; for (int32_t action = 0; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pArray, action); if (pAction == NULL) continue; if (pAction->msgSent && pAction->msgReceived) { - numOfReceivedMsgs++; + numOfReceived++; if (pAction->errCode != 0) { - errorCode = pAction->errCode; + errCode = pAction->errCode; } } } - if (numOfReceivedMsgs == numOfActions) { - mDebug("trans:%d, all %d actions executed, code:0x%x", pTrans->id, numOfActions, errorCode); - terrno = errorCode; - return errorCode; + if (numOfReceived == numOfActions) { + mDebug("trans:%d, all %d actions executed, code:0x%x", pTrans->id, numOfActions, errCode); + terrno = errCode; + return errCode; } else { - mDebug("trans:%d, %d of %d actions executed, code:0x%x", pTrans->id, numOfReceivedMsgs, numOfActions, errorCode); + mDebug("trans:%d, %d of %d actions executed, code:0x%x", pTrans->id, numOfReceived, numOfActions, errCode); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } } @@ -749,7 +715,7 @@ static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) { return mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions); } -static int32_t mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { +static void mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { int32_t code = mndTransExecuteRedoLogs(pMnode, pTrans); if (code == 0) { @@ -759,8 +725,6 @@ static int32_t mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { pTrans->stage = TRN_STAGE_ROLLBACK; mError("trans:%d, stage from prepare to rollback since %s", pTrans->id, terrstr()); } - - return 0; } static int32_t mndTransPerformExecuteStage(SMnode *pMnode, STrans *pTrans) { @@ -771,18 +735,18 @@ static int32_t mndTransPerformExecuteStage(SMnode *pMnode, STrans *pTrans) { mDebug("trans:%d, stage from execute to commit", pTrans->id); } else if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { mDebug("trans:%d, stage keep on execute since %s", pTrans->id, tstrerror(code)); - return code; } else { if (pTrans->policy == TRN_POLICY_ROLLBACK) { pTrans->stage = TRN_STAGE_ROLLBACK; mError("trans:%d, stage from execute to rollback since %s", pTrans->id, terrstr()); } else { pTrans->stage = TRN_STAGE_EXECUTE; + pTrans->retryTimes++; mError("trans:%d, stage keep on execute since %s", pTrans->id, terrstr()); } } - return 0; + return code; } static int32_t mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) { @@ -798,6 +762,7 @@ static int32_t mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { mDebug("trans:%d, rollbacked", pTrans->id); } else { pTrans->stage = TRN_STAGE_ROLLBACK; + pTrans->retryTimes++; mError("trans:%d, stage keep on rollback since %s", pTrans->id, terrstr()); } @@ -810,7 +775,7 @@ static void mndTransExecute(SMnode *pMnode, STrans *pTrans) { while (code == 0) { switch (pTrans->stage) { case TRN_STAGE_PREPARE: - code = mndTransPerformPrepareStage(pMnode, pTrans); + mndTransPerformPrepareStage(pMnode, pTrans); break; case TRN_STAGE_EXECUTE: code = mndTransPerformExecuteStage(pMnode, pTrans); @@ -822,9 +787,9 @@ static void mndTransExecute(SMnode *pMnode, STrans *pTrans) { } break; case TRN_STAGE_ROLLBACK: - code = mndTransPerformRollbackStage(pMnode, pTrans); + code = mndTransRollback(pMnode, pTrans); if (code == 0) { - code = mndTransRollback(pMnode, pTrans); + mndTransPerformRollbackStage(pMnode, pTrans); } break; default: diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index c96e6a80db..0926cf0c48 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -47,9 +47,9 @@ int32_t mndInitUser(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndUserActionUpdate, .deleteFp = (SdbDeleteFp)mndUserActionDelete}; - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_USER, mndProcessCreateUserMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_USER, mndProcessAlterUserMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_USER, mndProcessDropUserMsg); + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_USER, mndProcessCreateUserMsg); + mndSetMsgHandle(pMnode, TDMT_MND_ALTER_USER, mndProcessAlterUserMsg); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_USER, mndProcessDropUserMsg); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_USER, mndGetUserMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_USER, mndRetrieveUsers); @@ -197,7 +197,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, userObj.updateTime = userObj.createdTime; userObj.superUser = 0; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); if (pTrans == NULL) { mError("user:%s, failed to create since %s", user, terrstr()); return -1; @@ -267,7 +267,7 @@ static int32_t mndProcessCreateUserMsg(SMnodeMsg *pMsg) { } static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOldUser, SUserObj *pNewUser, SMnodeMsg *pMsg) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); if (pTrans == NULL) { mError("user:%s, failed to update since %s", pOldUser->user, terrstr()); return -1; @@ -342,7 +342,7 @@ static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg) { } static int32_t mndDropUser(SMnode *pMnode, SMnodeMsg *pMsg, SUserObj *pUser) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); if (pTrans == NULL) { mError("user:%s, failed to drop since %s", pUser->user, terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index c65436efcb..8ff2139314 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -51,11 +51,11 @@ int32_t mndInitVgroup(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndVgroupActionDelete, .deleteFp = (SdbDeleteFp)mndVgroupActionUpdate}; - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_VNODE_IN_RSP, mndProcessCreateVnodeRsp); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_VNODE_IN_RSP, mndProcessAlterVnodeRsp); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_VNODE_IN_RSP, mndProcessDropVnodeRsp); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_SYNC_VNODE_IN_RSP, mndProcessSyncVnodeRsp); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_COMPACT_VNODE_IN_RSP, mndProcessCompactVnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndProcessCreateVnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_ALTER_VNODE_RSP, mndProcessAlterVnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_DROP_VNODE_RSP, mndProcessDropVnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_SYNC_VNODE_RSP, mndProcessSyncVnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_COMPACT_VNODE_RSP, mndProcessCompactVnodeRsp); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndGetVgroupMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndRetrieveVgroups); @@ -337,8 +337,16 @@ static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg) { + mndTransHandleActionRsp(pMsg); + return 0; +} + +static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg) { + mndTransHandleActionRsp(pMsg); + return 0; +} + static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index a62a0a9296..27668a585a 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -27,6 +27,7 @@ #include "mndStb.h" #include "mndSync.h" #include "mndTelem.h" +#include "mndTopic.h" #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" @@ -132,6 +133,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1; if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1; if (pMnode->clusterId <= 0) { if (mndAllocStep(pMnode, "mnode-sdb-deploy", mndDeploySdb, NULL) != 0) return -1; @@ -200,7 +202,6 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->selfIndex = pOption->selfIndex; memcpy(&pMnode->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); pMnode->pDnode = pOption->pDnode; - pMnode->putMsgToApplyMsgFp = pOption->putMsgToApplyMsgFp; pMnode->sendMsgToDnodeFp = pOption->sendMsgToDnodeFp; pMnode->sendMsgToMnodeFp = pOption->sendMsgToMnodeFp; pMnode->sendRedirectMsgFp = pOption->sendRedirectMsgFp; @@ -215,8 +216,7 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->cfg.buildinfo = strdup(pOption->cfg.buildinfo); if (pMnode->sendMsgToDnodeFp == NULL || pMnode->sendMsgToMnodeFp == NULL || pMnode->sendRedirectMsgFp == NULL || - pMnode->putMsgToApplyMsgFp == NULL || pMnode->dnodeId < 0 || pMnode->clusterId < 0 || - pMnode->cfg.statusInterval < 1) { + pMnode->dnodeId < 0 || pMnode->clusterId < 0 || pMnode->cfg.statusInterval < 1) { terrno = TSDB_CODE_MND_INVALID_OPTIONS; return -1; } @@ -227,7 +227,7 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { } return 0; -} +} SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { mDebug("start to open mnode in %s", path); @@ -372,11 +372,11 @@ void mndSendRsp(SMnodeMsg *pMsg, int32_t code) { static void mndProcessRpcMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; int32_t code = 0; - int32_t msgType = pMsg->rpcMsg.msgType; + tmsg_t msgType = pMsg->rpcMsg.msgType; void *ahandle = pMsg->rpcMsg.ahandle; bool isReq = (msgType & 1U); - mTrace("msg:%p, app:%p type:%s will be processed", pMsg, ahandle, taosMsg[msgType]); + mTrace("msg:%p, app:%p type:%s will be processed", pMsg, ahandle, TMSG_INFO(msgType)); if (isReq && !mndIsMaster(pMnode)) { code = TSDB_CODE_APP_NOT_READY; @@ -390,10 +390,10 @@ static void mndProcessRpcMsg(SMnodeMsg *pMsg) { goto PROCESS_RPC_END; } - MndMsgFp fp = pMnode->msgFp[msgType]; + MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(msgType)]; if (fp == NULL) { code = TSDB_CODE_MSG_NOT_PROCESSED; - mError("msg:%p, app:%p failed to process since not handle", pMsg, ahandle); + mError("msg:%p, app:%p failed to process since no handle", pMsg, ahandle); goto PROCESS_RPC_END; } @@ -423,9 +423,10 @@ PROCESS_RPC_END: } } -void mndSetMsgHandle(SMnode *pMnode, int32_t msgType, MndMsgFp fp) { - if (msgType >= 0 && msgType < TSDB_MSG_TYPE_MAX) { - pMnode->msgFp[msgType] = fp; +void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) { + tmsg_t type = TMSG_INDEX(msgType); + if (type >= 0 && type < TDMT_MAX) { + pMnode->msgFp[type] = fp; } } @@ -435,11 +436,9 @@ void mndProcessWriteMsg(SMnodeMsg *pMsg) { mndProcessRpcMsg(pMsg); } void mndProcessSyncMsg(SMnodeMsg *pMsg) { mndProcessRpcMsg(pMsg); } -void mndProcessApplyMsg(SMnodeMsg *pMsg) {} - uint64_t mndGenerateUid(char *name, int32_t len) { int64_t us = taosGetTimestampUs(); int32_t hashval = MurmurHash3_32(name, len); uint64_t x = (us & 0x000000FFFFFFFFFF) << 24; return x + ((hashval & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); -} \ No newline at end of file +} diff --git a/source/dnode/mnode/sdb/inc/sdbInt.h b/source/dnode/mnode/sdb/inc/sdbInt.h index 7e2e2602a8..00f6d231d6 100644 --- a/source/dnode/mnode/sdb/inc/sdbInt.h +++ b/source/dnode/mnode/sdb/inc/sdbInt.h @@ -18,7 +18,7 @@ #include "os.h" #include "sdb.h" -#include "taosmsg.h" +#include "tmsg.h" #include "thash.h" #include "tlockfree.h" #include "tlog.h" diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 1c8b82fd92..0c311b86b6 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -199,6 +199,7 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, void *pKey) { SSdbRow *pRow = *ppRow; switch (pRow->status) { case SDB_STATUS_READY: + case SDB_STATUS_UPDATING: atomic_add_fetch_32(&pRow->refCount, 1); pRet = pRow->pObj; break; diff --git a/source/dnode/vnode/impl/CMakeLists.txt b/source/dnode/vnode/impl/CMakeLists.txt index 9e892bc4c4..944a4276db 100644 --- a/source/dnode/vnode/impl/CMakeLists.txt +++ b/source/dnode/vnode/impl/CMakeLists.txt @@ -20,5 +20,5 @@ target_link_libraries( # test if(${BUILD_TEST}) - add_subdirectory(test) +# add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h index bdafbf31a7..eb821a4b6c 100644 --- a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h +++ b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h @@ -22,17 +22,14 @@ extern "C" { #endif -typedef struct SVArenaNode SVArenaNode; -typedef struct SVMemAllocator SVMemAllocator; - -struct SVArenaNode { +typedef struct SVArenaNode { TD_SLIST_NODE(SVArenaNode); uint64_t size; // current node size void * ptr; char data[]; -}; +} SVArenaNode; -struct SVMemAllocator { +typedef struct SVMemAllocator { T_REF_DECLARE() TD_DLIST_NODE(SVMemAllocator); uint64_t capacity; @@ -40,7 +37,7 @@ struct SVMemAllocator { uint64_t lsize; SVArenaNode *pNode; TD_SLIST(SVArenaNode) nlist; -}; +} SVMemAllocator; SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize); void vmaDestroy(SVMemAllocator *pVMA); diff --git a/source/dnode/vnode/impl/src/vnodeRequest.c b/source/dnode/vnode/impl/src/vnodeRequest.c index 249bde4e56..efbba6352f 100644 --- a/source/dnode/vnode/impl/src/vnodeRequest.c +++ b/source/dnode/vnode/impl/src/vnodeRequest.c @@ -18,15 +18,15 @@ static int vnodeBuildCreateTableReq(void **buf, const SVCreateTableReq *pReq); static void *vnodeParseCreateTableReq(void *buf, SVCreateTableReq *pReq); -int vnodeBuildReq(void **buf, const SVnodeReq *pReq, uint8_t type) { +int vnodeBuildReq(void **buf, const SVnodeReq *pReq, tmsg_t type) { int tsize = 0; tsize += taosEncodeFixedU64(buf, pReq->ver); switch (type) { - case TSDB_MSG_TYPE_CREATE_TABLE: + case TDMT_MND_CREATE_TABLE: tsize += vnodeBuildCreateTableReq(buf, &(pReq->ctReq)); break; - case TSDB_MSG_TYPE_SUBMIT: + case TDMT_VND_SUBMIT: /* code */ break; default: @@ -36,11 +36,11 @@ int vnodeBuildReq(void **buf, const SVnodeReq *pReq, uint8_t type) { return tsize; } -void *vnodeParseReq(void *buf, SVnodeReq *pReq, uint8_t type) { +void *vnodeParseReq(void *buf, SVnodeReq *pReq, tmsg_t type) { buf = taosDecodeFixedU64(buf, &(pReq->ver)); switch (type) { - case TSDB_MSG_TYPE_CREATE_TABLE: + case TDMT_MND_CREATE_TABLE: buf = vnodeParseCreateTableReq(buf, &(pReq->ctReq)); break; diff --git a/source/dnode/vnode/impl/src/vnodeWrite.c b/source/dnode/vnode/impl/src/vnodeWrite.c index 5ec03f1fd3..18a808f955 100644 --- a/source/dnode/vnode/impl/src/vnodeWrite.c +++ b/source/dnode/vnode/impl/src/vnodeWrite.c @@ -17,7 +17,7 @@ int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) { switch (pMsg->msgType) { - case TSDB_MSG_TYPE_MQ_SET_CUR: + case TDMT_VND_MQ_SET_CUR: if (tqSetCursor(pVnode->pTq, pMsg->pCont) < 0) { // TODO: handle error } @@ -34,7 +34,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); // ser request version - void * pBuf = pMsg->pCont; + void * pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int64_t ver = pVnode->state.processed++; taosEncodeFixedU64(&pBuf, ver); @@ -51,8 +51,9 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { } int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - SVnodeReq vReq; - void * ptr = vnodeMalloc(pVnode, pMsg->contLen); + SVnodeReq vReq; + SVCreateTbReq vCreateTbReq; + void * ptr = vnodeMalloc(pVnode, pMsg->contLen); if (ptr == NULL) { // TODO: handle error } @@ -62,34 +63,33 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // todo: change the interface here uint64_t ver; - taosDecodeFixedU64(pMsg->pCont, &ver); + taosDecodeFixedU64(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &ver); if (tqPushMsg(pVnode->pTq, ptr, ver) < 0) { // TODO: handle error } - vnodeParseReq(pMsg->pCont, &vReq, pMsg->msgType); - switch (pMsg->msgType) { - case TSDB_MSG_TYPE_CREATE_STB_IN: - case TSDB_MSG_TYPE_CREATE_TABLE: - if (metaCreateTable(pVnode->pMeta, &(vReq.ctReq)) < 0) { + case TDMT_VND_CREATE_STB: + tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbReq); + if (metaCreateTable(pVnode->pMeta, &(vCreateTbReq)) < 0) { // TODO: handle error } // TODO: maybe need to clear the requst struct break; - case TSDB_MSG_TYPE_DROP_STB_IN: - case TSDB_MSG_TYPE_DROP_TABLE: + case TDMT_VND_DROP_STB: + case TDMT_MND_DROP_TABLE: if (metaDropTable(pVnode->pMeta, vReq.dtReq.uid) < 0) { // TODO: handle error } break; - case TSDB_MSG_TYPE_SUBMIT: + case TDMT_VND_SUBMIT: if (tsdbInsertData(pVnode->pTsdb, (SSubmitMsg *)ptr) < 0) { // TODO: handle error } break; default: + ASSERT(0); break; } diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index aa9fc741d9..13d703f0be 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -84,14 +84,14 @@ static void vtBuildCreateStbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { SVnodeReq vCreateSTbReq; vnodeSetCreateStbReq(&vCreateSTbReq, tbname, UINT32_MAX, UINT32_MAX, suid, pSchema, pTagSchema); - zs = vnodeBuildReq(NULL, &vCreateSTbReq, TSDB_MSG_TYPE_CREATE_TABLE); + zs = vnodeBuildReq(NULL, &vCreateSTbReq, TDMT_MND_CREATE_TABLE); pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + zs); - pMsg->msgType = TSDB_MSG_TYPE_CREATE_TABLE; + pMsg->msgType = TDMT_MND_CREATE_TABLE; pMsg->contLen = zs; pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(SRpcMsg)); pBuf = pMsg->pCont; - vnodeBuildReq(&pBuf, &vCreateSTbReq, TSDB_MSG_TYPE_CREATE_TABLE); + vnodeBuildReq(&pBuf, &vCreateSTbReq, TDMT_MND_CREATE_TABLE); META_CLEAR_TB_CFG(&vCreateSTbReq); tdFreeSchema(pSchema); @@ -108,14 +108,14 @@ static void vtBuildCreateCtbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { SVnodeReq vCreateCTbReq; vnodeSetCreateCtbReq(&vCreateCTbReq, tbname, UINT32_MAX, UINT32_MAX, suid, pTag); - tz = vnodeBuildReq(NULL, &vCreateCTbReq, TSDB_MSG_TYPE_CREATE_TABLE); + tz = vnodeBuildReq(NULL, &vCreateCTbReq, TDMT_MND_CREATE_TABLE); pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + tz); - pMsg->msgType = TSDB_MSG_TYPE_CREATE_TABLE; + pMsg->msgType = TDMT_MND_CREATE_TABLE; pMsg->contLen = tz; pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg)); void *pBuf = pMsg->pCont; - vnodeBuildReq(&pBuf, &vCreateCTbReq, TSDB_MSG_TYPE_CREATE_TABLE); + vnodeBuildReq(&pBuf, &vCreateCTbReq, TDMT_MND_CREATE_TABLE); META_CLEAR_TB_CFG(&vCreateCTbReq); free(pTag); @@ -133,7 +133,7 @@ static void vtBuildSubmitReq(SRpcMsg **ppMsg) { int tz = 1024; // TODO pMsg = (SRpcMsg *)malloc(sizeof(*pMsg) + tz); - pMsg->msgType = TSDB_MSG_TYPE_SUBMIT; + pMsg->msgType = TDMT_VND_SUBMIT; pMsg->contLen = tz; pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg)); diff --git a/source/dnode/vnode/meta/inc/metaTbCfg.h b/source/dnode/vnode/meta/inc/metaTbCfg.h index 68c609d6b4..b4ee095967 100644 --- a/source/dnode/vnode/meta/inc/metaTbCfg.h +++ b/source/dnode/vnode/meta/inc/metaTbCfg.h @@ -22,11 +22,11 @@ extern "C" { #endif -#define META_SUPER_TABLE 0 -#define META_CHILD_TABLE 1 -#define META_NORMAL_TABLE 2 +#define META_SUPER_TABLE TD_SUPER_TABLE +#define META_CHILD_TABLE TD_CHILD_TABLE +#define META_NORMAL_TABLE TD_NORMAL_TABLE -int metaValidateTbOptions(SMeta *pMeta, const STbCfg *); +int metaValidateTbCfg(SMeta *pMeta, const STbCfg *); size_t metaEncodeTbObjFromTbOptions(const STbCfg *, void *pBuf, size_t bsize); #ifdef __cplusplus diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index 3c1ccc72dc..735d33ac15 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -125,11 +125,11 @@ void metaCloseDB(SMeta *pMeta) { } int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { - tb_uid_t uid; - char buf[512]; - void * pBuf; - DBT key, value; - STSchema *pSchema = NULL; + tb_uid_t uid; + char buf[512]; + void * pBuf; + DBT key, value; + SSchema *pSchema = NULL; if (pTbCfg->type == META_SUPER_TABLE) { uid = pTbCfg->stbCfg.suid; @@ -156,9 +156,12 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { } // save schema + uint32_t ncols; if (pTbCfg->type == META_SUPER_TABLE) { + ncols = pTbCfg->stbCfg.nCols; pSchema = pTbCfg->stbCfg.pSchema; } else if (pTbCfg->type == META_NORMAL_TABLE) { + ncols = pTbCfg->ntbCfg.nCols; pSchema = pTbCfg->ntbCfg.pSchema; } @@ -166,12 +169,18 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { pBuf = buf; memset(&key, 0, sizeof(key)); memset(&value, 0, sizeof(key)); - SSchemaKey schemaKey = {uid, schemaVersion(pSchema)}; + SSchemaKey schemaKey = {uid, 0 /*TODO*/}; key.data = &schemaKey; key.size = sizeof(schemaKey); - tdEncodeSchema(&pBuf, pSchema); + taosEncodeFixedU32(&pBuf, ncols); + for (size_t i = 0; i < ncols; i++) { + taosEncodeFixedI8(&pBuf, pSchema[i].type); + taosEncodeFixedI32(&pBuf, pSchema[i].colId); + taosEncodeFixedI32(&pBuf, pSchema[i].bytes); + taosEncodeString(&pBuf, pSchema[i].name); + } value.data = buf; value.size = POINTER_DISTANCE(pBuf, buf); @@ -367,7 +376,15 @@ static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) { tsize += taosEncodeFixedU8(buf, pTbCfg->type); if (pTbCfg->type == META_SUPER_TABLE) { - tsize += tdEncodeSchema(buf, pTbCfg->stbCfg.pTagSchema); + tsize += taosEncodeVariantU32(buf, pTbCfg->stbCfg.nTagCols); + for (uint32_t i = 0; i < pTbCfg->stbCfg.nTagCols; i++) { + tsize += taosEncodeFixedI8(buf, pTbCfg->stbCfg.pSchema[i].type); + tsize += taosEncodeFixedI32(buf, pTbCfg->stbCfg.pSchema[i].colId); + tsize += taosEncodeFixedI32(buf, pTbCfg->stbCfg.pSchema[i].bytes); + tsize += taosEncodeString(buf, pTbCfg->stbCfg.pSchema[i].name); + } + + // tsize += tdEncodeSchema(buf, pTbCfg->stbCfg.pTagSchema); } else if (pTbCfg->type == META_CHILD_TABLE) { tsize += taosEncodeFixedU64(buf, pTbCfg->ctbCfg.suid); tsize += tdEncodeKVRow(buf, pTbCfg->ctbCfg.pTag); @@ -386,7 +403,14 @@ static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) { buf = taosDecodeFixedU8(buf, &(pTbCfg->type)); if (pTbCfg->type == META_SUPER_TABLE) { - buf = tdDecodeSchema(buf, &(pTbCfg->stbCfg.pTagSchema)); + buf = taosDecodeVariantU32(buf, pTbCfg->stbCfg.nTagCols); + pTbCfg->stbCfg.pTagSchema = (SSchema *)malloc(sizeof(SSchema) * pTbCfg->stbCfg.nTagCols); + for (uint32_t i = 0; i < pTbCfg->stbCfg.nTagCols; i++) { + buf = taosDecodeFixedI8(buf, &pTbCfg->stbCfg.pSchema[i].type); + buf = taosDecodeFixedI32(buf, &pTbCfg->stbCfg.pSchema[i].colId); + buf = taosDecodeFixedI32(buf, &pTbCfg->stbCfg.pSchema[i].bytes); + buf = taosDecodeStringTo(buf, pTbCfg->stbCfg.pSchema[i].name); + } } else if (pTbCfg->type == META_CHILD_TABLE) { buf = taosDecodeFixedU64(buf, &(pTbCfg->ctbCfg.suid)); buf = tdDecodeKVRow(buf, &(pTbCfg->ctbCfg.pTag)); diff --git a/source/dnode/vnode/meta/src/metaTable.c b/source/dnode/vnode/meta/src/metaTable.c index fc0f19302f..1398639f39 100644 --- a/source/dnode/vnode/meta/src/metaTable.c +++ b/source/dnode/vnode/meta/src/metaTable.c @@ -17,7 +17,7 @@ int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg) { // Validate the tbOptions - if (metaValidateTbOptions(pMeta, pTbCfg) < 0) { + if (metaValidateTbCfg(pMeta, pTbCfg) < 0) { // TODO: handle error return -1; } diff --git a/source/dnode/vnode/meta/src/metaTbCfg.c b/source/dnode/vnode/meta/src/metaTbCfg.c index da181a68fc..6f8a537966 100644 --- a/source/dnode/vnode/meta/src/metaTbCfg.c +++ b/source/dnode/vnode/meta/src/metaTbCfg.c @@ -16,7 +16,7 @@ #include "metaDef.h" #include "tcoding.h" -int metaValidateTbOptions(SMeta *pMeta, const STbCfg *pTbOptions) { +int metaValidateTbCfg(SMeta *pMeta, const STbCfg *pTbOptions) { // TODO return 0; } diff --git a/source/dnode/vnode/tsdb/inc/tsdbDef.h b/source/dnode/vnode/tsdb/inc/tsdbDef.h index b1375c9477..ded28727e4 100644 --- a/source/dnode/vnode/tsdb/inc/tsdbDef.h +++ b/source/dnode/vnode/tsdb/inc/tsdbDef.h @@ -17,7 +17,7 @@ #define _TD_TSDB_DEF_H_ #include "mallocator.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tlist.h" #include "thash.h" #include "tskiplist.h" diff --git a/source/dnode/vnode/tsdb/src/tsdbHealth.c b/source/dnode/vnode/tsdb/src/tsdbHealth.c index 8198c48033..4205f3e90f 100644 --- a/source/dnode/vnode/tsdb/src/tsdbHealth.c +++ b/source/dnode/vnode/tsdb/src/tsdbHealth.c @@ -14,7 +14,7 @@ */ #include "os.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tarray.h" #include "query.h" #include "tglobal.h" diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index bc7a8b8aff..b882e1a42c 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -49,7 +49,7 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp SEpSet *pVnodeEpSet = NULL; int32_t msgLen = 0; - CTG_ERR_RET(queryBuildMsg[TSDB_MSG_TYPE_USE_DB](input, &msg, 0, &msgLen)); + CTG_ERR_RET(queryBuildMsg[TDMT_MND_USE_DB](input, &msg, 0, &msgLen)); char *pMsg = rpcMallocCont(msgLen); if (NULL == pMsg) { @@ -63,7 +63,7 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp tfree(msg); SRpcMsg rpcMsg = { - .msgType = TSDB_MSG_TYPE_USE_DB, + .msgType = TDMT_MND_USE_DB, .pCont = pMsg, .contLen = msgLen, }; @@ -76,7 +76,7 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp CTG_ERR_RET(rpcRsp.code); } - CTG_ERR_RET(queryProcessMsgRsp[TSDB_MSG_TYPE_USE_DB](out, rpcRsp.pCont, rpcRsp.contLen)); + CTG_ERR_RET(queryProcessMsgRsp[TDMT_MND_USE_DB](out, rpcRsp.pCont, rpcRsp.contLen)); return TSDB_CODE_SUCCESS; } @@ -161,10 +161,10 @@ int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SE SEpSet *pVnodeEpSet = NULL; int32_t msgLen = 0; - CTG_ERR_RET(queryBuildMsg[TSDB_MSG_TYPE_TABLE_META](&bInput, &msg, 0, &msgLen)); + CTG_ERR_RET(queryBuildMsg[TDMT_VND_TABLE_META](&bInput, &msg, 0, &msgLen)); SRpcMsg rpcMsg = { - .msgType = TSDB_MSG_TYPE_TABLE_META, + .msgType = TDMT_VND_TABLE_META, .pCont = msg, .contLen = msgLen, }; @@ -181,7 +181,7 @@ int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SE CTG_ERR_RET(rpcRsp.code); } - CTG_ERR_RET(queryProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META](output, rpcRsp.pCont, rpcRsp.contLen)); + CTG_ERR_RET(queryProcessMsgRsp[TDMT_VND_TABLE_META](output, rpcRsp.pCont, rpcRsp.contLen)); return TSDB_CODE_SUCCESS; } @@ -197,15 +197,21 @@ int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) { return TSDB_CODE_SUCCESS; } -int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, SDBVgroupInfo *dbInfo, SArray* vgroupList) { +int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, SDBVgroupInfo *dbInfo, SArray** vgroupList) { SHashObj *vgroupHash = NULL; SVgroupInfo *vgInfo = NULL; + *vgroupList = taosArrayInit(taosHashGetSize(dbInfo->vgInfo), sizeof(SVgroupInfo)); + if (NULL == *vgroupList) { + ctgError("taosArrayInit failed"); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + void *pIter = taosHashIterate(dbInfo->vgInfo, NULL); while (pIter) { vgInfo = pIter; - if (NULL == taosArrayPush(vgroupList, vgInfo)) { + if (NULL == taosArrayPush(*vgroupList, vgInfo)) { ctgError("taosArrayPush failed"); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -295,14 +301,6 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - if (NULL == pCatalog->tableCache.cache) { - pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (NULL == pCatalog->tableCache.cache) { - ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - } - if (NULL == pCatalog->tableCache.cache) { pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == pCatalog->tableCache.cache) { @@ -329,7 +327,8 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out } } - if (taosHashPut(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname), output->tbMeta, sizeof(*output->tbMeta)) != 0) { + int32_t tbSize = sizeof(*output->tbMeta) + sizeof(SSchema) * (output->tbMeta->tableInfo.numOfColumns + output->tbMeta->tableInfo.numOfTags); + if (taosHashPut(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname), output->tbMeta, tbSize) != 0) { ctgError("push table[%s] to table cache failed", output->tbFname); goto error_exit; } @@ -529,7 +528,7 @@ int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const return ctgGetTableMetaImpl(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, true, pTableMeta); } -int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList) { +int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray** pVgroupList) { if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == pVgroupList) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } @@ -549,17 +548,29 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S int32_t vgId = tbMeta->vgId; if (NULL == taosHashGetClone(dbVgroup.vgInfo, &vgId, sizeof(vgId), &vgroupInfo)) { ctgError("vgId[%d] not found in vgroup list", vgId); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - if (NULL == taosArrayPush(pVgroupList, &vgroupInfo)) { + *pVgroupList = taosArrayInit(1, sizeof(SVgroupInfo)); + if (NULL == *pVgroupList) { + ctgError("taosArrayInit failed"); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + if (NULL == taosArrayPush(*pVgroupList, &vgroupInfo)) { ctgError("push vgroupInfo to array failed"); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } } + tfree(tbMeta); + + return TSDB_CODE_SUCCESS; + _return: tfree(tbMeta); + + taosArrayDestroy(*pVgroupList); CTG_RET(code); } @@ -634,8 +645,8 @@ _return: CTG_RET(code); } -int32_t catalogGetQnodeList(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SEpSet* pQnodeEpSet) { - if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pQnodeEpSet) { +int32_t catalogGetQnodeList(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SArray* pQnodeList) { + if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pQnodeList) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 22b5b7adc4..62279b9e1f 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -32,27 +32,28 @@ #include "stub.h" #include "addr_any.h" -typedef struct SAppInstInfo { - int64_t numOfConns; - SCorEpSet mgmtEp; -} SAppInstInfo; -typedef struct STscObj { - char user[TSDB_USER_LEN]; - char pass[TSDB_PASSWORD_LEN]; - char acctId[TSDB_ACCT_ID_LEN]; - char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; - uint32_t connId; - uint64_t id; // ref ID returned by taosAddRef -// struct SSqlObj *sqlList; - void *pTransporter; - pthread_mutex_t mutex; // used to protect the operation on db - int32_t numOfReqs; // number of sqlObj from this tscObj - SAppInstInfo *pAppInfo; -} STscObj; namespace { +void ctgTestSetPrepareTableMeta(); +void ctgTestSetPrepareCTableMeta(); +void ctgTestSetPrepareSTableMeta(); + + +int32_t ctgTestVgNum = 10; +int32_t ctgTestColNum = 2; +int32_t ctgTestTagNum = 1; +int32_t ctgTestSVersion = 1; +int32_t ctgTestTVersion = 1; + +char *ctgTestClusterId = "cluster1"; +char *ctgTestDbname = "1.db1"; +char *ctgTestTablename = "table1"; +char *ctgTestCTablename = "ctable1"; +char *ctgTestSTablename = "stable1"; + + void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); strcpy(pReq->db, "1.db1"); @@ -79,7 +80,7 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; rpcMsg.contLen = sizeof(SCreateDbMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DB; + rpcMsg.msgType = TDMT_MND_CREATE_DB; SRpcMsg rpcRsp = {0}; @@ -88,22 +89,281 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { ASSERT_EQ(rpcRsp.code, 0); } -void __rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { SUseDbRsp *rspMsg = NULL; //todo + pRsp->code =0; + pRsp->contLen = sizeof(SUseDbRsp) + ctgTestVgNum * sizeof(SVgroupInfo); + pRsp->pCont = calloc(1, pRsp->contLen); + rspMsg = (SUseDbRsp *)pRsp->pCont; + strcpy(rspMsg->db, ctgTestDbname); + rspMsg->vgVersion = htonl(1); + rspMsg->vgNum = htonl(ctgTestVgNum); + rspMsg->hashMethod = 0; + + SVgroupInfo *vg = NULL; + uint32_t hashUnit = UINT32_MAX / ctgTestVgNum; + for (int32_t i = 0; i < ctgTestVgNum; ++i) { + vg = &rspMsg->vgroupInfo[i]; + + vg->vgId = htonl(i + 1); + vg->hashBegin = htonl(i * hashUnit); + vg->hashEnd = htonl(hashUnit * (i + 1) - 1); + vg->numOfEps = i % TSDB_MAX_REPLICA + 1; + vg->inUse = i % vg->numOfEps; + for (int32_t n = 0; n < vg->numOfEps; ++n) { + SEpAddrMsg *addr = &vg->epAddr[n]; + strcpy(addr->fqdn, "a0"); + addr->port = htons(n + 22); + } + } + + vg->hashEnd = htonl(UINT32_MAX); + return; } -void initTestEnv() { + + +void ctgTestPrepareTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + STableMetaMsg *rspMsg = NULL; //todo + + pRsp->code =0; + pRsp->contLen = sizeof(STableMetaMsg) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); + pRsp->pCont = calloc(1, pRsp->contLen); + rspMsg = (STableMetaMsg *)pRsp->pCont; + sprintf(rspMsg->tbFname, "%s.%s", ctgTestDbname, ctgTestTablename); + rspMsg->numOfTags = 0; + rspMsg->numOfColumns = htonl(ctgTestColNum); + rspMsg->precision = 1; + rspMsg->tableType = TSDB_NORMAL_TABLE; + rspMsg->update = 1; + rspMsg->sversion = htonl(ctgTestSVersion); + rspMsg->tversion = htonl(ctgTestTVersion); + rspMsg->suid = 0; + rspMsg->tuid = htobe64(0x0000000000000001); + rspMsg->vgId = htonl(8); + + SSchema *s = NULL; + s = &rspMsg->pSchema[0]; + s->type = TSDB_DATA_TYPE_TIMESTAMP; + s->colId = htonl(0); + s->bytes = htonl(8); + strcpy(s->name, "ts"); + + s = &rspMsg->pSchema[1]; + s->type = TSDB_DATA_TYPE_INT; + s->colId = htonl(1); + s->bytes = htonl(4); + strcpy(s->name, "col1"); + + return; +} + + +void ctgTestPrepareCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + STableMetaMsg *rspMsg = NULL; //todo + + pRsp->code =0; + pRsp->contLen = sizeof(STableMetaMsg) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); + pRsp->pCont = calloc(1, pRsp->contLen); + rspMsg = (STableMetaMsg *)pRsp->pCont; + sprintf(rspMsg->tbFname, "%s.%s", ctgTestDbname, ctgTestCTablename); + sprintf(rspMsg->stbFname, "%s.%s", ctgTestDbname, ctgTestSTablename); + rspMsg->numOfTags = htonl(ctgTestTagNum); + rspMsg->numOfColumns = htonl(ctgTestColNum); + rspMsg->precision = 1; + rspMsg->tableType = TSDB_CHILD_TABLE; + rspMsg->update = 1; + rspMsg->sversion = htonl(ctgTestSVersion); + rspMsg->tversion = htonl(ctgTestTVersion); + rspMsg->suid = htobe64(0x0000000000000002); + rspMsg->tuid = htobe64(0x0000000000000003); + rspMsg->vgId = htonl(9); + + SSchema *s = NULL; + s = &rspMsg->pSchema[0]; + s->type = TSDB_DATA_TYPE_TIMESTAMP; + s->colId = htonl(0); + s->bytes = htonl(8); + strcpy(s->name, "ts"); + + s = &rspMsg->pSchema[1]; + s->type = TSDB_DATA_TYPE_INT; + s->colId = htonl(1); + s->bytes = htonl(4); + strcpy(s->name, "col1s"); + + s = &rspMsg->pSchema[2]; + s->type = TSDB_DATA_TYPE_BINARY; + s->colId = htonl(2); + s->bytes = htonl(12); + strcpy(s->name, "tag1s"); + + + return; +} + + +void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + STableMetaMsg *rspMsg = NULL; //todo + + pRsp->code =0; + pRsp->contLen = sizeof(STableMetaMsg) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); + pRsp->pCont = calloc(1, pRsp->contLen); + rspMsg = (STableMetaMsg *)pRsp->pCont; + sprintf(rspMsg->tbFname, "%s.%s", ctgTestDbname, ctgTestSTablename); + sprintf(rspMsg->stbFname, "%s.%s", ctgTestDbname, ctgTestSTablename); + rspMsg->numOfTags = htonl(ctgTestTagNum); + rspMsg->numOfColumns = htonl(ctgTestColNum); + rspMsg->precision = 1; + rspMsg->tableType = TSDB_SUPER_TABLE; + rspMsg->update = 1; + rspMsg->sversion = htonl(ctgTestSVersion); + rspMsg->tversion = htonl(ctgTestTVersion); + rspMsg->suid = htobe64(0x0000000000000002); + rspMsg->tuid = htobe64(0x0000000000000003); + rspMsg->vgId = 0; + + SSchema *s = NULL; + s = &rspMsg->pSchema[0]; + s->type = TSDB_DATA_TYPE_TIMESTAMP; + s->colId = htonl(0); + s->bytes = htonl(8); + strcpy(s->name, "ts"); + + s = &rspMsg->pSchema[1]; + s->type = TSDB_DATA_TYPE_INT; + s->colId = htonl(1); + s->bytes = htonl(4); + strcpy(s->name, "col1s"); + + s = &rspMsg->pSchema[2]; + s->type = TSDB_DATA_TYPE_BINARY; + s->colId = htonl(2); + s->bytes = htonl(12); + strcpy(s->name, "tag1s"); + + + return; +} + +void ctgTestPrepareDbVgroupsAndNormalMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); + + ctgTestSetPrepareTableMeta(); + + return; +} + + +void ctgTestPrepareDbVgroupsAndChildMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); + + ctgTestSetPrepareCTableMeta(); + + return; +} + +void ctgTestPrepareDbVgroupsAndSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); + + ctgTestSetPrepareSTableMeta(); + + return; +} + + + +void ctgTestSetPrepareDbVgroups() { static Stub stub; - stub.set(rpcSendRecv, __rpcSendRecv); + stub.set(rpcSendRecv, ctgTestPrepareDbVgroups); { AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); for (const auto& f : result) { - stub.set(f.second, __rpcSendRecv); + stub.set(f.second, ctgTestPrepareDbVgroups); + } + } +} + +void ctgTestSetPrepareTableMeta() { + static Stub stub; + stub.set(rpcSendRecv, ctgTestPrepareTableMeta); + { + AddrAny any("libtransport.so"); + std::map result; + any.get_global_func_addr_dynsym("^rpcSendRecv$", result); + for (const auto& f : result) { + stub.set(f.second, ctgTestPrepareTableMeta); + } + } +} + +void ctgTestSetPrepareCTableMeta() { + static Stub stub; + stub.set(rpcSendRecv, ctgTestPrepareCTableMeta); + { + AddrAny any("libtransport.so"); + std::map result; + any.get_global_func_addr_dynsym("^rpcSendRecv$", result); + for (const auto& f : result) { + stub.set(f.second, ctgTestPrepareCTableMeta); + } + } +} + +void ctgTestSetPrepareSTableMeta() { + static Stub stub; + stub.set(rpcSendRecv, ctgTestPrepareSTableMeta); + { + AddrAny any("libtransport.so"); + std::map result; + any.get_global_func_addr_dynsym("^rpcSendRecv$", result); + for (const auto& f : result) { + stub.set(f.second, ctgTestPrepareSTableMeta); + } + } +} + +void ctgTestSetPrepareDbVgroupsAndNormalMeta() { + static Stub stub; + stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndNormalMeta); + { + AddrAny any("libtransport.so"); + std::map result; + any.get_global_func_addr_dynsym("^rpcSendRecv$", result); + for (const auto& f : result) { + stub.set(f.second, ctgTestPrepareDbVgroupsAndNormalMeta); + } + } +} + + +void ctgTestSetPrepareDbVgroupsAndChildMeta() { + static Stub stub; + stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndChildMeta); + { + AddrAny any("libtransport.so"); + std::map result; + any.get_global_func_addr_dynsym("^rpcSendRecv$", result); + for (const auto& f : result) { + stub.set(f.second, ctgTestPrepareDbVgroupsAndChildMeta); + } + } +} + +void ctgTestSetPrepareDbVgroupsAndSuperMeta() { + static Stub stub; + stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndSuperMeta); + { + AddrAny any("libtransport.so"); + std::map result; + any.get_global_func_addr_dynsym("^rpcSendRecv$", result); + for (const auto& f : result) { + stub.set(f.second, ctgTestPrepareDbVgroupsAndSuperMeta); } } } @@ -111,33 +371,267 @@ void initTestEnv() { } -TEST(testCase, normalCase) { - STscObj* pConn = (STscObj *)taos_connect("127.0.0.1", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - char *clusterId = "cluster1"; - char *dbname = "1.db1"; - char *tablename = "table1"; +TEST(tableMeta, normalTable) { struct SCatalog* pCtg = NULL; void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; + ctgTestSetPrepareDbVgroups(); + initQueryModuleMsgHandle(); - sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); - code = catalogGetHandle(clusterId, &pCtg); + code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - code = catalogGetTableHashVgroup(pCtg, pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet, dbname, tablename, &vgInfo); + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestTablename, &vgInfo); ASSERT_EQ(code, 0); + ASSERT_EQ(vgInfo.vgId, 8); + ASSERT_EQ(vgInfo.numOfEps, 3); - taos_close(pConn); + ctgTestSetPrepareTableMeta(); + + STableMeta *tableMeta = NULL; + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestTablename, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 8); + ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + + tableMeta = NULL; + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestTablename, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 8); + ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + + catalogDestroy(); } +TEST(tableMeta, childTableCase) { + struct SCatalog* pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + + ctgTestSetPrepareDbVgroupsAndChildMeta(); + + initQueryModuleMsgHandle(); + + //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + STableMeta *tableMeta = NULL; + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestCTablename, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 9); + ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + + tableMeta = NULL; + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestCTablename, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 9); + ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + + tableMeta = NULL; + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestSTablename, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 0); + ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + + catalogDestroy(); +} + +TEST(tableMeta, superTableCase) { + struct SCatalog* pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + + ctgTestSetPrepareDbVgroupsAndSuperMeta(); + + initQueryModuleMsgHandle(); + + //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + STableMeta *tableMeta = NULL; + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestSTablename, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 0); + ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + + ctgTestSetPrepareCTableMeta(); + + tableMeta = NULL; + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestCTablename, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 9); + ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + + tableMeta = NULL; + code = catalogRenewAndGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestCTablename, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 9); + ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + + + + catalogDestroy(); +} + +TEST(tableDistVgroup, normalTable) { + struct SCatalog* pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo *vgInfo = NULL; + SArray *vgList = NULL; + + ctgTestSetPrepareDbVgroupsAndNormalMeta(); + + initQueryModuleMsgHandle(); + + //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + + code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestTablename, &vgList); + ASSERT_EQ(code, 0); + ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); + vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); + ASSERT_EQ(vgInfo->vgId, 8); + ASSERT_EQ(vgInfo->numOfEps, 3); + + catalogDestroy(); +} + +TEST(tableDistVgroup, childTableCase) { + struct SCatalog* pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo *vgInfo = NULL; + SArray *vgList = NULL; + + ctgTestSetPrepareDbVgroupsAndChildMeta(); + + initQueryModuleMsgHandle(); + + //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestCTablename, &vgList); + ASSERT_EQ(code, 0); + ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); + vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); + ASSERT_EQ(vgInfo->vgId, 9); + ASSERT_EQ(vgInfo->numOfEps, 4); + + + catalogDestroy(); +} + +TEST(tableDistVgroup, superTableCase) { + struct SCatalog* pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo *vgInfo = NULL; + SArray *vgList = NULL; + + ctgTestSetPrepareDbVgroupsAndSuperMeta(); + + initQueryModuleMsgHandle(); + + //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestSTablename, &vgList); + ASSERT_EQ(code, 0); + ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 10); + vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); + ASSERT_EQ(vgInfo->vgId, 1); + ASSERT_EQ(vgInfo->numOfEps, 1); + vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 1); + ASSERT_EQ(vgInfo->vgId, 2); + ASSERT_EQ(vgInfo->numOfEps, 2); + vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 2); + ASSERT_EQ(vgInfo->vgId, 3); + ASSERT_EQ(vgInfo->numOfEps, 3); + + + catalogDestroy(); +} + + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 1f7795e06f..ac91f906c7 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -14,7 +14,7 @@ */ #include "os.h" -#include "taosmsg.h" +#include "tmsg.h" #include "thash.h" #include "executil.h" diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index f37c7727f7..2e1c1f3f6e 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -14,7 +14,7 @@ */ #include #include "os.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tglobal.h" #include "ttime.h" diff --git a/source/libs/function/inc/texpr.h b/source/libs/function/inc/texpr.h index 4ef0a7ab21..eb4ab1b6e3 100644 --- a/source/libs/function/inc/texpr.h +++ b/source/libs/function/inc/texpr.h @@ -22,7 +22,7 @@ extern "C" { #include "os.h" -#include "taosmsg.h" +#include "tmsg.h" #include "taosdef.h" #include "tskiplist.h" #include "function.h" diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 00f59e8b87..3af4a8fe57 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -16,7 +16,7 @@ #include "tscalarfunction.h" #include "os.h" #include "taosdef.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tglobal.h" #include "thash.h" #include "ttypes.h" diff --git a/source/libs/function/src/texpr.c b/source/libs/function/src/texpr.c index 70a7e9973f..aa3e6ba0f0 100644 --- a/source/libs/function/src/texpr.c +++ b/source/libs/function/src/texpr.c @@ -18,7 +18,7 @@ #include "exception.h" #include "taosdef.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tarray.h" #include "tbuffer.h" #include "tcompare.h" diff --git a/source/libs/function/src/tfill.c b/source/libs/function/src/tfill.c index 9575416a03..db1bb3e7b8 100644 --- a/source/libs/function/src/tfill.c +++ b/source/libs/function/src/tfill.c @@ -17,7 +17,7 @@ #include "os.h" #include "taosdef.h" -#include "taosmsg.h" +#include "tmsg.h" #include "ttypes.h" #include "tfill.h" diff --git a/source/libs/function/src/thistogram.c b/source/libs/function/src/thistogram.c index 23238ebab7..2229ac8561 100644 --- a/source/libs/function/src/thistogram.c +++ b/source/libs/function/src/thistogram.c @@ -16,7 +16,7 @@ #include "thistogram.h" #include "taosdef.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tlosertree.h" /** diff --git a/source/libs/index/CMakeLists.txt b/source/libs/index/CMakeLists.txt index 4805bd3b77..50e76abd3f 100644 --- a/source/libs/index/CMakeLists.txt +++ b/source/libs/index/CMakeLists.txt @@ -3,7 +3,9 @@ add_library(index ${INDEX_SRC}) target_include_directories( index PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/index" + PUBLIC "${CMAKE_SOURCE_DIR}/include/os" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + ) target_link_libraries( index diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 2584a847ff..a8f231da0a 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -49,7 +49,6 @@ struct SIndex { SHashObj* colObj; // < field name, field id> int64_t suid; // current super table id, -1 is normal table - int colId; // field id allocated to cache int32_t cVersion; // current version allocated to cache SIndexStat stat; @@ -88,41 +87,39 @@ typedef struct SIndexTermQuery { EIndexQueryType qType; } SIndexTermQuery; -#define indexFatal(...) \ - do { \ - if (sDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("index FATAL ", 255, __VA_ARGS__); \ - } \ +typedef struct Iterate { + void* iter; + int8_t type; + char* colVal; + SArray* val; +} Iterate; +extern void* indexQhandle; + +int indexFlushCacheTFile(SIndex* sIdx, void*); + +#define indexFatal(...) \ + do { \ + if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("index FATAL ", 255, __VA_ARGS__); } \ } while (0) -#define indexError(...) \ - do { \ - if (sDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("index ERROR ", 255, __VA_ARGS__); \ - } \ +#define indexError(...) \ + do { \ + if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("index ERROR ", 255, __VA_ARGS__); } \ } while (0) -#define indexWarn(...) \ - do { \ - if (sDebugFlag & DEBUG_WARN) { \ - taosPrintLog("index WARN ", 255, __VA_ARGS__); \ - } \ +#define indexWarn(...) \ + do { \ + if (sDebugFlag & DEBUG_WARN) { taosPrintLog("index WARN ", 255, __VA_ARGS__); } \ } while (0) -#define indexInfo(...) \ - do { \ - if (sDebugFlag & DEBUG_INFO) { \ - taosPrintLog("index ", 255, __VA_ARGS__); \ - } \ +#define indexInfo(...) \ + do { \ + if (sDebugFlag & DEBUG_INFO) { taosPrintLog("index ", 255, __VA_ARGS__); } \ } while (0) -#define indexDebug(...) \ - do { \ - if (sDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \ - } \ +#define indexDebug(...) \ + do { \ + if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("index ", sDebugFlag, __VA_ARGS__); } \ } while (0) -#define indexTrace(...) \ - do { \ - if (sDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \ - } \ +#define indexTrace(...) \ + do { \ + if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("index ", sDebugFlag, __VA_ARGS__); } \ } while (0) #ifdef __cplusplus diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h index fb4f478ae9..07b5b8d564 100644 --- a/source/libs/index/inc/index_cache.h +++ b/source/libs/index/inc/index_cache.h @@ -22,10 +22,8 @@ // ----------------- key structure in skiplist --------------------- /* A data row, the format is like below: - * content: |<--totalLen-->|<-- fieldid-->|<--field type-->|<-- value len--->| - * |<-- value -->|<--uid -->|<--version--->|<-- itermType -->| - * len : |<--int32_t -->|<-- int16_t-->|<-- int8_t --->|<--- int32_t --->| - * <--valuelen->|<--uint64_t->| * <-- int32_t-->|<-- int8_t --->| + * content: |<--totalLen-->|<-- value len--->|<-- value -->|<--uid -->|<--version--->|<-- itermType -->| + * len : |<--int32_t -->|<--- int32_t --->|<--valuelen->|<--uint64_t->|<-- int32_t-->|<-- int8_t --->| */ #ifdef __cplusplus @@ -34,19 +32,40 @@ extern "C" { typedef struct IndexCache { T_REF_DECLARE() - SSkipList* skiplist; + SSkipList *mem, *imm; + SIndex* index; + char* colName; + int32_t version; + int32_t nTerm; + int8_t type; + } IndexCache; +typedef struct CacheTerm { + // key + int32_t nColVal; + char* colVal; + int32_t version; + // value + uint64_t uid; + int8_t colType; + SIndexOperOnColumn operaType; +} CacheTerm; // -IndexCache* indexCacheCreate(); + +IndexCache* indexCacheCreate(SIndex* idx, const char* colName, int8_t type); void indexCacheDestroy(void* cache); -int indexCachePut(void* cache, SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid); +int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid); // int indexCacheGet(void *cache, uint64_t *rst); -int indexCacheSearch(void* cache, SIndexTermQuery* query, int16_t colId, int32_t version, SArray* result, STermValueType* s); +int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermValueType* s); +void indexCacheRef(IndexCache* cache); +void indexCacheUnRef(IndexCache* cache); + +void indexCacheDebug(IndexCache* cache); #ifdef __cplusplus } #endif diff --git a/source/libs/index/inc/index_fst_automation.h b/source/libs/index/inc/index_fst_automation.h index c7269eda0f..be056f38fa 100644 --- a/source/libs/index/inc/index_fst_automation.h +++ b/source/libs/index/inc/index_fst_automation.h @@ -23,7 +23,7 @@ extern "C" { typedef struct AutomationCtx AutomationCtx; -typedef enum AutomationType { AUTOMATION_PREFIX, AUTMMATION_MATCH } AutomationType; +typedef enum AutomationType { AUTOMATION_ALWAYS, AUTOMATION_PREFIX, AUTMMATION_MATCH } AutomationType; typedef struct StartWith { AutomationCtx* autoSelf; diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h index 416b10bd14..550492ba50 100644 --- a/source/libs/index/inc/index_tfile.h +++ b/source/libs/index/inc/index_tfile.h @@ -105,9 +105,13 @@ void tfileCacheDestroy(TFileCache* tcache); TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key); void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader); +TFileReader* tfileGetReaderByCol(IndexTFile* tf, char* colName); + TFileReader* tfileReaderCreate(WriterCtx* ctx); void tfileReaderDestroy(TFileReader* reader); int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* result); +void tfileReaderRef(TFileReader* reader); +void tfileReaderUnRef(TFileReader* reader); TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header); void tfileWriterDestroy(TFileWriter* tw); diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index f0546afaf5..3f871af01d 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -18,11 +18,26 @@ #include "index_cache.h" #include "index_tfile.h" #include "tdef.h" +#include "tsched.h" #ifdef USE_LUCENE #include "lucene++/Lucene_c.h" #endif +#define INDEX_NUM_OF_THREADS 4 +#define INDEX_QUEUE_SIZE 4 + +void* indexQhandle = NULL; + +int32_t indexInit() { + indexQhandle = taosInitScheduler(INDEX_QUEUE_SIZE, INDEX_NUM_OF_THREADS, "index"); + return indexQhandle == NULL ? -1 : 0; + // do nothing +} +void indexCleanUp() { + taosCleanUpScheduler(indexQhandle); +} + static int uidCompare(const void* a, const void* b) { uint64_t u1 = *(uint64_t*)a; uint64_t u2 = *(uint64_t*)b; @@ -38,16 +53,15 @@ typedef struct SIdxColInfo { } SIdxColInfo; static pthread_once_t isInit = PTHREAD_ONCE_INIT; -static void indexInit(); +// static void indexInit(); static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result); -static int indexFlushCacheToTindex(SIndex* sIdx); static void indexInterResultsDestroy(SArray* results); static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* finalResult); int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { - pthread_once(&isInit, indexInit); + // pthread_once(&isInit, indexInit); SIndex* sIdx = calloc(1, sizeof(SIndex)); if (sIdx == NULL) { return -1; } @@ -57,10 +71,9 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { #endif #ifdef USE_INVERTED_INDEX - sIdx->cache = (void*)indexCacheCreate(); - sIdx->tindex = NULL; + // sIdx->cache = (void*)indexCacheCreate(sIdx); + sIdx->tindex = indexTFileCreate(path); sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - sIdx->colId = 1; sIdx->cVersion = 1; pthread_mutex_init(&sIdx->mtx, NULL); @@ -80,6 +93,12 @@ void indexClose(SIndex* sIdx) { #ifdef USE_INVERTED_INDEX indexCacheDestroy(sIdx->cache); + void* iter = taosHashIterate(sIdx->colObj, NULL); + while (iter) { + IndexCache** pCache = iter; + if (*pCache) { indexCacheUnRef(*pCache); } + iter = taosHashIterate(sIdx->colObj, iter); + } taosHashCleanup(sIdx->colObj); pthread_mutex_destroy(&sIdx->mtx); #endif @@ -110,29 +129,24 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { pthread_mutex_lock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); - SIdxColInfo* fi = taosHashGet(index->colObj, p->colName, p->nColName); - if (fi == NULL) { - SIdxColInfo tfi = {.colId = index->colId}; - index->cVersion++; - index->colId++; - taosHashPut(index->colObj, p->colName, p->nColName, &tfi, sizeof(tfi)); - } else { - // TODO, del + IndexCache** cache = taosHashGet(index->colObj, p->colName, p->nColName); + if (*cache == NULL) { + IndexCache* pCache = indexCacheCreate(index, p->colName, p->colType); + taosHashPut(index->colObj, p->colName, p->nColName, &pCache, sizeof(void*)); } } pthread_mutex_unlock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); - SIdxColInfo* fi = taosHashGet(index->colObj, p->colName, p->nColName); - assert(fi != NULL); - int32_t colId = fi->colId; - int32_t version = index->cVersion; - int ret = indexCachePut(index->cache, p, colId, version, uid); + IndexCache** cache = taosHashGet(index->colObj, p->colName, p->nColName); + + assert(*cache != NULL); + int ret = indexCachePut(*cache, p, uid); if (ret != 0) { return ret; } } -#endif +#endif return 0; } int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result) { @@ -281,32 +295,26 @@ void indexMultiTermDestroy(SIndexMultiTerm* terms) { taosArrayDestroy(terms); } -void indexInit() { - // do nothing -} static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result) { - int32_t version = -1; - int16_t colId = -1; - SIdxColInfo* colInfo = NULL; - SIndexTerm* term = query->term; const char* colName = term->colName; int32_t nColName = term->nColName; + // Get col info + IndexCache* cache = NULL; pthread_mutex_lock(&sIdx->mtx); - colInfo = taosHashGet(sIdx->colObj, colName, nColName); - if (colInfo == NULL) { + IndexCache** pCache = taosHashGet(sIdx->colObj, colName, nColName); + if (*pCache == NULL) { pthread_mutex_unlock(&sIdx->mtx); return -1; } - colId = colInfo->colId; - version = colInfo->cVersion; + cache = *pCache; pthread_mutex_unlock(&sIdx->mtx); *result = taosArrayInit(4, sizeof(uint64_t)); // TODO: iterator mem and tidex STermValueType s; - if (0 == indexCacheSearch(sIdx->cache, query, colId, version, *result, &s)) { + if (0 == indexCacheSearch(cache, query, *result, &s)) { if (s == kTypeDeletion) { indexInfo("col: %s already drop by other opera", term->colName); // coloum already drop by other oper, no need to query tindex @@ -353,10 +361,14 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType } return 0; } -static int indexFlushCacheToTindex(SIndex* sIdx) { +int indexFlushCacheTFile(SIndex* sIdx, void* cache) { if (sIdx == NULL) { return -1; } - indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid); + IndexCache* pCache = (IndexCache*)cache; + TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->colName); + + tfileReaderUnRef(pReader); + indexCacheUnRef(pCache); return 0; } diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index bb6a5e048a..8181c17505 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -16,142 +16,189 @@ #include "index_cache.h" #include "index_util.h" #include "tcompare.h" +#include "tsched.h" #define MAX_INDEX_KEY_LEN 256 // test only, change later +#define CACH_LIMIT 1000000 // ref index_cache.h:22 -#define CACHE_KEY_LEN(p) \ - (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + sizeof(p->operType)) +//#define CACHE_KEY_LEN(p) \ +// (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + sizeof(p->operType)) -static char* getIndexKey(const void* pData) { - return NULL; +static void cacheTermDestroy(CacheTerm* ct) { + if (ct == NULL) { return; } + + free(ct->colVal); + free(ct); } +static char* getIndexKey(const void* pData) { + CacheTerm* p = (CacheTerm*)pData; + return (char*)p; +} + static int32_t compareKey(const void* l, const void* r) { - char* lp = (char*)l; - char* rp = (char*)r; + CacheTerm* lt = (CacheTerm*)l; + CacheTerm* rt = (CacheTerm*)r; - // skip total len, not compare - int32_t ll, rl; // len - memcpy(&ll, lp, sizeof(int32_t)); - memcpy(&rl, rp, sizeof(int32_t)); - lp += sizeof(int32_t); - rp += sizeof(int32_t); - - // compare field id - int16_t lf, rf; // field id - memcpy(&lf, lp, sizeof(lf)); - memcpy(&rf, rp, sizeof(rf)); - if (lf != rf) { return lf < rf ? -1 : 1; } - lp += sizeof(lf); - rp += sizeof(rf); - - // compare field type - int8_t lft, rft; - memcpy(&lft, lp, sizeof(lft)); - memcpy(&rft, rp, sizeof(rft)); - lp += sizeof(lft); - rp += sizeof(rft); - assert(rft == rft); - - // skip value len - int32_t lfl, rfl; - memcpy(&lfl, lp, sizeof(lfl)); - memcpy(&rfl, rp, sizeof(rfl)); - lp += sizeof(lfl); - rp += sizeof(rfl); - - // compare value - int32_t i, j; - for (i = 0, j = 0; i < lfl && j < rfl; i++, j++) { - if (lp[i] == rp[j]) { + // compare colVal + int i, j; + for (i = 0, j = 0; i < lt->nColVal && j < rt->nColVal; i++, j++) { + if (lt->colVal[i] == rt->colVal[j]) { continue; } else { - return lp[i] < rp[j] ? -1 : 1; + return lt->colVal[i] < rt->colVal[j] ? -1 : 1; } } - if (i < lfl) { + if (i < lt->nColVal) { return 1; - } else if (j < rfl) { + } else if (j < rt->nColVal) { return -1; } - lp += lfl; - rp += rfl; - - // skip uid - uint64_t lu, ru; - memcpy(&lu, lp, sizeof(lu)); - memcpy(&ru, rp, sizeof(ru)); - lp += sizeof(lu); - rp += sizeof(ru); - - // compare version, desc order - int32_t lv, rv; - memcpy(&lv, lp, sizeof(lv)); - memcpy(&rv, rp, sizeof(rv)); - if (lv != rv) { return lv > rv ? -1 : 1; } - - lp += sizeof(lv); - rp += sizeof(rv); - // not care item type - - return 0; + // compare version + return rt->version - lt->version; } -IndexCache* indexCacheCreate() { + +static SSkipList* indexInternalCacheCreate(int8_t type) { + if (type == TSDB_DATA_TYPE_BINARY) { + return tSkipListCreate(MAX_SKIP_LIST_LEVEL, type, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey); + } +} + +IndexCache* indexCacheCreate(SIndex* idx, const char* colName, int8_t type) { IndexCache* cache = calloc(1, sizeof(IndexCache)); if (cache == NULL) { indexError("failed to create index cache"); return NULL; - } - cache->skiplist = - tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey); + }; + cache->mem = indexInternalCacheCreate(type); + + cache->colName = calloc(1, strlen(colName) + 1); + memcpy(cache->colName, colName, strlen(colName)); + cache->type = type; + cache->index = idx; + cache->version = 0; + + indexCacheRef(cache); return cache; } +void indexCacheDebug(IndexCache* cache) { + SSkipListIterator* iter = tSkipListCreateIter(cache->mem); + while (tSkipListIterNext(iter)) { + SSkipListNode* node = tSkipListIterGet(iter); + CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); + if (ct != NULL) { + // TODO, add more debug info + indexInfo("{colVal: %s, version: %d} \t", ct->colVal, ct->version); + } + } + tSkipListDestroyIter(iter); +} void indexCacheDestroy(void* cache) { IndexCache* pCache = cache; if (pCache == NULL) { return; } - tSkipListDestroy(pCache->skiplist); + tSkipListDestroy(pCache->mem); + tSkipListDestroy(pCache->imm); + free(pCache->colName); free(pCache); } -int indexCachePut(void* cache, SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid) { +static void doMergeWork(SSchedMsg* msg) { + IndexCache* pCache = msg->ahandle; + SIndex* sidx = (SIndex*)pCache->index; + indexFlushCacheTFile(sidx, pCache); +} + +int indexCacheSchedToMerge(IndexCache* pCache) { + SSchedMsg schedMsg = {0}; + schedMsg.fp = doMergeWork; + schedMsg.ahandle = pCache; + schedMsg.thandle = NULL; + schedMsg.msg = NULL; + + taosScheduleTask(indexQhandle, &schedMsg); +} +int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { if (cache == NULL) { return -1; } IndexCache* pCache = cache; + indexCacheRef(pCache); // encode data - int32_t total = CACHE_KEY_LEN(term); + CacheTerm* ct = calloc(1, sizeof(CacheTerm)); + if (cache == NULL) { return -1; } + // set up key + ct->colType = term->colType; + ct->nColVal = term->nColVal; + ct->colVal = (char*)calloc(1, sizeof(char) * (ct->nColVal + 1)); + memcpy(ct->colVal, term->colVal, ct->nColVal); + ct->version = atomic_add_fetch_32(&pCache->version, 1); + // set value + ct->uid = uid; + ct->operaType = term->operType; - char* buf = calloc(1, total); - char* p = buf; + tSkipListPut(pCache->mem, (char*)ct); + pCache->nTerm += 1; - SERIALIZE_VAR_TO_BUF(p, total, int32_t); - SERIALIZE_VAR_TO_BUF(p, colId, int16_t); + if (pCache->nTerm >= CACH_LIMIT) { + pCache->nTerm = 0; - SERIALIZE_MEM_TO_BUF(p, term, colType); - SERIALIZE_MEM_TO_BUF(p, term, nColVal); - SERIALIZE_STR_MEM_TO_BUF(p, term, colVal, term->nColVal); + while (pCache->imm != NULL) { + // do nothong + } - SERIALIZE_VAR_TO_BUF(p, version, int32_t); - SERIALIZE_VAR_TO_BUF(p, uid, uint64_t); + pCache->imm = pCache->mem; + pCache->mem = indexInternalCacheCreate(pCache->type); - SERIALIZE_MEM_TO_BUF(p, term, operType); - - tSkipListPut(pCache->skiplist, (void*)buf); + // sched to merge + // unref cache int bgwork + indexCacheSchedToMerge(pCache); + } + indexCacheUnRef(pCache); return 0; // encode end } -int indexCacheDel(void* cache, int32_t fieldId, const char* fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { +int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { IndexCache* pCache = cache; return 0; } -int indexCacheSearch(void* cache, SIndexTermQuery* query, int16_t colId, int32_t version, SArray* result, STermValueType* s) { +int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermValueType* s) { if (cache == NULL) { return -1; } IndexCache* pCache = cache; SIndexTerm* term = query->term; EIndexQueryType qtype = query->qType; - int32_t keyLen = CACHE_KEY_LEN(term); - char* buf = calloc(1, keyLen); + CacheTerm* ct = calloc(1, sizeof(CacheTerm)); + if (ct == NULL) { return -1; } + ct->nColVal = term->nColVal; + ct->colVal = calloc(1, sizeof(char) * (ct->nColVal + 1)); + memcpy(ct->colVal, term->colVal, ct->nColVal); + ct->version = atomic_load_32(&pCache->version); + + char* key = getIndexKey(ct); + // TODO handle multi situation later, and refactor + SSkipListIterator* iter = tSkipListCreateIterFromVal(pCache->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); + while (tSkipListIterNext(iter)) { + SSkipListNode* node = tSkipListIterGet(iter); + if (node != NULL) { + CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); + if (c->operaType == ADD_VALUE || qtype == QUERY_TERM) { + if (c->nColVal == ct->nColVal && strncmp(c->colVal, ct->colVal, c->nColVal) == 0) { + taosArrayPush(result, &c->uid); + *s = kTypeValue; + } else { + break; + } + } else if (c->operaType == DEL_VALUE) { + // table is del, not need + *s = kTypeDeletion; + break; + } + } + } + tSkipListDestroyIter(iter); + cacheTermDestroy(ct); + // int32_t keyLen = CACHE_KEY_LEN(term); + // char* buf = calloc(1, keyLen); if (qtype == QUERY_TERM) { // } else if (qtype == QUERY_PREFIX) { @@ -161,6 +208,14 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, int16_t colId, int32_t } else if (qtype == QUERY_REGEX) { // } - return 0; } + +void indexCacheRef(IndexCache* cache) { + int ref = T_REF_INC(cache); + UNUSED(ref); +} +void indexCacheUnRef(IndexCache* cache) { + int ref = T_REF_DEC(cache); + if (ref == 0) { indexCacheDestroy(cache); } +} diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 18514cd0d5..18024fa391 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -1083,7 +1083,7 @@ bool fstBoundWithDataExceededBy(FstBoundWithData* bound, FstSlice* slice) { } else if (bound->type == Excluded) { return comp >= 0 ? true : false; } else { - return true; + return false; } } bool fstBoundWithDataIsEmpty(FstBoundWithData* bound) { @@ -1224,7 +1224,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb void* start = automFuncs[aut->type].start(aut); if (automFuncs[aut->type].isMatch(aut, start)) { FstSlice s = fstSliceCreate(NULL, 0); - return swsResultCreate(&s, output, callback(start)); + return swsResultCreate(&s, output, callback == NULL ? NULL : callback(start)); } } SArray* nodes = taosArrayInit(8, sizeof(FstNode*)); @@ -1237,10 +1237,12 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb } FstTransition trn; fstNodeGetTransitionAt(p->node, p->trans, &trn); - Output out = p->out.out + trn.out; - void* nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp); - void* tState = callback(nextState); - bool isMatch = automFuncs[aut->type].isMatch(aut, nextState); + + Output out = p->out.out + trn.out; + void* nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp); + void* tState = (callback == NULL) ? NULL : callback(nextState); + bool isMatch = automFuncs[aut->type].isMatch(aut, nextState); + FstNode* nextNode = fstGetNode(sws->fst, trn.addr); taosArrayPush(nodes, &nextNode); taosArrayPush(sws->inp, &(trn.inp)); diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index eb4c479c94..72983809d1 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -64,6 +64,25 @@ StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) { return nsv; } +// iterate fst +static void* alwaysMatchStart(AutomationCtx* ctx) { + return NULL; +} +static bool alwaysMatchIsMatch(AutomationCtx* ctx, void* state) { + return true; +} +static bool alwaysMatchCanMatch(AutomationCtx* ctx, void* state) { + return true; +} +static bool alwaysMatchWillAlwaysMatch(AutomationCtx* ctx, void* state) { + return true; +} +static void* alwaysMatchAccpet(AutomationCtx* ctx, void* state, uint8_t byte) { + return NULL; +} +static void* alwaysMatchAccpetEof(AutomationCtx* ctx, void* state) { + return NULL; +} // prefix query, impl later static void* prefixStart(AutomationCtx* ctx) { @@ -127,6 +146,7 @@ static void* patternAcceptEof(AutomationCtx* ctx, void* state) { } AutomationFunc automFuncs[] = { + {alwaysMatchStart, alwaysMatchIsMatch, alwaysMatchCanMatch, alwaysMatchWillAlwaysMatch, alwaysMatchAccpet, alwaysMatchAccpetEof}, {prefixStart, prefixIsMatch, prefixCanMatch, prefixWillAlwaysMatch, prefixAccept, prefixAcceptEof}, {patternStart, patternIsMatch, patternCanMatch, patternWillAlwaysMatch, patternAccept, patternAcceptEof} // add more search type @@ -137,7 +157,11 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) { if (ctx == NULL) { return NULL; } StartWithStateValue* sv = NULL; - if (atype == AUTOMATION_PREFIX) { + if (atype == AUTOMATION_ALWAYS) { + int val = 0; + sv = startWithStateValueCreate(Running, FST_INT, &val); + ctx->stdata = (void*)sv; + } else if (atype == AUTOMATION_PREFIX) { int val = 0; sv = startWithStateValueCreate(Running, FST_INT, &val); ctx->stdata = (void*)sv; diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 9afc29457d..0dfb14cc8d 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -33,11 +33,9 @@ static int tfileWriteHeader(TFileWriter* writer); static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset); static int tfileWriteData(TFileWriter* write, TFileValue* tval); -static int tfileReaderLoadHeader(TFileReader* reader); -static int tfileReaderLoadFst(TFileReader* reader); -static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result); -static void tfileReaderRef(TFileReader* reader); -static void tfileReaderUnRef(TFileReader* reader); +static int tfileReaderLoadHeader(TFileReader* reader); +static int tfileReaderLoadFst(TFileReader* reader); +static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result); static int tfileGetFileList(const char* path, SArray* result); static int tfileRmExpireFile(SArray* result); @@ -131,7 +129,6 @@ void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader) taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*)); return; } - TFileReader* tfileReaderCreate(WriterCtx* ctx) { TFileReader* reader = calloc(1, sizeof(TFileReader)); if (reader == NULL) { return NULL; } @@ -317,6 +314,11 @@ int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) { return 0; } +TFileReader* tfileGetReaderByCol(IndexTFile* tf, char* colName) { + if (tf == NULL) { return NULL; } + TFileCacheKey key = {.suid = 0, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)}; + return tfileCacheGet(tf->cache, &key); +} static int tfileStrCompare(const void* a, const void* b) { int ret = strcmp((char*)a, (char*)b); @@ -423,12 +425,12 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* free(buf); return 0; } -static void tfileReaderRef(TFileReader* reader) { +void tfileReaderRef(TFileReader* reader) { int ref = T_REF_INC(reader); UNUSED(ref); } -static void tfileReaderUnRef(TFileReader* reader) { +void tfileReaderUnRef(TFileReader* reader) { int ref = T_REF_DEC(reader); if (ref == 0) { tfileReaderDestroy(reader); } } @@ -462,7 +464,9 @@ static int tfileCompare(const void* a, const void* b) { size_t aLen = strlen(aName); size_t bLen = strlen(bName); - return strncmp(aName, bName, aLen > bLen ? aLen : bLen); + int ret = strncmp(aName, bName, aLen > bLen ? aLen : bLen); + if (ret == 0) { return ret; } + return ret < 0 ? -1 : 1; } // tfile name suid-colId-version.tindex static void tfileGenFileName(char* filename, uint64_t suid, int colId, int version) { @@ -477,9 +481,9 @@ static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, return -1; } static void tfileSerialCacheKey(TFileCacheKey* key, char* buf) { - SERIALIZE_MEM_TO_BUF(buf, key, suid); - SERIALIZE_VAR_TO_BUF(buf, '_', char); - SERIALIZE_MEM_TO_BUF(buf, key, colType); - SERIALIZE_VAR_TO_BUF(buf, '_', char); + // SERIALIZE_MEM_TO_BUF(buf, key, suid); + // SERIALIZE_VAR_TO_BUF(buf, '_', char); + // SERIALIZE_MEM_TO_BUF(buf, key, colType); + // SERIALIZE_VAR_TO_BUF(buf, '_', char); SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); } diff --git a/source/libs/index/test/CMakeLists.txt b/source/libs/index/test/CMakeLists.txt index 6eb532b41e..3957554748 100644 --- a/source/libs/index/test/CMakeLists.txt +++ b/source/libs/index/test/CMakeLists.txt @@ -1,13 +1,23 @@ add_executable(indexTest "") +add_executable(fstTest "") target_sources(indexTest PRIVATE "indexTests.cc" ) +target_sources(fstTest + PRIVATE + "fstTest.cc" +) target_include_directories ( indexTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/index" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories ( fstTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/index" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries (indexTest os util @@ -15,8 +25,16 @@ target_link_libraries (indexTest gtest_main index ) - -add_test( - NAME index_test - COMMAND indexTest +target_link_libraries (fstTest + os + util + common + gtest_main + index ) + + +#add_test( +# NAME index_test +# COMMAND indexTest +#) diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc new file mode 100644 index 0000000000..85bb5e2b15 --- /dev/null +++ b/source/libs/index/test/fstTest.cc @@ -0,0 +1,174 @@ + +#include +#include +#include +#include "index.h" +#include "indexInt.h" +#include "index_cache.h" +#include "index_fst.h" +#include "index_fst_counting_writer.h" +#include "index_fst_util.h" +#include "index_tfile.h" +#include "tskiplist.h" +#include "tutil.h" + +void* callback(void* s) { + return s; +} + +static std::string fileName = "/tmp/tindex.tindex"; +class FstWriter { + public: + FstWriter() { + remove(fileName.c_str()); + _wc = writerCtxCreate(TFile, fileName.c_str(), false, 64 * 1024 * 1024); + _b = fstBuilderCreate(_wc, 0); + } + bool Put(const std::string& key, uint64_t val) { + FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size()); + bool ok = fstBuilderInsert(_b, skey, val); + fstSliceDestroy(&skey); + return ok; + } + ~FstWriter() { + fstBuilderFinish(_b); + fstBuilderDestroy(_b); + + writerCtxDestroy(_wc); + } + + private: + FstBuilder* _b; + WriterCtx* _wc; +}; + +class FstReadMemory { + public: + FstReadMemory(size_t size) { + _wc = writerCtxCreate(TFile, fileName.c_str(), true, 64 * 1024); + _w = fstCountingWriterCreate(_wc); + _size = size; + memset((void*)&_s, 0, sizeof(_s)); + } + bool init() { + char* buf = (char*)calloc(1, sizeof(char) * _size); + int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); + if (nRead <= 0) { return false; } + _size = nRead; + _s = fstSliceCreate((uint8_t*)buf, _size); + _fst = fstCreate(&_s); + free(buf); + return _fst != NULL; + } + bool Get(const std::string& key, uint64_t* val) { + FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size()); + bool ok = fstGet(_fst, &skey, val); + fstSliceDestroy(&skey); + return ok; + } + bool GetWithTimeCostUs(const std::string& key, uint64_t* val, uint64_t* elapse) { + int64_t s = taosGetTimestampUs(); + bool ok = this->Get(key, val); + int64_t e = taosGetTimestampUs(); + *elapse = e - s; + return ok; + } + // add later + bool Search(AutomationCtx* ctx, std::vector& result) { + FstStreamBuilder* sb = fstSearch(_fst, ctx); + StreamWithState* st = streamBuilderIntoStream(sb); + StreamWithStateResult* rt = NULL; + while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { + // result.push_back((uint64_t)(rt->out.out)); + FstSlice* s = &rt->data; + int32_t sz = 0; + char* ch = (char*)fstSliceData(s, &sz); + std::string key(ch, sz); + printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out)); + swsResultDestroy(rt); + } + for (size_t i = 0; i < result.size(); i++) {} + std::cout << std::endl; + return true; + } + bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector& result) { + int64_t s = taosGetTimestampUs(); + bool ok = this->Search(ctx, result); + int64_t e = taosGetTimestampUs(); + return ok; + } + + ~FstReadMemory() { + fstCountingWriterDestroy(_w); + fstDestroy(_fst); + fstSliceDestroy(&_s); + writerCtxDestroy(_wc); + } + + private: + FstCountingWriter* _w; + Fst* _fst; + FstSlice _s; + WriterCtx* _wc; + size_t _size; +}; + +#define L 100 +#define M 100 +#define N 100 + +int Performance_fstWriteRecords(FstWriter* b) { + std::string str("aa"); + for (int i = 0; i < L; i++) { + str[0] = 'a' + i; + str.resize(2); + for (int j = 0; j < M; j++) { + str[1] = 'a' + j; + str.resize(2); + for (int k = 0; k < N; k++) { + str.push_back('a'); + b->Put(str, k); + printf("(%d, %d, %d, %s)\n", i, j, k, str.c_str()); + } + } + } + return L * M * N; +} +void checkFstCheckIterator() { + tfInit(); + FstWriter* fw = new FstWriter; + int64_t s = taosGetTimestampUs(); + int count = 2; + Performance_fstWriteRecords(fw); + int64_t e = taosGetTimestampUs(); + + std::cout << "insert data count : " << count << "elapas time: " << e - s << std::endl; + delete fw; + + FstReadMemory* m = new FstReadMemory(1024 * 64); + if (m->init() == false) { + std::cout << "init readMemory failed" << std::endl; + delete m; + return; + } + + // prefix search + std::vector result; + + AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS); + m->Search(ctx, result); + std::cout << "size: " << result.size() << std::endl; + // assert(result.size() == count); + for (int i = 0; i < result.size(); i++) { + // assert(result[i] == i); // check result + } + + free(ctx); + delete m; + tfCleanup(); +} +int main() { + checkFstCheckIterator(); + // checkFstPrefixSearch(); + return 1; +} diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 6c6f45bd11..17733dd284 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -16,12 +16,30 @@ #include #include "index.h" #include "indexInt.h" +#include "index_cache.h" #include "index_fst.h" #include "index_fst_counting_writer.h" #include "index_fst_util.h" #include "index_tfile.h" +#include "tskiplist.h" #include "tutil.h" using namespace std; +class DebugInfo { + public: + DebugInfo(const char* str) : info(str) { + std::cout << "------------" << info << "\t" + << "begin" + << "-------------" << std::endl; + } + ~DebugInfo() { + std::cout << "-----------" << info << "\t" + << "end" + << "--------------" << std::endl; + } + + private: + std::string info; +}; class FstWriter { public: FstWriter() { @@ -110,58 +128,6 @@ class FstReadMemory { size_t _size; }; -// TEST(IndexTest, index_create_test) { -// SIndexOpts *opts = indexOptsCreate(); -// SIndex *index = indexOpen(opts, "./test"); -// if (index == NULL) { -// std::cout << "index open failed" << std::endl; -// } -// -// -// // write -// for (int i = 0; i < 100000; i++) { -// SIndexMultiTerm* terms = indexMultiTermCreate(); -// std::string val = "field"; -// -// indexMultiTermAdd(terms, "tag1", strlen("tag1"), val.c_str(), val.size()); -// -// val.append(std::to_string(i)); -// indexMultiTermAdd(terms, "tag2", strlen("tag2"), val.c_str(), val.size()); -// -// val.insert(0, std::to_string(i)); -// indexMultiTermAdd(terms, "tag3", strlen("tag3"), val.c_str(), val.size()); -// -// val.append("const"); -// indexMultiTermAdd(terms, "tag4", strlen("tag4"), val.c_str(), val.size()); -// -// -// indexPut(index, terms, i); -// indexMultiTermDestroy(terms); -// } -// -// -// // query -// SIndexMultiTermQuery *multiQuery = indexMultiTermQueryCreate(MUST); -// -// indexMultiTermQueryAdd(multiQuery, "tag1", strlen("tag1"), "field", strlen("field"), QUERY_PREFIX); -// indexMultiTermQueryAdd(multiQuery, "tag3", strlen("tag3"), "0field0", strlen("0field0"), QUERY_TERM); -// -// SArray *result = (SArray *)taosArrayInit(10, sizeof(int)); -// indexSearch(index, multiQuery, result); -// -// std::cout << "taos'size : " << taosArrayGetSize(result) << std::endl; -// for (int i = 0; i < taosArrayGetSize(result); i++) { -// int *v = (int *)taosArrayGet(result, i); -// std::cout << "value --->" << *v << std::endl; -// } -// // add more test case -// indexMultiTermQueryDestroy(multiQuery); -// -// indexOptsDestroy(opts); -// indexClose(index); -// // -//} - #define L 100 #define M 100 #define N 100 @@ -183,7 +149,6 @@ int Performance_fstWriteRecords(FstWriter* b) { } return L * M * N; } - void Performance_fstReadRecords(FstReadMemory* m) { std::string str("aa"); for (int i = 0; i < M; i++) { @@ -218,7 +183,6 @@ void checkFstPerf() { Performance_fstReadRecords(m); delete m; } - void checkFstPrefixSearch() { FstWriter* fw = new FstWriter; int64_t s = taosGetTimestampUs(); @@ -296,7 +260,6 @@ void validateFst() { } delete m; } - class IndexEnv : public ::testing::Test { protected: virtual void SetUp() { @@ -315,44 +278,51 @@ class IndexEnv : public ::testing::Test { SIndex* index; }; -// TEST_F(IndexEnv, testPut) { -// // single index column -// { -// std::string colName("tag1"), colVal("Hello world"); -// SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), -// colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); -// -// for (size_t i = 0; i < 100; i++) { -// int tableId = i; -// int ret = indexPut(index, terms, tableId); -// assert(ret == 0); -// } -// indexMultiTermDestroy(terms); -// } -// // multi index column -// { +/// TEST_F(IndexEnv, testPut) { +// / // single index column +// / { +// / std::string colName("tag1"), colVal("Hello world"); +// / SIndexTerm* term = +// indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), / colVal.size()); // SIndexMultiTerm* terms = indexMultiTermCreate(); -// { -// std::string colName("tag1"), colVal("Hello world"); -// SIndexTerm* term = -// indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); -// indexMultiTermAdd(terms, term); +// indexMultiTermAdd(terms, term); +// / / for (size_t i = 0; i < 100; i++) { +// / int tableId = i; +// / int ret = indexPut(index, terms, tableId); +// / assert(ret == 0); +// / // } -// { -// std::string colName("tag2"), colVal("Hello world"); -// SIndexTerm* term = -// indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); -// indexMultiTermAdd(terms, term); -// } -// -// for (int i = 0; i < 100; i++) { -// int tableId = i; -// int ret = indexPut(index, terms, tableId); -// assert(ret == 0); -// } -// indexMultiTermDestroy(terms); +// / indexMultiTermDestroy(terms); +// / // } -// // +// / // multi index column +// / { +// / SIndexMultiTerm* terms = indexMultiTermCreate(); +// / { +// / std::string colName("tag1"), colVal("Hello world"); +// / SIndexTerm* term = +// / indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); +// / indexMultiTermAdd(terms, term); +// / +// } +// / { +// / std::string colName("tag2"), colVal("Hello world"); +// / SIndexTerm* term = +// / indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); +// / indexMultiTermAdd(terms, term); +// / +// } +// / / for (int i = 0; i < 100; i++) { +// / int tableId = i; +// / int ret = indexPut(index, terms, tableId); +// / assert(ret == 0); +// / +// } +// / indexMultiTermDestroy(terms); +// / +// } +// / // +// / //} class TFileObj { @@ -421,6 +391,7 @@ class TFileObj { int colId_; }; + class IndexTFileEnv : public ::testing::Test { protected: virtual void SetUp() { @@ -428,22 +399,6 @@ class IndexTFileEnv : public ::testing::Test { taosMkDir(dir.c_str()); tfInit(); fObj = new TFileObj(dir, colName); - - // std::string colName("voltage"); - // header.suid = 1; - // header.version = 1; - // memcpy(header.colName, colName.c_str(), colName.size()); - // header.colType = TSDB_DATA_TYPE_BINARY; - - // std::string path(dir); - // int colId = 2; - // char buf[64] = {0}; - // sprintf(buf, "%" PRIu64 "-%d-%d.tindex", header.suid, colId, header.version); - // path.append("/").append(buf); - - // ctx = writerCtxCreate(TFile, path.c_str(), false, 64 * 1024 * 1024); - - // twrite = tfileWriterCreate(ctx, &header); } virtual void TearDown() { @@ -460,16 +415,8 @@ class IndexTFileEnv : public ::testing::Test { int coldId = 2; int version = 1; int colType = TSDB_DATA_TYPE_BINARY; - - // WriterCtx* ctx = NULL; - // TFileHeader header; - // TFileWriter* twrite = NULL; }; -// static TFileWriter* genTFileWriter(const char* path, TFileHeader* header) { -// char buf[128] = {0}; -// WriterCtx* ctx = writerCtxCreate(TFile, path, false, ) -//} static TFileValue* genTFileValue(const char* val) { TFileValue* tv = (TFileValue*)calloc(1, sizeof(TFileValue)); int32_t vlen = strlen(val) + 1; @@ -489,10 +436,9 @@ static void destroyTFileValue(void* val) { taosArrayDestroy(tv->tableId); free(tv); } - TEST_F(IndexTFileEnv, test_tfile_write) { TFileValue* v1 = genTFileValue("c"); - TFileValue* v2 = genTFileValue("a"); + TFileValue* v2 = genTFileValue("ab"); TFileValue* v3 = genTFileValue("b"); TFileValue* v4 = genTFileValue("d"); @@ -510,7 +456,7 @@ TEST_F(IndexTFileEnv, test_tfile_write) { taosArrayDestroy(data); std::string colName("voltage"); - std::string colVal("b"); + std::string colVal("ab"); SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; @@ -521,3 +467,128 @@ TEST_F(IndexTFileEnv, test_tfile_write) { // tfileWriterDestroy(twrite); } +class CacheObj { + public: + CacheObj() { + // TODO + cache = indexCacheCreate(NULL, "voltage", TSDB_DATA_TYPE_BINARY); + } + int Put(SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid) { + int ret = indexCachePut(cache, term, uid); + if (ret != 0) { + // + std::cout << "failed to put into cache: " << ret << std::endl; + } + return ret; + } + void Debug() { + // + indexCacheDebug(cache); + } + int Get(SIndexTermQuery* query, int16_t colId, int32_t version, SArray* result, STermValueType* s) { + int ret = indexCacheSearch(cache, query, result, s); + if (ret != 0) { + // + std::cout << "failed to get from cache:" << ret << std::endl; + } + return ret; + } + ~CacheObj() { + // TODO + indexCacheDestroy(cache); + } + + private: + IndexCache* cache = NULL; +}; + +class IndexCacheEnv : public ::testing::Test { + protected: + virtual void SetUp() { + // TODO + coj = new CacheObj(); + } + virtual void TearDown() { + delete coj; + // formate + } + CacheObj* coj; +}; + +#define MAX_TERM_KEY_LEN 128 +TEST_F(IndexCacheEnv, cache_test) { + int version = 0; + int16_t colId = 0; + int16_t othColId = 10; + + uint64_t suid = 0; + std::string colName("voltage"); + { + std::string colVal("v1"); + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); + coj->Put(term, colId, version++, suid++); + } + { + std::string colVal("v3"); + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); + coj->Put(term, colId, version++, suid++); + } + { + std::string colVal("v2"); + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); + coj->Put(term, colId, version++, suid++); + } + { + std::string colVal("v3"); + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); + coj->Put(term, colId, version++, suid++); + } + { + std::string colVal("v3"); + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); + coj->Put(term, colId, version++, suid++); + } + + { + std::string colVal("v3"); + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); + coj->Put(term, othColId, version++, suid++); + } + { + std::string colVal("v4"); + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); + coj->Put(term, othColId, version++, suid++); + } + { + std::string colVal("v4"); + for (size_t i = 0; i < 10; i++) { + colVal[colVal.size() - 1] = 'a' + i; + SIndexTerm* term = + indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); + coj->Put(term, colId, version++, suid++); + } + } + coj->Debug(); + // begin query + { + std::string colVal("v3"); + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); + SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; + SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); + STermValueType valType; + + coj->Get(&query, colId, 10000, ret, &valType); + // std::cout << "size : " << taosArrayGetSize(ret) << std::endl; + assert(taosArrayGetSize(ret) == 4); + } + { + std::string colVal("v2"); + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); + SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; + SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); + STermValueType valType; + + coj->Get(&query, colId, 10000, ret, &valType); + assert(taosArrayGetSize(ret) == 1); + } +} diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h index 361b4e71cb..d6bed8ef70 100644 --- a/source/libs/parser/inc/astToMsg.h +++ b/source/libs/parser/inc/astToMsg.h @@ -2,7 +2,7 @@ #define TDENGINE_ASTTOMSG_H #include "parserInt.h" -#include "taosmsg.h" +#include "tmsg.h" SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SCreateAcctMsg* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 2510f354fb..1c5f9f5cb3 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -4358,7 +4358,7 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm } pDcl->pMsg = (char*)buildUserManipulationMsg(pInfo, &pDcl->msgLen, pCtx->requestId, msgBuf, msgBufLen); - pDcl->msgType = (pInfo->type == TSDB_SQL_CREATE_USER)? TSDB_MSG_TYPE_CREATE_USER:TSDB_MSG_TYPE_ALTER_USER; + pDcl->msgType = (pInfo->type == TSDB_SQL_CREATE_USER)? TDMT_MND_CREATE_USER:TDMT_MND_ALTER_USER; break; } @@ -4395,20 +4395,20 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm } pDcl->pMsg = (char*)buildAcctManipulationMsg(pInfo, &pDcl->msgLen, pCtx->requestId, msgBuf, msgBufLen); - pDcl->msgType = (pInfo->type == TSDB_SQL_CREATE_ACCT)? TSDB_MSG_TYPE_CREATE_ACCT:TSDB_MSG_TYPE_ALTER_ACCT; + pDcl->msgType = (pInfo->type == TSDB_SQL_CREATE_ACCT)? TDMT_MND_CREATE_ACCT:TDMT_MND_ALTER_ACCT; break; } case TSDB_SQL_DROP_ACCT: case TSDB_SQL_DROP_USER: { pDcl->pMsg = (char*)buildDropUserMsg(pInfo, &pDcl->msgLen, pCtx->requestId, msgBuf, msgBufLen); - pDcl->msgType = (pInfo->type == TSDB_SQL_DROP_ACCT)? TSDB_MSG_TYPE_DROP_ACCT:TSDB_MSG_TYPE_DROP_USER; + pDcl->msgType = (pInfo->type == TSDB_SQL_DROP_ACCT)? TDMT_MND_DROP_ACCT:TDMT_MND_DROP_USER; break; } case TSDB_SQL_SHOW: { code = setShowInfo(&pInfo->pMiscInfo->showOpt, pCtx, (void**)&pDcl->pMsg, &pDcl->msgLen, pMsgBuf); - pDcl->msgType = TSDB_MSG_TYPE_SHOW; + pDcl->msgType = TDMT_MND_SHOW; break; } @@ -4431,7 +4431,7 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm pDcl->pMsg = (char*)pUseDbMsg; pDcl->msgLen = sizeof(SUseDbMsg); - pDcl->msgType = TSDB_MSG_TYPE_USE_DB; + pDcl->msgType = TDMT_MND_USE_DB; break; } @@ -4459,7 +4459,7 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm pDcl->pMsg = (char*)pCreateMsg; pDcl->msgLen = sizeof(SCreateDbMsg); - pDcl->msgType = (pInfo->type == TSDB_SQL_CREATE_DB)? TSDB_MSG_TYPE_CREATE_DB:TSDB_MSG_TYPE_ALTER_DB; + pDcl->msgType = (pInfo->type == TSDB_SQL_CREATE_DB)? TDMT_MND_CREATE_DB:TDMT_MND_ALTER_DB; break; } @@ -4481,7 +4481,7 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm pDropDbMsg->ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; assert(code == TSDB_CODE_SUCCESS && name.type == TSDB_DB_NAME_T); - pDcl->msgType = TSDB_MSG_TYPE_DROP_DB; + pDcl->msgType = TDMT_MND_DROP_DB; pDcl->msgLen = sizeof(SDropDbMsg); pDcl->pMsg = (char*)pDropDbMsg; return TSDB_CODE_SUCCESS; @@ -4495,7 +4495,7 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm return code; } pDcl->pMsg = (char*)buildCreateTableMsg(pCreateTable, &pDcl->msgLen, pCtx, pMsgBuf); - pDcl->msgType = (pCreateTable->type == TSQL_CREATE_TABLE)? TSDB_MSG_TYPE_CREATE_TABLE:TSDB_MSG_TYPE_CREATE_STB; + pDcl->msgType = (pCreateTable->type == TSQL_CREATE_TABLE)? TDMT_MND_CREATE_TABLE:TDMT_MND_CREATE_STB; } else if (pCreateTable->type == TSQL_CREATE_CTABLE) { // if ((code = doCheckForCreateFromStable(pSql, pInfo)) != TSDB_CODE_SUCCESS) { // return code; @@ -4515,7 +4515,7 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm return terrno; } - pDcl->msgType = TSDB_MSG_TYPE_DROP_STB; + pDcl->msgType = TDMT_MND_DROP_STB; return TSDB_CODE_SUCCESS; break; } diff --git a/source/libs/parser/src/dataBlockMgt.c b/source/libs/parser/src/dataBlockMgt.c index 2f3d5aab77..b315ea613f 100644 --- a/source/libs/parser/src/dataBlockMgt.c +++ b/source/libs/parser/src/dataBlockMgt.c @@ -18,7 +18,7 @@ #include "catalog.h" #include "parserUtil.h" #include "queryInfoUtil.h" -#include "taosmsg.h" +#include "tmsg.h" #define IS_RAW_PAYLOAD(t) \ (((int)(t)) == PAYLOAD_TYPE_RAW) // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 9922642df3..0f77135ec1 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -31,7 +31,7 @@ bool isInsertSql(const char* pStr, size_t length) { } while (1); } -bool qIsDclQuery(const SQueryNode* pQuery) { +bool qIsDdlQuery(const SQueryNode* pQuery) { return TSDB_SQL_INSERT != pQuery->type && TSDB_SQL_SELECT != pQuery->type; } @@ -227,3 +227,7 @@ void qParserClearupMetaRequestInfo(SCatalogReq* pMetaReq) { taosArrayDestroy(pMetaReq->pTableName); taosArrayDestroy(pMetaReq->pUdf); } + +void qDestoryQuery(SQueryNode* pQuery) { + // todo +} diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 307062ef15..8133ee62f7 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -14,7 +14,7 @@ */ -#include "taosmsg.h" +#include "tmsg.h" #include "parser.h" #include "taoserror.h" #include "tutil.h" @@ -1825,7 +1825,7 @@ int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t lengt SSqlCmd *pCmd = &pSql->cmd; pCmd->command = TSDB_SQL_MULTI_META; - pCmd->msgType = TSDB_MSG_TYPE_TABLES_META; + pCmd->msgType = TDMT_VND_TABLES_META; int code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; char *str = (char *)pNameList; diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index ed29839905..35c6d59ffe 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -24,7 +24,7 @@ extern "C" { #include "tarray.h" #include "planner.h" #include "parser.h" -#include "taosmsg.h" +#include "tmsg.h" #define QNODE_TAGSCAN 1 #define QNODE_TABLESCAN 2 diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index e8523249e4..6381771400 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -24,7 +24,7 @@ void qDestroyQueryDag(struct SQueryDag* pDag) { // todo } -int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SEpSet* pEpSet, struct SQueryDag** pDag) { +int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag) { SQueryPlanNode* logicPlan; int32_t code = createQueryPlan(pNode, &logicPlan); if (TSDB_CODE_SUCCESS != code) { diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp index f14fd50e03..d0f6fd5adf 100644 --- a/source/libs/planner/test/phyPlanTests.cpp +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -60,7 +60,7 @@ protected: return code; } SQueryDag* dag = nullptr; - code = qCreateQueryDag(query, nullptr, &dag); + code = qCreateQueryDag(query, &dag); dag_.reset(dag); return code; } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index df3fa40004..12fb6e3385 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -1,5 +1,5 @@ #include "os.h" -#include "taosmsg.h" +#include "tmsg.h" #include "query.h" #include "tglobal.h" #include "tsched.h" diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index eb5527f076..34db262d5d 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -13,13 +13,13 @@ * along with this program. If not, see . */ -#include "taosmsg.h" +#include "tmsg.h" #include "queryInt.h" #include "query.h" -int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen) = {0}; +int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen) = {0}; -int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize) = {0}; +int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize) = {0}; int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) { if (NULL == input || NULL == msg || NULL == msgLen) { @@ -92,8 +92,8 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } - pRsp->vgVersion = htonl(pRsp->vgVersion); - pRsp->vgNum = htonl(pRsp->vgNum); + pRsp->vgVersion = ntohl(pRsp->vgVersion); + pRsp->vgNum = ntohl(pRsp->vgNum); if (pRsp->vgNum < 0) { qError("invalid db[%s] vgroup number[%d]", pRsp->db, pRsp->vgNum); @@ -115,12 +115,12 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { } for (int32_t i = 0; i < pRsp->vgNum; ++i) { - pRsp->vgroupInfo[i].vgId = htonl(pRsp->vgroupInfo[i].vgId); - pRsp->vgroupInfo[i].hashBegin = htonl(pRsp->vgroupInfo[i].hashBegin); - pRsp->vgroupInfo[i].hashEnd = htonl(pRsp->vgroupInfo[i].hashEnd); + pRsp->vgroupInfo[i].vgId = ntohl(pRsp->vgroupInfo[i].vgId); + pRsp->vgroupInfo[i].hashBegin = ntohl(pRsp->vgroupInfo[i].hashBegin); + pRsp->vgroupInfo[i].hashEnd = ntohl(pRsp->vgroupInfo[i].hashEnd); for (int32_t n = 0; n < pRsp->vgroupInfo[i].numOfEps; ++n) { - pRsp->vgroupInfo[i].epAddr[n].port = htons(pRsp->vgroupInfo[i].epAddr[n].port); + pRsp->vgroupInfo[i].epAddr[n].port = ntohs(pRsp->vgroupInfo[i].epAddr[n].port); } if (0 != taosHashPut(pOut->dbVgroup.vgInfo, &pRsp->vgroupInfo[i].vgId, sizeof(pRsp->vgroupInfo[i].vgId), &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i]))) { @@ -142,13 +142,13 @@ _return: } static int32_t queryConvertTableMetaMsg(STableMetaMsg* pMetaMsg) { - pMetaMsg->numOfTags = htonl(pMetaMsg->numOfTags); - pMetaMsg->numOfColumns = htonl(pMetaMsg->numOfColumns); - pMetaMsg->sversion = htonl(pMetaMsg->sversion); - pMetaMsg->tversion = htonl(pMetaMsg->tversion); + pMetaMsg->numOfTags = ntohl(pMetaMsg->numOfTags); + pMetaMsg->numOfColumns = ntohl(pMetaMsg->numOfColumns); + pMetaMsg->sversion = ntohl(pMetaMsg->sversion); + pMetaMsg->tversion = ntohl(pMetaMsg->tversion); pMetaMsg->tuid = htobe64(pMetaMsg->tuid); pMetaMsg->suid = htobe64(pMetaMsg->suid); - pMetaMsg->vgId = htonl(pMetaMsg->vgId); + pMetaMsg->vgId = ntohl(pMetaMsg->vgId); if (pMetaMsg->numOfTags < 0 || pMetaMsg->numOfTags > TSDB_MAX_TAGS) { qError("invalid numOfTags[%d] in table meta rsp msg", pMetaMsg->numOfTags); @@ -179,8 +179,8 @@ static int32_t queryConvertTableMetaMsg(STableMetaMsg* pMetaMsg) { int32_t numOfTotalCols = pMetaMsg->numOfColumns + pMetaMsg->numOfTags; for (int i = 0; i < numOfTotalCols; ++i) { - pSchema->bytes = htonl(pSchema->bytes); - pSchema->colId = htonl(pSchema->colId); + pSchema->bytes = ntohl(pSchema->bytes); + pSchema->colId = ntohl(pSchema->colId); pSchema++; } @@ -202,7 +202,8 @@ int32_t queryCreateTableMetaFromMsg(STableMetaMsg* msg, bool isSuperTable, STabl qError("calloc size[%d] failed", metaSize); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - + + pTableMeta->vgId = isSuperTable ? 0 : msg->vgId; pTableMeta->tableType = isSuperTable ? TSDB_SUPER_TABLE : msg->tableType; pTableMeta->uid = msg->suid; pTableMeta->suid = msg->suid; @@ -213,12 +214,12 @@ int32_t queryCreateTableMetaFromMsg(STableMetaMsg* msg, bool isSuperTable, STabl pTableMeta->tableInfo.precision = msg->precision; pTableMeta->tableInfo.numOfColumns = msg->numOfColumns; + memcpy(pTableMeta->schema, msg->pSchema, sizeof(SSchema) * total); + for(int32_t i = 0; i < msg->numOfColumns; ++i) { pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; } - memcpy(pTableMeta->schema, msg->pSchema, sizeof(SSchema) * total); - *pMeta = pTableMeta; return TSDB_CODE_SUCCESS; @@ -264,11 +265,11 @@ int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) { void initQueryModuleMsgHandle() { - queryBuildMsg[TSDB_MSG_TYPE_TABLE_META] = queryBuildTableMetaReqMsg; - queryBuildMsg[TSDB_MSG_TYPE_USE_DB] = queryBuildUseDbMsg; + queryBuildMsg[TDMT_VND_TABLE_META] = queryBuildTableMetaReqMsg; + queryBuildMsg[TDMT_MND_USE_DB] = queryBuildUseDbMsg; - queryProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META] = queryProcessTableMetaRsp; - queryProcessMsgRsp[TSDB_MSG_TYPE_USE_DB] = queryProcessUseDBRsp; + queryProcessMsgRsp[TDMT_VND_TABLE_META] = queryProcessTableMetaRsp; + queryProcessMsgRsp[TDMT_MND_USE_DB] = queryProcessUseDBRsp; } diff --git a/source/libs/qcom/test/queryTest.cpp b/source/libs/qcom/test/queryTest.cpp index d2aedff8fd..ddf89c6272 100644 --- a/source/libs/qcom/test/queryTest.cpp +++ b/source/libs/qcom/test/queryTest.cpp @@ -15,7 +15,7 @@ #include #include -#include "taosmsg.h" +#include "tmsg.h" #include "query.h" #pragma GCC diagnostic ignored "-Wwrite-strings" diff --git a/source/libs/qworker/CMakeLists.txt b/source/libs/qworker/CMakeLists.txt index 4eafa50bdc..001756d7c3 100644 --- a/source/libs/qworker/CMakeLists.txt +++ b/source/libs/qworker/CMakeLists.txt @@ -10,3 +10,5 @@ target_link_libraries( qworker PRIVATE os util transport planner qcom ) + +ADD_SUBDIRECTORY(test) \ No newline at end of file diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 37d3e655c2..149f46273c 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1,4 +1,4 @@ -#include "taosmsg.h" +#include "tmsg.h" #include "query.h" #include "qworker.h" #include "qworkerInt.h" @@ -943,6 +943,11 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { qError("invalid query msg"); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } + + msg->schedulerId = htobe64(msg->schedulerId); + msg->queryId = htobe64(msg->queryId); + msg->taskId = htobe64(msg->taskId); + msg->contentLen = ntohl(msg->contentLen); bool queryDone = false; bool queryRsp = false; diff --git a/source/libs/qworker/test/CMakeLists.txt b/source/libs/qworker/test/CMakeLists.txt new file mode 100644 index 0000000000..6d755ad487 --- /dev/null +++ b/source/libs/qworker/test/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build qworker unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(qworkerTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + qworkerTest + PUBLIC os util common transport gtest qcom planner qworker +) + +TARGET_INCLUDE_DIRECTORIES( + qworkerTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qworker/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/qworker/inc" +) diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp new file mode 100644 index 0000000000..4b54b77544 --- /dev/null +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#pragma GCC diagnostic ignored "-Wwrite-strings" + +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#include "os.h" + +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "tep.h" +#include "trpc.h" +#include "planner.h" +#include "qworker.h" +#include "stub.h" +#include "addr_any.h" + + +namespace { + +int32_t qwtStringToPlan(const char* str, SSubplan** subplan) { + return 0; +} + + +void stubSetStringToPlan() { + static Stub stub; + stub.set(qStringToSubplan, qwtStringToPlan); + { + AddrAny any("libplanner.so"); + std::map result; + any.get_global_func_addr_dynsym("^qStringToSubplan$", result); + for (const auto& f : result) { + stub.set(f.second, qwtStringToPlan); + } + } +} + + +} + + +TEST(testCase, normalCase) { + void *mgmt = NULL; + int32_t code = 0; + void *mockPointer = (void *)0x1; + SRpcMsg queryRpc = {0}; + SRpcMsg readyRpc = {0}; + SRpcMsg fetchRpc = {0}; + SRpcMsg dropRpc = {0}; + SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); + queryMsg->queryId = htobe64(1); + queryMsg->schedulerId = htobe64(1); + queryMsg->taskId = htobe64(1); + queryMsg->contentLen = htonl(100); + queryRpc.pCont = queryMsg; + + SResReadyMsg readyMsg = {0}; + readyMsg.schedulerId = htobe64(1); + readyMsg.queryId = htobe64(1); + readyMsg.taskId = htobe64(1); + readyRpc.pCont = &readyMsg; + + SResFetchMsg fetchMsg = {0}; + fetchMsg.schedulerId = htobe64(1); + fetchMsg.queryId = htobe64(1); + fetchMsg.taskId = htobe64(1); + fetchRpc.pCont = &fetchMsg; + + STaskDropMsg dropMsg = {0}; + dropMsg.schedulerId = htobe64(1); + dropMsg.queryId = htobe64(1); + dropMsg.taskId = htobe64(1); + dropRpc.pCont = &dropMsg; + + stubSetStringToPlan(); + + code = qWorkerInit(NULL, &mgmt); + ASSERT_EQ(code, 0); + + code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); + ASSERT_EQ(code, 0); + + code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); + ASSERT_EQ(code, 0); + + code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc); + ASSERT_EQ(code, 0); + + code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); + ASSERT_EQ(code, 0); + +} + + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + + + diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index bc7bc44350..2381a1dd49 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -43,7 +43,7 @@ typedef struct SSchedulerMgmt { SHashObj *jobs; // key: queryId, value: SQueryJob* } SSchedulerMgmt; -typedef struct SQueryLevel { +typedef struct SSchLevel { int32_t level; int8_t status; SRWLatch lock; @@ -51,12 +51,12 @@ typedef struct SQueryLevel { int32_t taskSucceed; int32_t taskNum; SArray *subTasks; // Element is SQueryTask -} SQueryLevel; +} SSchLevel; -typedef struct SQueryTask { +typedef struct SSchTask { uint64_t taskId; // task id - SQueryLevel *level; // level + SSchLevel *level; // level SSubplan *plan; // subplan char *msg; // operator tree int32_t msgLen; // msg length @@ -66,13 +66,20 @@ typedef struct SQueryTask { int32_t childReady; // child task ready number SArray *children; // the datasource tasks,from which to fetch the result, element is SQueryTask* SArray *parents; // the data destination tasks, get data from current task, element is SQueryTask* -} SQueryTask; +} SSchTask; -typedef struct SQueryJob { +typedef struct SSchJobAttr { + bool needFetch; + bool syncSchedule; + bool queryJob; +} SSchJobAttr; + +typedef struct SSchJob { uint64_t queryId; int32_t levelNum; int32_t levelIdx; int8_t status; + SSchJobAttr attr; SQueryProfileSummary summary; SEpSet dataSrcEps; SEpAddr resEp; @@ -81,15 +88,19 @@ typedef struct SQueryJob { tsem_t rspSem; int32_t userFetch; int32_t remoteFetch; - void *res; + SSchTask *fetchTask; + int32_t errCode; + void *res; + int32_t resNumOfRows; + SHashObj *execTasks; // executing tasks, key:taskid, value:SQueryTask* SHashObj *succTasks; // succeed tasks, key:taskid, value:SQueryTask* SHashObj *failTasks; // failed tasks, key:taskid, value:SQueryTask* SArray *levels; // Element is SQueryLevel, starting from 0. SArray *subPlans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0. -} SQueryJob; +} SSchJob; #define SCH_HAS_QNODE_IN_CLUSTER(type) (false) //TODO CLUSTER TYPE #define SCH_TASK_READY_TO_LUNCH(task) ((task)->childReady >= taosArrayGetSize((task)->children)) // MAY NEED TO ENHANCE @@ -108,7 +119,7 @@ typedef struct SQueryJob { #define SCH_UNLOCK(type, _lock) (SCH_READ == (type) ? taosRUnLockLatch(_lock) : taosWUnLockLatch(_lock)) -extern int32_t schLaunchTask(SQueryJob *job, SQueryTask *task); +extern int32_t schLaunchTask(SSchJob *job, SSchTask *task); #ifdef __cplusplus } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 8e57f12777..978e10cf94 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -14,7 +14,7 @@ */ #include "schedulerInt.h" -#include "taosmsg.h" +#include "tmsg.h" #include "query.h" #include "catalog.h" @@ -51,12 +51,12 @@ int32_t schBuildAndSendRequest(void *pRpc, const SEpSet* pMgmtEps, __taos_async_ */ } -int32_t schBuildTaskRalation(SQueryJob *job, SHashObj *planToTask) { +int32_t schBuildTaskRalation(SSchJob *job, SHashObj *planToTask) { for (int32_t i = 0; i < job->levelNum; ++i) { - SQueryLevel *level = taosArrayGet(job->levels, i); + SSchLevel *level = taosArrayGet(job->levels, i); for (int32_t m = 0; m < level->taskNum; ++m) { - SQueryTask *task = taosArrayGet(level->subTasks, m); + SSchTask *task = taosArrayGet(level->subTasks, m); SSubplan *plan = task->plan; int32_t childNum = plan->pChildern ? (int32_t)taosArrayGetSize(plan->pChildern) : 0; int32_t parentNum = plan->pParents ? (int32_t)taosArrayGetSize(plan->pParents) : 0; @@ -70,14 +70,14 @@ int32_t schBuildTaskRalation(SQueryJob *job, SHashObj *planToTask) { } for (int32_t n = 0; n < childNum; ++n) { - SSubplan *child = taosArrayGet(plan->pChildern, n); - SQueryTask *childTask = taosHashGet(planToTask, &child, POINTER_BYTES); - if (childTask) { + SSubplan **child = taosArrayGet(plan->pChildern, n); + SSchTask **childTask = taosHashGet(planToTask, child, POINTER_BYTES); + if (NULL == childTask || NULL == *childTask) { qError("subplan relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - if (NULL == taosArrayPush(task->children, &childTask)) { + if (NULL == taosArrayPush(task->children, childTask)) { qError("taosArrayPush failed"); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -92,14 +92,14 @@ int32_t schBuildTaskRalation(SQueryJob *job, SHashObj *planToTask) { } for (int32_t n = 0; n < parentNum; ++n) { - SSubplan *parent = taosArrayGet(plan->pParents, n); - SQueryTask *parentTask = taosHashGet(planToTask, &parent, POINTER_BYTES); - if (parentTask) { + SSubplan **parent = taosArrayGet(plan->pParents, n); + SSchTask **parentTask = taosHashGet(planToTask, parent, POINTER_BYTES); + if (NULL == parentTask || NULL == *parentTask) { qError("subplan relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - if (NULL == taosArrayPush(task->parents, &parentTask)) { + if (NULL == taosArrayPush(task->parents, parentTask)) { qError("taosArrayPush failed"); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -107,13 +107,13 @@ int32_t schBuildTaskRalation(SQueryJob *job, SHashObj *planToTask) { } } - SQueryLevel *level = taosArrayGet(job->levels, 0); - if (level->taskNum > 1) { + SSchLevel *level = taosArrayGet(job->levels, 0); + if (job->attr.queryJob && level->taskNum > 1) { qError("invalid plan info, level 0, taskNum:%d", level->taskNum); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - SQueryTask *task = taosArrayGet(level->subTasks, 0); + SSchTask *task = taosArrayGet(level->subTasks, 0); if (task->parents && taosArrayGetSize(task->parents) > 0) { qError("invalid plan info, level 0, parentNum:%d", (int32_t)taosArrayGetSize(task->parents)); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); @@ -124,7 +124,7 @@ int32_t schBuildTaskRalation(SQueryJob *job, SHashObj *planToTask) { } -int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) { +int32_t schValidateAndBuildJob(SQueryDag *dag, SSchJob *job) { int32_t code = 0; job->queryId = dag->queryId; @@ -146,21 +146,23 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - job->levels = taosArrayInit(levelNum, sizeof(SQueryLevel)); + job->levels = taosArrayInit(levelNum, sizeof(SSchLevel)); if (NULL == job->levels) { qError("taosArrayInit %d failed", levelNum); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + job->attr.needFetch = true; + job->levelNum = levelNum; job->levelIdx = levelNum - 1; job->subPlans = dag->pSubplans; - SQueryLevel level = {0}; + SSchLevel level = {0}; SArray *levelPlans = NULL; int32_t levelPlanNum = 0; - SQueryLevel *pLevel = NULL; + SSchLevel *pLevel = NULL; level.status = JOB_TASK_STATUS_NOT_START; @@ -187,7 +189,7 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) { pLevel->taskNum = levelPlanNum; - pLevel->subTasks = taosArrayInit(levelPlanNum, sizeof(SQueryTask)); + pLevel->subTasks = taosArrayInit(levelPlanNum, sizeof(SSchTask)); if (NULL == pLevel->subTasks) { qError("taosArrayInit %d failed", levelPlanNum); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -195,7 +197,14 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) { for (int32_t n = 0; n < levelPlanNum; ++n) { SSubplan *plan = taosArrayGet(levelPlans, n); - SQueryTask task = {0}; + SSchTask task = {0}; + + if (plan->type == QUERY_TYPE_MODIFY) { + job->attr.needFetch = false; + } else { + job->attr.queryJob = true; + } + task.taskId = atomic_add_fetch_64(&schMgmt.taskId, 1); task.plan = plan; @@ -236,7 +245,7 @@ _return: SCH_RET(code); } -int32_t schSetTaskExecEpSet(SQueryJob *job, SEpSet *epSet) { +int32_t schSetTaskExecEpSet(SSchJob *job, SEpSet *epSet) { if (epSet->numOfEps >= SCH_MAX_CONDIDATE_EP_NUM) { return TSDB_CODE_SUCCESS; } @@ -263,7 +272,7 @@ int32_t schSetTaskExecEpSet(SQueryJob *job, SEpSet *epSet) { } -int32_t schPushTaskToExecList(SQueryJob *job, SQueryTask *task) { +int32_t schPushTaskToExecList(SSchJob *job, SSchTask *task) { if (0 != taosHashPut(job->execTasks, &task->taskId, sizeof(task->taskId), &task, POINTER_BYTES)) { qError("taosHashPut failed"); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -272,7 +281,7 @@ int32_t schPushTaskToExecList(SQueryJob *job, SQueryTask *task) { return TSDB_CODE_SUCCESS; } -int32_t schMoveTaskToSuccList(SQueryJob *job, SQueryTask *task, bool *moved) { +int32_t schMoveTaskToSuccList(SSchJob *job, SSchTask *task, bool *moved) { if (0 != taosHashRemove(job->execTasks, &task->taskId, sizeof(task->taskId))) { qWarn("remove task[%"PRIx64"] from execTasks failed", task->taskId); return TSDB_CODE_SUCCESS; @@ -288,10 +297,9 @@ int32_t schMoveTaskToSuccList(SQueryJob *job, SQueryTask *task, bool *moved) { return TSDB_CODE_SUCCESS; } -int32_t schMoveTaskToFailList(SQueryJob *job, SQueryTask *task, bool *moved) { +int32_t schMoveTaskToFailList(SSchJob *job, SSchTask *task, bool *moved) { if (0 != taosHashRemove(job->execTasks, &task->taskId, sizeof(task->taskId))) { - qWarn("remove task[%"PRIx64"] from execTasks failed", task->taskId); - return TSDB_CODE_SUCCESS; + qWarn("remove task[%"PRIx64"] from execTasks failed, it may not exist", task->taskId); } if (0 != taosHashPut(job->failTasks, &task->taskId, sizeof(task->taskId), &task, POINTER_BYTES)) { @@ -305,12 +313,12 @@ int32_t schMoveTaskToFailList(SQueryJob *job, SQueryTask *task, bool *moved) { } -int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) { +int32_t schAsyncSendMsg(SSchJob *job, SSchTask *task, int32_t msgType) { int32_t msgSize = 0; void *msg = NULL; switch (msgType) { - case TSDB_MSG_TYPE_SUBMIT: { + case TDMT_VND_SUBMIT: { if (NULL == task->msg || task->msgLen <= 0) { qError("submit msg is NULL"); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); @@ -320,7 +328,7 @@ int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) { msg = task->msg; break; } - case TSDB_MSG_TYPE_QUERY: { + case TDMT_VND_QUERY: { if (NULL == task->msg) { qError("query msg is NULL"); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); @@ -342,7 +350,7 @@ int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) { memcpy(pMsg->msg, task->msg, task->msgLen); break; } - case TSDB_MSG_TYPE_RES_READY: { + case TDMT_VND_RES_READY: { msgSize = sizeof(SResReadyMsg); msg = calloc(1, msgSize); if (NULL == msg) { @@ -356,7 +364,10 @@ int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) { pMsg->taskId = htobe64(task->taskId); break; } - case TSDB_MSG_TYPE_FETCH: { + case TDMT_VND_FETCH: { + if (NULL == task) { + SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } msgSize = sizeof(SResFetchMsg); msg = calloc(1, msgSize); if (NULL == msg) { @@ -370,7 +381,7 @@ int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) { pMsg->taskId = htobe64(task->taskId); break; } - case TSDB_MSG_TYPE_DROP_TASK:{ + case TDMT_VND_DROP_TASK:{ msgSize = sizeof(STaskDropMsg); msg = calloc(1, msgSize); if (NULL == msg) { @@ -395,7 +406,7 @@ int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) { return TSDB_CODE_SUCCESS; } -int32_t schTaskCheckAndSetRetry(SQueryJob *job, SQueryTask *task, int32_t errCode, bool *needRetry) { +int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, bool *needRetry) { // TODO set retry or not based on task type/errCode/retry times/job status/available eps... // TODO if needRetry, set task retry info @@ -405,7 +416,7 @@ int32_t schTaskCheckAndSetRetry(SQueryJob *job, SQueryTask *task, int32_t errCod } -int32_t schFetchFromRemote(SQueryJob *job) { +int32_t schFetchFromRemote(SSchJob *job) { int32_t code = 0; if (atomic_val_compare_exchange_32(&job->remoteFetch, 0, 1) != 0) { @@ -413,7 +424,7 @@ int32_t schFetchFromRemote(SQueryJob *job) { return TSDB_CODE_SUCCESS; } - SCH_ERR_JRET(schAsyncSendMsg(job, NULL, TSDB_MSG_TYPE_FETCH)); + SCH_ERR_JRET(schAsyncSendMsg(job, job->fetchTask, TDMT_VND_FETCH)); return TSDB_CODE_SUCCESS; @@ -424,8 +435,12 @@ _return: } -int32_t schProcessOnJobSuccess(SQueryJob *job) { - job->status = JOB_TASK_STATUS_SUCCEED; +int32_t schProcessOnJobPartialSuccess(SSchJob *job) { + job->status = JOB_TASK_STATUS_PARTIAL_SUCCEED; + + if ((!job->attr.needFetch) && job->attr.syncSchedule) { + tsem_post(&job->rspSem); + } if (job->userFetch) { SCH_ERR_RET(schFetchFromRemote(job)); @@ -434,26 +449,27 @@ int32_t schProcessOnJobSuccess(SQueryJob *job) { return TSDB_CODE_SUCCESS; } -int32_t schProcessOnJobFailure(SQueryJob *job) { +int32_t schProcessOnJobFailure(SSchJob *job, int32_t errCode) { job->status = JOB_TASK_STATUS_FAILED; + job->errCode = errCode; atomic_val_compare_exchange_32(&job->remoteFetch, 1, 0); - if (job->userFetch) { + if (job->userFetch || ((!job->attr.needFetch) && job->attr.syncSchedule)) { tsem_post(&job->rspSem); } return TSDB_CODE_SUCCESS; } -int32_t schProcessOnDataFetched(SQueryJob *job) { +int32_t schProcessOnDataFetched(SSchJob *job) { atomic_val_compare_exchange_32(&job->remoteFetch, 1, 0); tsem_post(&job->rspSem); } -int32_t schProcessOnTaskSuccess(SQueryJob *job, SQueryTask *task) { +int32_t schProcessOnTaskSuccess(SSchJob *job, SSchTask *task) { bool moved = false; SCH_ERR_RET(schMoveTaskToSuccList(job, task, &moved)); @@ -464,7 +480,7 @@ int32_t schProcessOnTaskSuccess(SQueryJob *job, SQueryTask *task) { task->status = JOB_TASK_STATUS_SUCCEED; - int32_t parentNum = (int32_t)taosArrayGetSize(task->parents); + int32_t parentNum = task->parents ? (int32_t)taosArrayGetSize(task->parents) : 0; if (parentNum == 0) { if (task->plan->level != 0) { qError("level error"); @@ -475,7 +491,7 @@ int32_t schProcessOnTaskSuccess(SQueryJob *job, SQueryTask *task) { if (SCH_TASK_NEED_WAIT_ALL(task)) { SCH_LOCK(SCH_WRITE, &task->level->lock); - task->level->taskFailed++; + task->level->taskSucceed++; taskDone = task->level->taskSucceed + task->level->taskFailed; SCH_UNLOCK(SCH_WRITE, &task->level->lock); @@ -486,7 +502,7 @@ int32_t schProcessOnTaskSuccess(SQueryJob *job, SQueryTask *task) { if (task->level->taskFailed > 0) { job->status = JOB_TASK_STATUS_FAILED; - SCH_ERR_RET(schProcessOnJobFailure(job)); + SCH_ERR_RET(schProcessOnJobFailure(job, TSDB_CODE_QRY_APP_ERROR)); return TSDB_CODE_SUCCESS; } @@ -495,7 +511,9 @@ int32_t schProcessOnTaskSuccess(SQueryJob *job, SQueryTask *task) { job->resEp.port = task->execAddr.port; } - SCH_ERR_RET(schProcessOnJobSuccess(job)); + job->fetchTask = task; + + SCH_ERR_RET(schProcessOnJobPartialSuccess(job)); return TSDB_CODE_SUCCESS; } @@ -508,21 +526,21 @@ int32_t schProcessOnTaskSuccess(SQueryJob *job, SQueryTask *task) { } for (int32_t i = 0; i < parentNum; ++i) { - SQueryTask *par = taosArrayGet(task->parents, i); + SSchTask *par = *(SSchTask **)taosArrayGet(task->parents, i); ++par->childReady; SCH_ERR_RET(qSetSubplanExecutionNode(par->plan, task->plan->id.templateId, &task->execAddr)); if (SCH_TASK_READY_TO_LUNCH(par)) { - SCH_ERR_RET(schLaunchTask(job, task)); + SCH_ERR_RET(schLaunchTask(job, par)); } } return TSDB_CODE_SUCCESS; } -int32_t schProcessOnTaskFailure(SQueryJob *job, SQueryTask *task, int32_t errCode) { +int32_t schProcessOnTaskFailure(SSchJob *job, SSchTask *task, int32_t errCode) { bool needRetry = false; bool moved = false; int32_t taskDone = 0; @@ -534,7 +552,6 @@ int32_t schProcessOnTaskFailure(SQueryJob *job, SQueryTask *task, int32_t errCod SCH_ERR_RET(schMoveTaskToFailList(job, task, &moved)); if (!moved) { SCH_TASK_ERR_LOG("task may already moved, status:%d", task->status); - return TSDB_CODE_SUCCESS; } if (SCH_TASK_NEED_WAIT_ALL(task)) { @@ -550,7 +567,7 @@ int32_t schProcessOnTaskFailure(SQueryJob *job, SQueryTask *task, int32_t errCod } job->status = JOB_TASK_STATUS_FAILED; - SCH_ERR_RET(schProcessOnJobFailure(job)); + SCH_ERR_RET(schProcessOnJobFailure(job, errCode)); return TSDB_CODE_SUCCESS; } @@ -560,34 +577,60 @@ int32_t schProcessOnTaskFailure(SQueryJob *job, SQueryTask *task, int32_t errCod return TSDB_CODE_SUCCESS; } -int32_t schHandleRspMsg(SQueryJob *job, SQueryTask *task, int32_t msgType, int32_t rspCode) { +int32_t schHandleRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode) { int32_t code = 0; switch (msgType) { - case TSDB_MSG_TYPE_QUERY: - if (rspCode != TSDB_CODE_SUCCESS) { - SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode)); - } else { - code = schAsyncSendMsg(job, task, TSDB_MSG_TYPE_RES_READY); - if (code) { - goto _task_error; + case TDMT_VND_SUBMIT: { + SShellSubmitRspMsg *rsp = (SShellSubmitRspMsg *)msg; + if (rsp->code != TSDB_CODE_SUCCESS) { + SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code)); + } else { + job->resNumOfRows += rsp->affectedRows; + + code = schProcessOnTaskSuccess(job, task); + if (code) { + goto _task_error; + } } + break; } - break; - case TSDB_MSG_TYPE_RES_READY: - if (rspCode != TSDB_CODE_SUCCESS) { - SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode)); - } else { - code = schProcessOnTaskSuccess(job, task); - if (code) { - goto _task_error; - } + case TDMT_VND_QUERY: { + SQueryTableRsp *rsp = (SQueryTableRsp *)msg; + + if (rsp->code != TSDB_CODE_SUCCESS) { + SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code)); + } else { + code = schAsyncSendMsg(job, task, TDMT_VND_RES_READY); + if (code) { + goto _task_error; + } + } + break; + } + case TDMT_VND_RES_READY: { + SResReadyRsp *rsp = (SResReadyRsp *)msg; + + if (rsp->code != TSDB_CODE_SUCCESS) { + SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code)); + } else { + code = schProcessOnTaskSuccess(job, task); + if (code) { + goto _task_error; + } + } + break; + } + case TDMT_VND_FETCH: { + SCH_ERR_JRET(rspCode); + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; + + job->res = rsp; + job->resNumOfRows = rsp->numOfRows; + + SCH_ERR_JRET(schProcessOnDataFetched(job)); + break; } - break; - case TSDB_MSG_TYPE_FETCH: - SCH_ERR_JRET(rspCode); - SCH_ERR_JRET(schProcessOnDataFetched(job)); - break; default: qError("unknown msg type:%d received", msgType); return TSDB_CODE_QRY_INVALID_INPUT; @@ -600,14 +643,14 @@ _task_error: return TSDB_CODE_SUCCESS; _return: - code = schProcessOnJobFailure(job); + code = schProcessOnJobFailure(job, code); return code; } -int32_t schLaunchTask(SQueryJob *job, SQueryTask *task) { +int32_t schLaunchTask(SSchJob *job, SSchTask *task) { SSubplan *plan = task->plan; SCH_ERR_RET(qSubPlanToString(plan, &task->msg, &task->msgLen)); if (plan->execEpSet.numOfEps <= 0) { @@ -619,7 +662,7 @@ int32_t schLaunchTask(SQueryJob *job, SQueryTask *task) { SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - int32_t msgType = (plan->type == QUERY_TYPE_MODIFY) ? TSDB_MSG_TYPE_SUBMIT : TSDB_MSG_TYPE_QUERY; + int32_t msgType = (plan->type == QUERY_TYPE_MODIFY) ? TDMT_VND_SUBMIT : TDMT_VND_QUERY; SCH_ERR_RET(schAsyncSendMsg(job, task, msgType)); @@ -630,10 +673,10 @@ int32_t schLaunchTask(SQueryJob *job, SQueryTask *task) { return TSDB_CODE_SUCCESS; } -int32_t schLaunchJob(SQueryJob *job) { - SQueryLevel *level = taosArrayGet(job->levels, job->levelIdx); +int32_t schLaunchJob(SSchJob *job) { + SSchLevel *level = taosArrayGet(job->levels, job->levelIdx); for (int32_t i = 0; i < level->taskNum; ++i) { - SQueryTask *task = taosArrayGet(level->subTasks, i); + SSchTask *task = taosArrayGet(level->subTasks, i); SCH_ERR_RET(schLaunchTask(job, task)); } @@ -642,21 +685,21 @@ int32_t schLaunchJob(SQueryJob *job) { return TSDB_CODE_SUCCESS; } -void schDropJobAllTasks(SQueryJob *job) { +void schDropJobAllTasks(SSchJob *job) { void *pIter = taosHashIterate(job->succTasks, NULL); while (pIter) { - SQueryTask *task = *(SQueryTask **)pIter; + SSchTask *task = *(SSchTask **)pIter; - schAsyncSendMsg(job, task, TSDB_MSG_TYPE_DROP_TASK); + schAsyncSendMsg(job, task, TDMT_VND_DROP_TASK); pIter = taosHashIterate(job->succTasks, pIter); } pIter = taosHashIterate(job->failTasks, NULL); while (pIter) { - SQueryTask *task = *(SQueryTask **)pIter; + SSchTask *task = *(SSchTask **)pIter; - schAsyncSendMsg(job, task, TSDB_MSG_TYPE_DROP_TASK); + schAsyncSendMsg(job, task, TDMT_VND_DROP_TASK); pIter = taosHashIterate(job->succTasks, pIter); } @@ -680,7 +723,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { } -int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob) { +int32_t scheduleExecJobImpl(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob, bool syncSchedule) { if (NULL == transport || NULL == transport ||NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -690,11 +733,12 @@ int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, voi } int32_t code = 0; - SQueryJob *job = calloc(1, sizeof(SQueryJob)); + SSchJob *job = calloc(1, sizeof(SSchJob)); if (NULL == job) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + job->attr.syncSchedule = syncSchedule; job->transport = transport; job->qnodeList = qnodeList; @@ -719,54 +763,104 @@ int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, voi } tsem_init(&job->rspSem, 0, 0); - - if (0 != taosHashPut(schMgmt.jobs, &job->queryId, sizeof(job->queryId), &job, POINTER_BYTES)) { - qError("taosHashPut queryId:%"PRIx64" failed", job->queryId); - SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + + code = taosHashPut(schMgmt.jobs, &job->queryId, sizeof(job->queryId), &job, POINTER_BYTES); + if (0 != code) { + if (HASH_NODE_EXIST(code)) { + qError("taosHashPut queryId:%"PRIx64" already exist", job->queryId); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } else { + qError("taosHashPut queryId:%"PRIx64" failed", job->queryId); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } } job->status = JOB_TASK_STATUS_NOT_START; SCH_ERR_JRET(schLaunchJob(job)); - *(SQueryJob **)pJob = job; + *(SSchJob **)pJob = job; + + if (syncSchedule) { + tsem_wait(&job->rspSem); + } return TSDB_CODE_SUCCESS; _return: - *(SQueryJob **)pJob = NULL; + *(SSchJob **)pJob = NULL; scheduleFreeJob(job); SCH_RET(code); } +int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob, uint64_t *numOfRows) { + *numOfRows = 0; + + SCH_ERR_RET(scheduleExecJobImpl(transport, qnodeList, pDag, pJob, true)); + + SSchJob *job = *(SSchJob **)pJob; + + *numOfRows = job->resNumOfRows; + + return TSDB_CODE_SUCCESS; +} + +int32_t scheduleAsyncExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob) { + return scheduleExecJobImpl(transport, qnodeList, pDag, pJob, false); +} + + int32_t scheduleFetchRows(void *pJob, void **data) { if (NULL == pJob || NULL == data) { - return TSDB_CODE_QRY_INVALID_INPUT; + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SQueryJob *job = pJob; + SSchJob *job = pJob; int32_t code = 0; - if (atomic_val_compare_exchange_32(&job->userFetch, 0, 1) != 0) { - qError("prior fetching not finished"); - return TSDB_CODE_QRY_APP_ERROR; + if (!job->attr.needFetch) { + qError("no need to fetch data"); + SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + if (job->status == JOB_TASK_STATUS_FAILED) { + job->res = NULL; + SCH_RET(job->errCode); } if (job->status == JOB_TASK_STATUS_SUCCEED) { + job->res = NULL; + return TSDB_CODE_SUCCESS; + } + + if (atomic_val_compare_exchange_32(&job->userFetch, 0, 1) != 0) { + qError("prior fetching not finished"); + SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + if (job->status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { SCH_ERR_JRET(schFetchFromRemote(job)); } tsem_wait(&job->rspSem); + if (job->status == JOB_TASK_STATUS_FAILED) { + code = job->errCode; + } + + if (job->res && ((SRetrieveTableRsp *)job->res)->completed) { + job->status = JOB_TASK_STATUS_SUCCEED; + } + *data = job->res; job->res = NULL; _return: atomic_val_compare_exchange_32(&job->userFetch, 1, 0); - return code; + SCH_RET(code); } int32_t scheduleCancelJob(void *pJob) { @@ -782,7 +876,7 @@ void scheduleFreeJob(void *pJob) { return; } - SQueryJob *job = pJob; + SSchJob *job = pJob; if (job->status > 0) { if (0 != taosHashRemove(schMgmt.jobs, &job->queryId, sizeof(job->queryId))) { diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 9e94553058..4732429d0b 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -26,72 +26,310 @@ #include "taos.h" #include "tdef.h" #include "tvariant.h" -#include "catalog.h" -#include "scheduler.h" +#include "catalog.h" +#include "scheduler.h" #include "tep.h" #include "trpc.h" - +#include "schedulerInt.h" +#include "stub.h" +#include "addr_any.h" + namespace { -void mockBuildDag(SQueryDag *dag) { - uint64_t qId = 0x111111111111; - - dag->queryId = qId; - dag->numOfSubplans = 2; - dag->pSubplans = taosArrayInit(dag->numOfSubplans, POINTER_BYTES); - SArray *scan = taosArrayInit(1, sizeof(SSubplan)); - SArray *merge = taosArrayInit(1, sizeof(SSubplan)); - - SSubplan scanPlan = {0}; - SSubplan mergePlan = {0}; - - scanPlan.id.queryId = qId; - scanPlan.id.templateId = 0x2222222222; - scanPlan.id.subplanId = 0x3333333333; - scanPlan.type = QUERY_TYPE_SCAN; - scanPlan.level = 1; - scanPlan.execEpSet.numOfEps = 1; - scanPlan.pChildern = NULL; - scanPlan.pParents = taosArrayInit(1, POINTER_BYTES); - - mergePlan.id.queryId = qId; - mergePlan.id.templateId = 0x4444444444; - mergePlan.id.subplanId = 0x5555555555; - mergePlan.type = QUERY_TYPE_MERGE; - mergePlan.level = 0; - mergePlan.execEpSet.numOfEps = 1; - mergePlan.pChildern = taosArrayInit(1, POINTER_BYTES); - mergePlan.pParents = NULL; - - SSubplan *mergePointer = (SSubplan *)taosArrayPush(merge, &mergePlan); - SSubplan *scanPointer = (SSubplan *)taosArrayPush(scan, &scanPlan); - - taosArrayPush(mergePointer->pChildern, &scanPointer); - taosArrayPush(scanPointer->pParents, &mergePointer); - - taosArrayPush(dag->pSubplans, &merge); - taosArrayPush(dag->pSubplans, &scan); -} - + +extern "C" int32_t schHandleRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode); + +void schtBuildQueryDag(SQueryDag *dag) { + uint64_t qId = 0x0000000000000001; + + dag->queryId = qId; + dag->numOfSubplans = 2; + dag->pSubplans = taosArrayInit(dag->numOfSubplans, POINTER_BYTES); + SArray *scan = taosArrayInit(1, sizeof(SSubplan)); + SArray *merge = taosArrayInit(1, sizeof(SSubplan)); + + SSubplan scanPlan = {0}; + SSubplan mergePlan = {0}; + + scanPlan.id.queryId = qId; + scanPlan.id.templateId = 0x0000000000000002; + scanPlan.id.subplanId = 0x0000000000000003; + scanPlan.type = QUERY_TYPE_SCAN; + scanPlan.level = 1; + scanPlan.execEpSet.numOfEps = 1; + scanPlan.execEpSet.port[0] = 6030; + strcpy(scanPlan.execEpSet.fqdn[0], "ep0"); + scanPlan.pChildern = NULL; + scanPlan.pParents = taosArrayInit(1, POINTER_BYTES); + scanPlan.pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); + + mergePlan.id.queryId = qId; + mergePlan.id.templateId = 0x4444444444; + mergePlan.id.subplanId = 0x5555555555; + mergePlan.type = QUERY_TYPE_MERGE; + mergePlan.level = 0; + mergePlan.execEpSet.numOfEps = 0; + mergePlan.pChildern = taosArrayInit(1, POINTER_BYTES); + mergePlan.pParents = NULL; + mergePlan.pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); + + SSubplan *mergePointer = (SSubplan *)taosArrayPush(merge, &mergePlan); + SSubplan *scanPointer = (SSubplan *)taosArrayPush(scan, &scanPlan); + + taosArrayPush(mergePointer->pChildern, &scanPointer); + taosArrayPush(scanPointer->pParents, &mergePointer); + + taosArrayPush(dag->pSubplans, &merge); + taosArrayPush(dag->pSubplans, &scan); } -TEST(testCase, normalCase) { - void *mockPointer = (void *)0x1; +void schtBuildInsertDag(SQueryDag *dag) { + uint64_t qId = 0x0000000000000002; + + dag->queryId = qId; + dag->numOfSubplans = 2; + dag->pSubplans = taosArrayInit(1, POINTER_BYTES); + SArray *inserta = taosArrayInit(dag->numOfSubplans, sizeof(SSubplan)); + + SSubplan insertPlan[2] = {0}; + + insertPlan[0].id.queryId = qId; + insertPlan[0].id.templateId = 0x0000000000000003; + insertPlan[0].id.subplanId = 0x0000000000000004; + insertPlan[0].type = QUERY_TYPE_MODIFY; + insertPlan[0].level = 0; + insertPlan[0].execEpSet.numOfEps = 1; + insertPlan[0].execEpSet.port[0] = 6030; + strcpy(insertPlan[0].execEpSet.fqdn[0], "ep0"); + insertPlan[0].pChildern = NULL; + insertPlan[0].pParents = NULL; + insertPlan[0].pNode = NULL; + insertPlan[0].pDataSink = (SDataSink*)calloc(1, sizeof(SDataSink)); + + insertPlan[1].id.queryId = qId; + insertPlan[1].id.templateId = 0x0000000000000003; + insertPlan[1].id.subplanId = 0x0000000000000005; + insertPlan[1].type = QUERY_TYPE_MODIFY; + insertPlan[1].level = 0; + insertPlan[1].execEpSet.numOfEps = 1; + insertPlan[1].execEpSet.port[0] = 6030; + strcpy(insertPlan[1].execEpSet.fqdn[0], "ep1"); + insertPlan[1].pChildern = NULL; + insertPlan[1].pParents = NULL; + insertPlan[1].pNode = NULL; + insertPlan[1].pDataSink = (SDataSink*)calloc(1, sizeof(SDataSink)); + + + taosArrayPush(inserta, &insertPlan[0]); + taosArrayPush(inserta, &insertPlan[1]); + + taosArrayPush(dag->pSubplans, &inserta); +} + + +int32_t schtPlanToString(const SSubplan *subplan, char** str, int32_t* len) { + *str = (char *)calloc(1, 20); + *len = 20; + return 0; +} + +int32_t schtExecNode(SSubplan* subplan, uint64_t templateId, SEpAddr* ep) { + return 0; +} + + +void schtSetPlanToString() { + static Stub stub; + stub.set(qSubPlanToString, schtPlanToString); + { + AddrAny any("libplanner.so"); + std::map result; + any.get_global_func_addr_dynsym("^qSubPlanToString$", result); + for (const auto& f : result) { + stub.set(f.second, schtPlanToString); + } + } +} + +void schtSetExecNode() { + static Stub stub; + stub.set(qSetSubplanExecutionNode, schtExecNode); + { + AddrAny any("libplanner.so"); + std::map result; + any.get_global_func_addr_dynsym("^qSetSubplanExecutionNode$", result); + for (const auto& f : result) { + stub.set(f.second, schtExecNode); + } + } +} + +void *schtSendRsp(void *param) { + SSchJob *job = NULL; + int32_t code = 0; + + while (true) { + job = *(SSchJob **)param; + if (job) { + break; + } + + usleep(1000); + } + + void *pIter = taosHashIterate(job->execTasks, NULL); + while (pIter) { + SSchTask *task = *(SSchTask **)pIter; + + SShellSubmitRspMsg rsp = {0}; + rsp.affectedRows = 10; + schHandleRspMsg(job, task, TDMT_VND_SUBMIT, (char *)&rsp, sizeof(rsp), 0); + + pIter = taosHashIterate(job->execTasks, pIter); + } + + return NULL; +} + +void *pInsertJob = NULL; + + +} + +TEST(queryTest, normalCase) { + void *mockPointer = (void *)0x1; char *clusterId = "cluster1"; char *dbname = "1.db1"; char *tablename = "table1"; - SVgroupInfo vgInfo = {0}; - void *pJob = NULL; - SQueryDag dag = {0}; - SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); - - int32_t code = schedulerInit(NULL); + SVgroupInfo vgInfo = {0}; + void *pJob = NULL; + SQueryDag dag = {0}; + SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); + + SEpAddr qnodeAddr = {0}; + strcpy(qnodeAddr.fqdn, "qnode0.ep"); + qnodeAddr.port = 6031; + taosArrayPush(qnodeList, &qnodeAddr); + + int32_t code = schedulerInit(NULL); ASSERT_EQ(code, 0); - - mockBuildDag(&dag); - - code = scheduleExecJob(mockPointer, qnodeList, &dag, &pJob); + + schtBuildQueryDag(&dag); + + schtSetPlanToString(); + schtSetExecNode(); + + code = scheduleAsyncExecJob(mockPointer, qnodeList, &dag, &pJob); ASSERT_EQ(code, 0); -} + + SSchJob *job = (SSchJob *)pJob; + void *pIter = taosHashIterate(job->execTasks, NULL); + while (pIter) { + SSchTask *task = *(SSchTask **)pIter; + + SQueryTableRsp rsp = {0}; + code = schHandleRspMsg(job, task, TDMT_VND_QUERY, (char *)&rsp, sizeof(rsp), 0); + + ASSERT_EQ(code, 0); + pIter = taosHashIterate(job->execTasks, pIter); + } + + pIter = taosHashIterate(job->execTasks, NULL); + while (pIter) { + SSchTask *task = *(SSchTask **)pIter; + + SResReadyRsp rsp = {0}; + code = schHandleRspMsg(job, task, TDMT_VND_RES_READY, (char *)&rsp, sizeof(rsp), 0); + + ASSERT_EQ(code, 0); + pIter = taosHashIterate(job->execTasks, pIter); + } + + pIter = taosHashIterate(job->execTasks, NULL); + while (pIter) { + SSchTask *task = *(SSchTask **)pIter; + + SQueryTableRsp rsp = {0}; + code = schHandleRspMsg(job, task, TDMT_VND_QUERY, (char *)&rsp, sizeof(rsp), 0); + + ASSERT_EQ(code, 0); + pIter = taosHashIterate(job->execTasks, pIter); + } + + pIter = taosHashIterate(job->execTasks, NULL); + while (pIter) { + SSchTask *task = *(SSchTask **)pIter; + + SResReadyRsp rsp = {0}; + code = schHandleRspMsg(job, task, TDMT_VND_RES_READY, (char *)&rsp, sizeof(rsp), 0); + ASSERT_EQ(code, 0); + + pIter = taosHashIterate(job->execTasks, pIter); + } + + SRetrieveTableRsp rsp = {0}; + rsp.completed = 1; + rsp.numOfRows = 10; + code = schHandleRspMsg(job, NULL, TDMT_VND_FETCH, (char *)&rsp, sizeof(rsp), 0); + + ASSERT_EQ(code, 0); + + + void *data = NULL; + + code = scheduleFetchRows(job, &data); + ASSERT_EQ(code, 0); + + SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data; + ASSERT_EQ(pRsp->completed, 1); + ASSERT_EQ(pRsp->numOfRows, 10); + + data = NULL; + code = scheduleFetchRows(job, &data); + ASSERT_EQ(code, 0); + ASSERT_EQ(data, (void*)NULL); + + scheduleFreeJob(pJob); +} + + + + +TEST(insertTest, normalCase) { + void *mockPointer = (void *)0x1; + char *clusterId = "cluster1"; + char *dbname = "1.db1"; + char *tablename = "table1"; + SVgroupInfo vgInfo = {0}; + SQueryDag dag = {0}; + uint64_t numOfRows = 0; + SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); + + SEpAddr qnodeAddr = {0}; + strcpy(qnodeAddr.fqdn, "qnode0.ep"); + qnodeAddr.port = 6031; + taosArrayPush(qnodeList, &qnodeAddr); + + int32_t code = schedulerInit(NULL); + ASSERT_EQ(code, 0); + + schtBuildInsertDag(&dag); + + schtSetPlanToString(); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + + pthread_t thread1; + pthread_create(&(thread1), &thattr, schtSendRsp, &pInsertJob); + + code = scheduleExecJob(mockPointer, qnodeList, &dag, &pInsertJob, &numOfRows); + ASSERT_EQ(code, 0); + ASSERT_EQ(numOfRows, 20); + + scheduleFreeJob(pInsertJob); +} + + int main(int argc, char** argv) { @@ -101,4 +339,4 @@ int main(int argc, char** argv) { - + diff --git a/source/libs/transport/inc/rpcHead.h b/source/libs/transport/inc/rpcHead.h index 5b401ac54b..6e98bbd563 100644 --- a/source/libs/transport/inc/rpcHead.h +++ b/source/libs/transport/inc/rpcHead.h @@ -52,7 +52,7 @@ typedef struct { char user[TSDB_UNI_LEN]; // user ID uint16_t port; // for UDP only, port may be changed char empty[1]; // reserved - uint8_t msgType; // message type + uint16_t msgType; // message type int32_t msgLen; // message length including the header iteslf uint32_t msgVer; int32_t code; // code in response message diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 2f710f3efd..a9098b5053 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -23,7 +23,7 @@ #include "tref.h" #include "taoserror.h" #include "tglobal.h" -#include "taosmsg.h" +#include "tmsg.h" #include "trpc.h" #include "thash.h" #include "rpcLog.h" @@ -74,7 +74,7 @@ typedef struct { SEpSet epSet; // ip list provided by app void *ahandle; // handle provided by app struct SRpcConn *pConn; // pConn allocated - char msgType; // message type + tmsg_t msgType; // message type uint8_t *pCont; // content provided by app int32_t contLen; // content length int32_t code; // error code @@ -108,8 +108,8 @@ typedef struct SRpcConn { uint16_t tranId; // outgoing transcation ID, for build message uint16_t outTranId; // outgoing transcation ID uint16_t inTranId; // transcation ID for incoming msg - uint8_t outType; // message type for outgoing request - uint8_t inType; // message type for incoming request + tmsg_t outType; // message type for outgoing request + tmsg_t inType; // message type for incoming request void *chandle; // handle passed by TCP/UDP connection layer void *ahandle; // handle provided by upper app layter int retry; // number of retry for sending request @@ -409,11 +409,11 @@ void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t // connection type is application specific. // for TDengine, all the query, show commands shall have TCP connection - char type = pMsg->msgType; - if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_SHOW_RETRIEVE - || type == TSDB_MSG_TYPE_FETCH || type == TSDB_MSG_TYPE_VGROUP_LIST - || type == TSDB_MSG_TYPE_TABLES_META || type == TSDB_MSG_TYPE_TABLE_META - || type == TSDB_MSG_TYPE_SHOW || type == TSDB_MSG_TYPE_STATUS || type == TSDB_MSG_TYPE_ALTER_TABLE) + tmsg_t type = pMsg->msgType; + if (type == TDMT_VND_QUERY || type == TDMT_MND_SHOW_RETRIEVE + || type == TDMT_VND_FETCH || type == TDMT_MND_VGROUP_LIST + || type == TDMT_VND_TABLES_META || type == TDMT_VND_TABLE_META + || type == TDMT_MND_SHOW || type == TDMT_MND_STATUS || type == TDMT_VND_ALTER_TABLE) pContext->connType = RPC_CONN_TCPC; pContext->rid = taosAddRef(tsRpcRefId, pContext); @@ -837,16 +837,16 @@ static int rpcProcessReqHead(SRpcConn *pConn, SRpcHead *pHead) { if (pConn->inTranId == pHead->tranId) { if (pConn->inType == pHead->msgType) { if (pHead->code == 0) { - tDebug("%s, %s is retransmitted", pConn->info, taosMsg[pHead->msgType]); + tDebug("%s, %s is retransmitted", pConn->info, TMSG_INFO(pHead->msgType)); rpcSendQuickRsp(pConn, TSDB_CODE_RPC_ACTION_IN_PROGRESS); } else { // do nothing, it is heart beat from client } } else if (pConn->inType == 0) { - tDebug("%s, %s is already processed, tranId:%d", pConn->info, taosMsg[pHead->msgType], pConn->inTranId); + tDebug("%s, %s is already processed, tranId:%d", pConn->info, TMSG_INFO(pHead->msgType), pConn->inTranId); rpcSendMsgToPeer(pConn, pConn->pRspMsg, pConn->rspMsgLen); // resend the response } else { - tDebug("%s, mismatched message %s and tranId", pConn->info, taosMsg[pHead->msgType]); + tDebug("%s, mismatched message %s and tranId", pConn->info, TMSG_INFO(pHead->msgType)); } // do not reply any message @@ -957,19 +957,19 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont sid = htonl(pHead->destId); *ppContext = NULL; - if (pHead->msgType >= TSDB_MSG_TYPE_MAX || pHead->msgType <= 0) { + if (TMSG_INDEX(pHead->msgType) >= TDMT_MAX || TMSG_INDEX(pHead->msgType) <= 0) { tDebug("%s sid:%d, invalid message type:%d", pRpc->label, sid, pHead->msgType); terrno = TSDB_CODE_RPC_INVALID_MSG_TYPE; return NULL; } if (sid < 0 || sid >= pRpc->sessions) { tDebug("%s sid:%d, sid is out of range, max sid:%d, %s discarded", pRpc->label, sid, - pRpc->sessions, taosMsg[pHead->msgType]); + pRpc->sessions, TMSG_INFO(pHead->msgType)); terrno = TSDB_CODE_RPC_INVALID_SESSION_ID; return NULL; } if (rpcIsReq(pHead->msgType) && htonl(pHead->msgVer) != tsVersion >> 8) { - tDebug("%s sid:%d, invalid client version:%x/%x %s", pRpc->label, sid, htonl(pHead->msgVer), tsVersion, taosMsg[pHead->msgType]); + tDebug("%s sid:%d, invalid client version:%x/%x %s", pRpc->label, sid, htonl(pHead->msgVer), tsVersion, TMSG_INFO(pHead->msgType)); terrno = TSDB_CODE_RPC_INVALID_VERSION; return NULL; } @@ -1094,9 +1094,9 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { SRpcReqContext *pContext; pConn = rpcProcessMsgHead(pRpc, pRecv, &pContext); - if (pHead->msgType >= 1 && pHead->msgType < TSDB_MSG_TYPE_MAX) { + if (TMSG_INDEX(pHead->msgType) >= 1 && TMSG_INDEX(pHead->msgType) < TDMT_MAX) { tDebug("%s %p %p, %s received from 0x%x:%hu, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label, - pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType], pRecv->ip, pRecv->port, terrno, pRecv->msgLen, + pConn, (void *)pHead->ahandle, TMSG_INFO(pHead->msgType), pRecv->ip, pRecv->port, terrno, pRecv->msgLen, pHead->sourceId, pHead->destId, pHead->tranId, pHead->code); } else { tDebug("%s %p %p, %d received from 0x%x:%hu, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label, @@ -1112,11 +1112,11 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE) { rpcCloseConn(pConn); } - if (pHead->msgType + 1 > 1 && pHead->msgType+1 < TSDB_MSG_TYPE_MAX) { - tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code); + if (TMSG_INDEX(pHead->msgType) + 1 > 1 && TMSG_INDEX(pHead->msgType) + 1 < TDMT_MAX) { + tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, TMSG_INFO(pHead->msgType+1), code); } else { - tError("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType], code); - } + tError("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, TMSG_INFO(pHead->msgType), code); + } } } else { // msg is passed to app only parsing is ok rpcProcessIncomingMsg(pConn, pHead, pContext); @@ -1262,7 +1262,7 @@ static void rpcSendErrorMsgToPeer(SRecvInfo *pRecv, int32_t code) { memset(msg, 0, sizeof(SRpcHead)); pReplyHead->version = pRecvHead->version; - pReplyHead->msgType = (char)(pRecvHead->msgType + 1); + pReplyHead->msgType = (tmsg_t)(pRecvHead->msgType + 1); pReplyHead->spi = 0; pReplyHead->encrypt = pRecvHead->encrypt; pReplyHead->tranId = pRecvHead->tranId; @@ -1292,7 +1292,7 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) { SRpcHead *pHead = rpcHeadFromCont(pContext->pCont); char *msg = (char *)pHead; int msgLen = rpcMsgLenFromCont(pContext->contLen); - char msgType = pContext->msgType; + tmsg_t msgType = pContext->msgType; pContext->numOfTry++; SRpcConn *pConn = rpcSetupConnToServer(pContext); @@ -1343,12 +1343,12 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) { if ( rpcIsReq(pHead->msgType)) { tDebug("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d", - pConn->info, taosMsg[pHead->msgType], pConn->peerFqdn, pConn->peerPort, + pConn->info, TMSG_INFO(pHead->msgType), pConn->peerFqdn, pConn->peerPort, msgLen, pHead->sourceId, pHead->destId, pHead->tranId); } else { if (pHead->code == 0) pConn->secured = 1; // for success response, set link as secured tDebug("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d", - pConn->info, taosMsg[pHead->msgType], pConn->peerIp, pConn->peerPort, + pConn->info, TMSG_INFO(pHead->msgType), pConn->peerIp, pConn->peerPort, htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId); } @@ -1373,7 +1373,7 @@ static void rpcProcessConnError(void *param, void *id) { tDebug("%s %p, connection error happens", pRpc->label, pContext->ahandle); - if (pContext->numOfTry >= pContext->epSet.numOfEps || pContext->msgType == TSDB_MSG_TYPE_FETCH) { + if (pContext->numOfTry >= pContext->epSet.numOfEps || pContext->msgType == TDMT_VND_FETCH) { rpcMsg.msgType = pContext->msgType+1; rpcMsg.ahandle = pContext->ahandle; rpcMsg.code = pContext->code; @@ -1396,17 +1396,17 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) { rpcLockConn(pConn); if (pConn->outType && pConn->user[0]) { - tDebug("%s, expected %s is not received", pConn->info, taosMsg[(int)pConn->outType + 1]); + tDebug("%s, expected %s is not received", pConn->info, TMSG_INFO((int)pConn->outType + 1)); pConn->pTimer = NULL; pConn->retry++; if (pConn->retry < 4) { - tDebug("%s, re-send msg:%s to %s:%hu", pConn->info, taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort); + tDebug("%s, re-send msg:%s to %s:%hu", pConn->info, TMSG_INFO(pConn->outType), pConn->peerFqdn, pConn->peerPort); rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); pConn->pTimer = taosTmrStart(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl); } else { // close the connection - tDebug("%s, failed to send msg:%s to %s:%hu", pConn->info, taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort); + tDebug("%s, failed to send msg:%s to %s:%hu", pConn->info, TMSG_INFO(pConn->outType), pConn->peerFqdn, pConn->peerPort); if (pConn->pContext) { pConn->pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; pConn->pContext->pConn = NULL; diff --git a/source/util/test/skiplistTest.cpp b/source/util/test/skiplistTest.cpp index df4c5af5e2..f2e696b0e5 100644 --- a/source/util/test/skiplistTest.cpp +++ b/source/util/test/skiplistTest.cpp @@ -5,7 +5,7 @@ #include #include "os.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tskiplist.h" #include "tutil.h" diff --git a/src/client/inc/tscGlobalmerge.h b/src/client/inc/tscGlobalmerge.h index a462d78ff0..7aa16f843f 100644 --- a/src/client/inc/tscGlobalmerge.h +++ b/src/client/inc/tscGlobalmerge.h @@ -22,7 +22,7 @@ extern "C" { #include "qExtbuffer.h" #include "qFill.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tlosertree.h" #include "qExecutor.h" diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 653c7770ef..7dc9c0e28b 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -25,7 +25,7 @@ extern "C" { #include "../../../include/client/taos.h" #include "qAggMain.h" #include "taosdef.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tarray.h" #include "tcache.h" #include "tglobal.h" diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index dcfb2d6a87..31e60ff279 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -14,7 +14,7 @@ */ #include "os.h" -#include "taosmsg.h" +#include "tmsg.h" #include "../../../include/client/taos.h" #include "qTableMeta.h" diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 17742d71e1..ab582ec249 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -15,7 +15,7 @@ #include "os.h" #include "../../../include/client/taos.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tscLog.h" #include "tscSubquery.h" #include "tscUtil.h" diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index a9f65b84ce..e4b1602661 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -18,7 +18,7 @@ #include "tsclient.h" #include "tsocket.h" #include "ttimer.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tcq.h" #include "../../../include/client/taos.h" diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 4fc78d73e4..eb3f8be25a 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -28,7 +28,7 @@ #include "qTableMeta.h" #include "qUtil.h" #include "regex.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tcompare.h" #include "texpr.h" #include "tfilter.h" diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 7a754ee698..42799927e6 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -313,7 +313,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { char *pMsg = rpcMallocCont(pCmd->payloadLen); if (NULL == pMsg) { - tscError("0x%"PRIx64" msg:%s malloc failed", pSql->self, taosMsg[pSql->cmd.msgType]); + tscError("0x%"PRIx64" msg:%s malloc failed", pSql->self, TMSG_INFO(pSql->cmd.msgType)); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -489,7 +489,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { * There is not response callback function for submit response. * The actual inserted number of points is the first number. */ - if (rpcMsg->msgType == TSDB_MSG_TYPE_SUBMIT_RSP && pRes->pRsp != NULL) { + if (rpcMsg->msgType == TDMT_VND_SUBMIT_RSP && pRes->pRsp != NULL) { SShellSubmitRspMsg *pMsg = (SShellSubmitRspMsg*)pRes->pRsp; pMsg->code = htonl(pMsg->code); pMsg->numOfRows = htonl(pMsg->numOfRows); @@ -639,7 +639,7 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } pSql->cmd.payloadLen = sizeof(SRetrieveTableMsg); - pSql->cmd.msgType = TSDB_MSG_TYPE_FETCH; + pSql->cmd.msgType = TDMT_VND_FETCH; pRetrieveMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg)); @@ -651,7 +651,7 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; // pSql->cmd.payloadLen is set during copying data into payload - pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; + pSql->cmd.msgType = TDMT_VND_SUBMIT; SNewVgroupInfo vgroupInfo = {0}; taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); @@ -1150,7 +1150,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { tscDebug("0x%"PRIx64" msg built success, len:%d bytes", pSql->self, msgLen); pCmd->payloadLen = msgLen; - pSql->cmd.msgType = TSDB_MSG_TYPE_QUERY; + pSql->cmd.msgType = TDMT_VND_QUERY; pQueryMsg->head.contLen = htonl(msgLen); assert(msgLen + minMsgSize() <= (int32_t)pCmd->allocSize); @@ -1166,7 +1166,7 @@ int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; pCmd->payloadLen = sizeof(SCreateDbMsg); - pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_CREATE_DB : TSDB_MSG_TYPE_CREATE_TP; + pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TDMT_MND_CREATE_DB : TSDB_MSG_TYPE_CREATE_TP; SCreateDbMsg *pCreateDbMsg = (SCreateDbMsg *)pCmd->payload; @@ -1182,7 +1182,7 @@ int32_t tscBuildCreateFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; SCreateFuncMsg *pCreateFuncMsg = (SCreateFuncMsg *)pCmd->payload; - pCmd->msgType = TSDB_MSG_TYPE_CREATE_FUNCTION; + pCmd->msgType = TDMT_MND_CREATE_FUNCTION; pCmd->payloadLen = sizeof(SCreateFuncMsg) + htonl(pCreateFuncMsg->codeLen); @@ -1203,7 +1203,7 @@ int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SStrToken* t0 = taosArrayGet(pInfo->pMiscInfo->a, 0); strncpy(pCreate->ep, t0->z, t0->n); - pCmd->msgType = TSDB_MSG_TYPE_CREATE_DNODE; + pCmd->msgType = TDMT_MND_CREATE_DNODE; return TSDB_CODE_SUCCESS; } @@ -1249,7 +1249,7 @@ int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } } - pCmd->msgType = TSDB_MSG_TYPE_CREATE_ACCT; + pCmd->msgType = TDMT_MND_CREATE_ACCT; return TSDB_CODE_SUCCESS; } @@ -1277,9 +1277,9 @@ int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } if (pUser->type == TSDB_ALTER_USER_PASSWD || pUser->type == TSDB_ALTER_USER_PRIVILEGES) { - pCmd->msgType = TSDB_MSG_TYPE_ALTER_USER; + pCmd->msgType = TDMT_MND_ALTER_USER; } else { - pCmd->msgType = TSDB_MSG_TYPE_CREATE_USER; + pCmd->msgType = TDMT_MND_CREATE_USER; } return TSDB_CODE_SUCCESS; @@ -1288,7 +1288,7 @@ int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildCfgDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; pCmd->payloadLen = sizeof(SCfgDnodeMsg); - pCmd->msgType = TSDB_MSG_TYPE_CONFIG_DNODE; + pCmd->msgType = TDMT_MND_CONFIG_DNODE; return TSDB_CODE_SUCCESS; } @@ -1310,14 +1310,14 @@ int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pDropDbMsg->ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; - pCmd->msgType = (pInfo->pMiscInfo->dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_DROP_DB : TSDB_MSG_TYPE_DROP_TP; + pCmd->msgType = (pInfo->pMiscInfo->dbType == TSDB_DB_TYPE_DEFAULT) ? TDMT_MND_DROP_DB : TSDB_MSG_TYPE_DROP_TP; return TSDB_CODE_SUCCESS; } int32_t tscBuildDropFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->msgType = TSDB_MSG_TYPE_DROP_FUNCTION; + pCmd->msgType = TDMT_MND_DROP_FUNCTION; pCmd->payloadLen = sizeof(SDropFuncMsg); @@ -1340,7 +1340,7 @@ int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pDropTableMsg->supertable = (pInfo->pMiscInfo->tableType == TSDB_SUPER_TABLE)? 1:0; pDropTableMsg->igNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; - pCmd->msgType = TSDB_MSG_TYPE_DROP_TABLE; + pCmd->msgType = TDMT_MND_DROP_TABLE; return TSDB_CODE_SUCCESS; } @@ -1358,7 +1358,7 @@ int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SDropDnodeMsg * pDrop = (SDropDnodeMsg *)pCmd->payload; tstrncpy(pDrop->ep, dnodeEp, tListLen(pDrop->ep)); - pCmd->msgType = TSDB_MSG_TYPE_DROP_DNODE; + pCmd->msgType = TDMT_MND_DROP_DNODE; return TSDB_CODE_SUCCESS; } @@ -1370,7 +1370,7 @@ int32_t tscBuildDropUserAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { tstrncpy(user, pCmd->payload, TSDB_USER_LEN); pCmd->payloadLen = sizeof(SDropUserMsg); - pCmd->msgType = (pInfo->type == TSDB_SQL_DROP_USER)? TSDB_MSG_TYPE_DROP_USER:TSDB_MSG_TYPE_DROP_ACCT; + pCmd->msgType = (pInfo->type == TSDB_SQL_DROP_USER)? TDMT_MND_DROP_USER:TDMT_MND_DROP_ACCT; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); @@ -1395,7 +1395,7 @@ int32_t tscBuildUseDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SUseDbMsg *pUseDbMsg = (SUseDbMsg *)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); tNameExtractFullName(&pTableMetaInfo->name, pUseDbMsg->db); - pCmd->msgType = TSDB_MSG_TYPE_USE_DB; + pCmd->msgType = TDMT_MND_USE_DB; return TSDB_CODE_SUCCESS; } @@ -1412,14 +1412,14 @@ int32_t tscBuildSyncDbReplicaMsg(SSqlObj* pSql, SSqlInfo *pInfo) { SSyncDbMsg *pSyncMsg = (SSyncDbMsg *)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); tNameExtractFullName(&pTableMetaInfo->name, pSyncMsg->db); - pCmd->msgType = TSDB_MSG_TYPE_SYNC_DB; + pCmd->msgType = TDMT_MND_SYNC_DB; return TSDB_CODE_SUCCESS; } int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->msgType = TSDB_MSG_TYPE_SHOW; + pCmd->msgType = TDMT_MND_SHOW; pCmd->payloadLen = sizeof(SShowMsg) + 100; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { @@ -1473,10 +1473,10 @@ int32_t tscBuildKillMsg(SSqlObj *pSql, SSqlInfo *pInfo) { switch (pCmd->command) { case TSDB_SQL_KILL_QUERY: - pCmd->msgType = TSDB_MSG_TYPE_KILL_QUERY; + pCmd->msgType = TDMT_MND_KILL_QUERY; break; case TSDB_SQL_KILL_CONNECTION: - pCmd->msgType = TSDB_MSG_TYPE_KILL_CONN; + pCmd->msgType = TDMT_MND_KILL_CONN; break; case TSDB_SQL_KILL_STREAM: pCmd->msgType = TSDB_MSG_TYPE_KILL_STREAM; @@ -1592,7 +1592,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { msgLen = (int32_t)(pMsg - (char*)pCreateTableMsg); pCreateTableMsg->contLen = htonl(msgLen); pCmd->payloadLen = msgLen; - pCmd->msgType = TSDB_MSG_TYPE_CREATE_TABLE; + pCmd->msgType = TDMT_MND_CREATE_TABLE; assert(msgLen + minMsgSize() <= size); return TSDB_CODE_SUCCESS; @@ -1645,7 +1645,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { msgLen = (int32_t)(pMsg - (char*)pAlterTableMsg); pCmd->payloadLen = msgLen; - pCmd->msgType = TSDB_MSG_TYPE_ALTER_TABLE; + pCmd->msgType = TDMT_VND_ALTER_TABLE; assert(msgLen + minMsgSize() <= size); @@ -1654,7 +1654,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { SSqlCmd* pCmd = &pSql->cmd; - pCmd->msgType = TSDB_MSG_TYPE_UPDATE_TAG_VAL; + pCmd->msgType = TDMT_VND_UPDATE_TAG_VAL; SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload; pCmd->payloadLen = htonl(pUpdateMsg->head.contLen); @@ -1674,7 +1674,7 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; pCmd->payloadLen = sizeof(SAlterDbMsg); - pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_ALTER_DB : TSDB_MSG_TYPE_ALTER_TP; + pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TDMT_MND_ALTER_DB : TSDB_MSG_TYPE_ALTER_TP; SAlterDbMsg *pAlterDbMsg = (SAlterDbMsg* )pCmd->payload; pAlterDbMsg->dbType = -1; @@ -1741,7 +1741,7 @@ int tscBuildCompactMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int tscBuildRetrieveFromMgmtMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + pCmd->msgType = TDMT_MND_SHOW_RETRIEVE; pCmd->payloadLen = sizeof(SRetrieveTableMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { @@ -1861,7 +1861,7 @@ int tscProcessEmptyResultRsp(SSqlObj *pSql) { return tscLocalResultCommonBuilder int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { STscObj *pObj = pSql->pTscObj; SSqlCmd *pCmd = &pSql->cmd; - pCmd->msgType = TSDB_MSG_TYPE_CONNECT; + pCmd->msgType = TDMT_MND_CONNECT; pCmd->payloadLen = sizeof(SConnectMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { @@ -1898,7 +1898,7 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int tscBuildMultiTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->msgType = TSDB_MSG_TYPE_TABLES_META; + pCmd->msgType = TDMT_VND_TABLES_META; assert(pCmd->payloadLen + minMsgSize() <= pCmd->allocSize); tscDebug("0x%"PRIx64" build load multi-tablemeta msg completed, numOfTables:%d, msg size:%d", pSql->self, pCmd->count, @@ -1996,7 +1996,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pthread_mutex_unlock(&pObj->mutex); pCmd->payloadLen = msgLen; - pCmd->msgType = TSDB_MSG_TYPE_HEARTBEAT; + pCmd->msgType = TDMT_MND_HEARTBEAT; return TSDB_CODE_SUCCESS; } @@ -2838,7 +2838,7 @@ static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaIn } pNew->cmd.payloadLen = (int32_t)(pMsg - (char*)pInfoMsg); - pNew->cmd.msgType = TSDB_MSG_TYPE_TABLE_META; + pNew->cmd.msgType = TDMT_VND_TABLE_META; } int32_t code = tscBuildAndSendRequest(pNew, NULL); @@ -2913,7 +2913,7 @@ int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVg } pNew->cmd.payloadLen = (int32_t) ((start - pInfo->tableNames) + sizeof(SMultiTableInfoMsg)); - pNew->cmd.msgType = TSDB_MSG_TYPE_TABLES_META; + pNew->cmd.msgType = TDMT_VND_TABLES_META; registerSqlObj(pNew); tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get %d tableMeta, vgroupInfo:%d, udf:%d, msg size:%d", pSql->self, diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 9f2b79e891..c8a305e804 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -14,7 +14,7 @@ */ #include "os.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tscLog.h" #include "tscUtil.h" #include "tsched.h" diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index badc97ba35..897a629d57 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -14,7 +14,7 @@ */ #include "os.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tconfig.h" #include "tglobal.h" #include "tnote.h" diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 986ce41a9c..b7d88f94ca 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -16,7 +16,7 @@ #include "tscUtil.h" #include "hash.h" #include "os.h" -#include "taosmsg.h" +#include "tmsg.h" #include "texpr.h" #include "tkey.h" #include "tmd5.h" @@ -5011,7 +5011,7 @@ int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t lengt SSqlCmd *pCmd = &pSql->cmd; pCmd->command = TSDB_SQL_MULTI_META; - pCmd->msgType = TSDB_MSG_TYPE_TABLES_META; + pCmd->msgType = TDMT_VND_TABLES_META; int code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; char *str = (char *)pNameList; diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index d2558e4650..d610ed605f 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -22,7 +22,7 @@ #include "../../../include/client/taos.h" #include "taosdef.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tcq.h" #include "tdataformat.h" #include "tglobal.h" @@ -520,7 +520,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { pMsg->length = pMsg->header.contLen; pMsg->numOfBlocks = htonl(1); - pHead->msgType = TSDB_MSG_TYPE_SUBMIT; + pHead->msgType = TDMT_VND_SUBMIT; pHead->version = 0; // write into vnode write queue diff --git a/src/cq/test/cqtest.c b/src/cq/test/cqtest.c index b1153397ba..ee52152ad8 100644 --- a/src/cq/test/cqtest.c +++ b/src/cq/test/cqtest.c @@ -16,7 +16,7 @@ //#define _DEFAULT_SOURCE #include "os.h" #include "taosdef.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tglobal.h" #include "tlog.h" #include "tcq.h" diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 089e30ac37..65c8a45d00 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -20,7 +20,7 @@ #include #include "taosdef.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tarray.h" #include "tdataformat.h" #include "tname.h" diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 3d59dfff4e..8e01920004 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -20,7 +20,7 @@ #include "../../../include/client/taos.h" #include "os.h" #include "taosdef.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tglobal.h" #include "tsclient.h" #include "tsdb.h" diff --git a/src/query/inc/qExtbuffer.h b/src/query/inc/qExtbuffer.h index d4a9ed0cbc..1238348237 100644 --- a/src/query/inc/qExtbuffer.h +++ b/src/query/inc/qExtbuffer.h @@ -20,7 +20,7 @@ extern "C" { #endif #include "os.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tarray.h" #include "tutil.h" diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index 0c60319f39..ecb5c664aa 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -21,7 +21,7 @@ extern "C" { #endif #include "../../../include/client/taos.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tstrbuild.h" #include "ttoken.h" #include "tvariant.h" diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 037b60343e..d57532ac8f 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -15,7 +15,7 @@ #include "os.h" #include "taosdef.h" -#include "taosmsg.h" +#include "tmsg.h" #include "texpr.h" #include "ttype.h" #include "tsdb.h" diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 85f961b829..be1bfb8143 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -14,7 +14,7 @@ */ #include "os.h" #include "qFill.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tglobal.h" #include "exception.h" diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c index 35cd7eeac2..35e95a6845 100644 --- a/src/query/src/qExtbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -19,7 +19,7 @@ #include "qExecutor.h" #include "queryLog.h" #include "taosdef.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tcompare.h" #include "tulog.h" diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c index cdcc164152..47ea63eeb0 100644 --- a/src/query/src/qFill.c +++ b/src/query/src/qFill.c @@ -17,7 +17,7 @@ #include "qAggMain.h" #include "taosdef.h" -#include "taosmsg.h" +#include "tmsg.h" #include "ttype.h" #include "qFill.h" diff --git a/src/query/src/qFilterfunc.c b/src/query/src/qFilterfunc.c index 1c1ec21d65..6ad7a6ef63 100644 --- a/src/query/src/qFilterfunc.c +++ b/src/query/src/qFilterfunc.c @@ -17,7 +17,7 @@ #include "os.h" #include "qExecutor.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tcompare.h" #include "ttype.h" diff --git a/src/query/src/qHistogram.c b/src/query/src/qHistogram.c index 5fa35d0ee5..b406c0f703 100644 --- a/src/query/src/qHistogram.c +++ b/src/query/src/qHistogram.c @@ -16,7 +16,7 @@ #include "qHistogram.h" #include "taosdef.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tlosertree.h" /** diff --git a/src/query/src/qSqlParser.c b/src/query/src/qSqlParser.c index cca7bc6148..b4ad30b291 100644 --- a/src/query/src/qSqlParser.c +++ b/src/query/src/qSqlParser.c @@ -16,7 +16,7 @@ #include "qSqlparser.h" #include "os.h" #include "taosdef.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tmsgtype.h" #include "tstrbuild.h" #include "ttoken.h" diff --git a/src/query/src/qTableMeta.c b/src/query/src/qTableMeta.c index f687b8aa1f..4e8954ed6f 100644 --- a/src/query/src/qTableMeta.c +++ b/src/query/src/qTableMeta.c @@ -1,6 +1,6 @@ #include "os.h" -#include "taosmsg.h" +#include "tmsg.h" #include "qTableMeta.h" #include "ttokendef.h" #include "taosdef.h" diff --git a/src/query/src/queryMain.c b/src/query/src/queryMain.c index 1460fbdc0f..3bf031ebd5 100644 --- a/src/query/src/queryMain.c +++ b/src/query/src/queryMain.c @@ -14,7 +14,7 @@ */ #include "os.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tcache.h" #include "tglobal.h" diff --git a/src/query/tests/astTest.cpp b/src/query/tests/astTest.cpp index 75a703ac1f..b7df749983 100644 --- a/src/query/tests/astTest.cpp +++ b/src/query/tests/astTest.cpp @@ -4,7 +4,7 @@ #include #include "texpr.h" -#include "taosmsg.h" +#include "tmsg.h" #include "tsdb.h" #include "tskiplist.h" diff --git a/tests/script/general/db/basic1.sim b/tests/script/general/db/basic1.sim index 9ec1aabe98..666a9dfd9e 100644 --- a/tests/script/general/db/basic1.sim +++ b/tests/script/general/db/basic1.sim @@ -1,7 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 2000 sql connect print =============== create database @@ -15,7 +14,7 @@ if $data00 != d1 then return -1 endi -if $data02 != 0 then +if $data02 != 2 then return -1 endi @@ -51,7 +50,7 @@ if $data00 != d4 then return -1 endi -if $data02 != 0 then +if $data02 != 2 then return -1 endi diff --git a/tests/script/general/table/basic1.sim b/tests/script/general/table/basic1.sim index c26c3c33e4..ded8d79a3f 100644 --- a/tests/script/general/table/basic1.sim +++ b/tests/script/general/table/basic1.sim @@ -1,7 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 2000 sql connect print =============== create database @@ -13,29 +12,24 @@ endi print $data00 $data01 $data02 -print =============== create normal table -sql create table d1.n1 (ts timestamp, i int) -sql show d1.tables -if $rows != 1 then - return -1 -endi - -print $data00 $data01 $data02 +sql use d1 print =============== create super table -sql create table d1.st (ts timestamp, i int) tags (j int) -sql show d1.stables +sql create table st (ts timestamp, i int) tags (j int) +sql show stables if $rows != 1 then return -1 endi print $data00 $data01 $data02 +return + print =============== create child table -sql create table d1.c1 using d1.st tags(1) -sql create table d1.c2 using d1.st tags(2) -sql show d1.tables -if $rows != 3 then +sql create table c1 using st tags(1) +sql create table c2 using st tags(2) +sql show tables +if $rows != 2 then return -1 endi @@ -44,12 +38,12 @@ print $data10 $data11 $data22 print $data20 $data11 $data22 print =============== insert data -sql insert into d1.n1 values(now+1s, 1) -sql insert into d1.n1 values(now+2s, 2) -sql insert into d1.n1 values(now+3s, 3) +sql insert into c1 values(now+1s, 1) +sql insert into c1 values(now+2s, 2) +sql insert into c1 values(now+3s, 3) print =============== query data -sql select * from d1.n1 +sql select * from c1 if $rows != 3 then return -1 endi diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 2408faefa0..1cc15f731d 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -1,7 +1,11 @@ #======================b1-start=============== +# ---- user ./test.sh -f general/user/basic1.sim +# ---- db +./test.sh -f general/db/basic1.sim + #======================b1-end=============== diff --git a/tools/shell/src/backup/tnettest.c b/tools/shell/src/backup/tnettest.c index c8a6b36fc7..d6904ddc04 100644 --- a/tools/shell/src/backup/tnettest.c +++ b/tools/shell/src/backup/tnettest.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taosdef.h" -#include "taosmsg.h" +#include "tmsg.h" #include "taoserror.h" #include "tulog.h" #include "tglobal.h" @@ -358,7 +358,7 @@ static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t p epSet.port[0] = port; strcpy(epSet.fqdn[0], serverFqdn); - reqMsg.msgType = TSDB_MSG_TYPE_NETWORK_TEST; + reqMsg.msgType = TDMT_DND_NETWORK_TEST; reqMsg.pCont = rpcMallocCont(pktLen); reqMsg.contLen = pktLen; reqMsg.code = 0; @@ -368,7 +368,7 @@ static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t p rpcSendRecv(pRpcConn, &epSet, &reqMsg, &rspMsg); - if ((rspMsg.code != 0) || (rspMsg.msgType != TSDB_MSG_TYPE_NETWORK_TEST + 1)) { + if ((rspMsg.code != 0) || (rspMsg.msgType != TDMT_DND_NETWORK_TEST + 1)) { uDebug("ret code 0x%x %s", rspMsg.code, tstrerror(rspMsg.code)); return rspMsg.code; } @@ -602,7 +602,7 @@ static void taosNetCheckSpeed(char *host, int32_t port, int32_t pkgLen, epSet.port[0] = port; strcpy(epSet.fqdn[0], host); - reqMsg.msgType = TSDB_MSG_TYPE_NETWORK_TEST; + reqMsg.msgType = TDMT_DND_NETWORK_TEST; reqMsg.pCont = rpcMallocCont(pkgLen); reqMsg.contLen = pkgLen; reqMsg.code = 0; @@ -613,7 +613,7 @@ static void taosNetCheckSpeed(char *host, int32_t port, int32_t pkgLen, rpcSendRecv(pRpcConn, &epSet, &reqMsg, &rspMsg); int code = 0; - if ((rspMsg.code != 0) || (rspMsg.msgType != TSDB_MSG_TYPE_NETWORK_TEST + 1)) { + if ((rspMsg.code != 0) || (rspMsg.msgType != TDMT_DND_NETWORK_TEST + 1)) { uError("ret code 0x%x %s", rspMsg.code, tstrerror(rspMsg.code)); code = -1; }else{