diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 9cf68b87f9..f8d1fb3911 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -213,6 +213,7 @@ endif(${BUILD_WITH_TRAFT}) # LIBUV if(${BUILD_WITH_UV}) + add_compile_options(-Wno-sign-compare) if (${TD_WINDOWS}) file(READ "libuv/include/uv.h" CONTENTS) string(REGEX REPLACE "/([\r]*)\nstruct uv_tcp_s {" "/\\1\ntypedef BOOL (PASCAL *LPFN_CONNECTEX) (SOCKET s, const struct sockaddr* name, int namelen, PVOID lpSendBuffer, DWORD dwSendDataLength,LPDWORD lpdwBytesSent, LPOVERLAPPED lpOverlapped);\\1\nstruct uv_tcp_s {" CONTENTS_NEW "${CONTENTS}") diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 342d78382a..90aac6edcd 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -51,6 +51,7 @@ extern int32_t tsCompatibleModel; extern bool tsEnableSlaveQuery; extern bool tsPrintAuth; extern int64_t tsTickPerDay[3]; +extern int32_t tsMultiProcess; // monitor extern bool tsEnableMonitor; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 0a06f2174c..783f439d64 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -111,15 +111,16 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_MAX, } EShowType; -#define TSDB_ALTER_TABLE_ADD_TAG 1 -#define TSDB_ALTER_TABLE_DROP_TAG 2 -#define TSDB_ALTER_TABLE_UPDATE_TAG_NAME 3 -#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4 - +#define TSDB_ALTER_TABLE_ADD_TAG 1 +#define TSDB_ALTER_TABLE_DROP_TAG 2 +#define TSDB_ALTER_TABLE_UPDATE_TAG_NAME 3 +#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4 #define TSDB_ALTER_TABLE_ADD_COLUMN 5 #define TSDB_ALTER_TABLE_DROP_COLUMN 6 #define TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES 7 #define TSDB_ALTER_TABLE_UPDATE_TAG_BYTES 8 +#define TSDB_ALTER_TABLE_UPDATE_OPTIONS 9 +#define TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME 10 #define TSDB_FILL_NONE 0 #define TSDB_FILL_NULL 1 @@ -696,6 +697,7 @@ typedef struct { int32_t tSerializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp); int32_t tDeserializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp); +void tFreeSStatusRsp(SStatusRsp* pRsp); typedef struct { int32_t reserved; @@ -1127,10 +1129,10 @@ typedef struct { typedef struct { char name[TSDB_TOPIC_FNAME_LEN]; + char outputTbName[TSDB_TABLE_NAME_LEN]; int8_t igExists; char* sql; - char* physicalPlan; - char* logicalPlan; + char* ast; } SCMCreateStreamReq; typedef struct { @@ -1918,13 +1920,14 @@ typedef struct { } SVCreateTSmaReq; typedef struct { - int8_t type; // 0 status report, 1 update data - char indexName[TSDB_INDEX_NAME_LEN]; // - STimeWindow windows; + int8_t type; // 0 status report, 1 update data + int64_t indexUid; + int64_t skey; // start TS key of interval/sliding window } STSmaMsg; typedef struct { int64_t ver; // use a general definition + int64_t indexUid; char indexName[TSDB_INDEX_NAME_LEN]; } SVDropTSmaReq; @@ -2271,13 +2274,23 @@ enum { STREAM_TASK_STATUS__STOP, }; +typedef struct { + void* inputHandle; + void** executor; +} SStreamTaskParRunner; + typedef struct { int64_t streamId; int32_t taskId; int32_t level; int8_t status; + int8_t pipeEnd; + int8_t parallel; + SEpSet NextOpEp; char* qmsg; - void* executor; + // not applied to encoder and decoder + SStreamTaskParRunner runner; + // void* executor; // void* stateStore; // storage handle } SStreamTask; @@ -2317,6 +2330,14 @@ typedef struct { #pragma pack(pop) +struct SRpcMsg; +struct SEpSet; +struct SMgmtWrapper; +typedef int32_t (*PutToQueueFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* pReq); +typedef int32_t (*SendReqFp)(struct SMgmtWrapper* pWrapper, struct SEpSet* epSet, struct SRpcMsg* pReq); +typedef int32_t (*SendMnodeReqFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* pReq); +typedef void (*SendRspFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* pRsp); + #ifdef __cplusplus } #endif diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index a596794b3d..73a78131dc 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -190,6 +190,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp) TD_DEF_MSG_TYPE(TDMT_VND_TASK_DEPLOY, "vnode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TASK_EXEC, "vnode-task-exec", SStreamTaskExecReq, SStreamTaskExecRsp) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 167c72a778..27d509e931 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -34,128 +34,147 @@ #define TK_NK_REM 16 #define TK_NK_CONCAT 17 #define TK_CREATE 18 -#define TK_USER 19 -#define TK_PASS 20 -#define TK_NK_STRING 21 -#define TK_ALTER 22 -#define TK_PRIVILEGE 23 -#define TK_DROP 24 -#define TK_SHOW 25 -#define TK_USERS 26 -#define TK_DNODE 27 -#define TK_PORT 28 -#define TK_NK_INTEGER 29 -#define TK_DNODES 30 -#define TK_NK_ID 31 -#define TK_NK_IPTOKEN 32 -#define TK_QNODE 33 -#define TK_ON 34 -#define TK_QNODES 35 -#define TK_DATABASE 36 -#define TK_DATABASES 37 -#define TK_USE 38 -#define TK_IF 39 -#define TK_NOT 40 -#define TK_EXISTS 41 -#define TK_BLOCKS 42 -#define TK_CACHE 43 -#define TK_CACHELAST 44 -#define TK_COMP 45 -#define TK_DAYS 46 -#define TK_FSYNC 47 -#define TK_MAXROWS 48 -#define TK_MINROWS 49 -#define TK_KEEP 50 -#define TK_PRECISION 51 -#define TK_QUORUM 52 -#define TK_REPLICA 53 -#define TK_TTL 54 -#define TK_WAL 55 -#define TK_VGROUPS 56 -#define TK_SINGLE_STABLE 57 -#define TK_STREAM_MODE 58 -#define TK_TABLE 59 -#define TK_NK_LP 60 -#define TK_NK_RP 61 -#define TK_STABLE 62 -#define TK_TABLES 63 -#define TK_STABLES 64 -#define TK_USING 65 -#define TK_TAGS 66 -#define TK_NK_DOT 67 -#define TK_NK_COMMA 68 -#define TK_COMMENT 69 -#define TK_BOOL 70 -#define TK_TINYINT 71 -#define TK_SMALLINT 72 -#define TK_INT 73 -#define TK_INTEGER 74 -#define TK_BIGINT 75 -#define TK_FLOAT 76 -#define TK_DOUBLE 77 -#define TK_BINARY 78 -#define TK_TIMESTAMP 79 -#define TK_NCHAR 80 -#define TK_UNSIGNED 81 -#define TK_JSON 82 -#define TK_VARCHAR 83 -#define TK_MEDIUMBLOB 84 -#define TK_BLOB 85 -#define TK_VARBINARY 86 -#define TK_DECIMAL 87 -#define TK_SMA 88 -#define TK_INDEX 89 -#define TK_FULLTEXT 90 -#define TK_FUNCTION 91 -#define TK_INTERVAL 92 -#define TK_TOPIC 93 -#define TK_AS 94 -#define TK_MNODES 95 -#define TK_NK_FLOAT 96 -#define TK_NK_BOOL 97 -#define TK_NK_VARIABLE 98 -#define TK_BETWEEN 99 -#define TK_IS 100 -#define TK_NULL 101 -#define TK_NK_LT 102 -#define TK_NK_GT 103 -#define TK_NK_LE 104 -#define TK_NK_GE 105 -#define TK_NK_NE 106 -#define TK_NK_EQ 107 -#define TK_LIKE 108 -#define TK_MATCH 109 -#define TK_NMATCH 110 -#define TK_IN 111 -#define TK_FROM 112 -#define TK_JOIN 113 -#define TK_INNER 114 -#define TK_SELECT 115 -#define TK_DISTINCT 116 -#define TK_WHERE 117 -#define TK_PARTITION 118 -#define TK_BY 119 -#define TK_SESSION 120 -#define TK_STATE_WINDOW 121 -#define TK_SLIDING 122 -#define TK_FILL 123 -#define TK_VALUE 124 -#define TK_NONE 125 -#define TK_PREV 126 -#define TK_LINEAR 127 -#define TK_NEXT 128 -#define TK_GROUP 129 -#define TK_HAVING 130 -#define TK_ORDER 131 -#define TK_SLIMIT 132 -#define TK_SOFFSET 133 -#define TK_LIMIT 134 -#define TK_OFFSET 135 -#define TK_ASC 136 -#define TK_DESC 137 -#define TK_NULLS 138 -#define TK_FIRST 139 -#define TK_LAST 140 +#define TK_ACCOUNT 19 +#define TK_NK_ID 20 +#define TK_PASS 21 +#define TK_NK_STRING 22 +#define TK_ALTER 23 +#define TK_PPS 24 +#define TK_TSERIES 25 +#define TK_STORAGE 26 +#define TK_STREAMS 27 +#define TK_QTIME 28 +#define TK_DBS 29 +#define TK_USERS 30 +#define TK_CONNS 31 +#define TK_STATE 32 +#define TK_USER 33 +#define TK_PRIVILEGE 34 +#define TK_DROP 35 +#define TK_SHOW 36 +#define TK_DNODE 37 +#define TK_PORT 38 +#define TK_NK_INTEGER 39 +#define TK_DNODES 40 +#define TK_NK_IPTOKEN 41 +#define TK_LOCAL 42 +#define TK_QNODE 43 +#define TK_ON 44 +#define TK_QNODES 45 +#define TK_DATABASE 46 +#define TK_DATABASES 47 +#define TK_USE 48 +#define TK_IF 49 +#define TK_NOT 50 +#define TK_EXISTS 51 +#define TK_BLOCKS 52 +#define TK_CACHE 53 +#define TK_CACHELAST 54 +#define TK_COMP 55 +#define TK_DAYS 56 +#define TK_FSYNC 57 +#define TK_MAXROWS 58 +#define TK_MINROWS 59 +#define TK_KEEP 60 +#define TK_PRECISION 61 +#define TK_QUORUM 62 +#define TK_REPLICA 63 +#define TK_TTL 64 +#define TK_WAL 65 +#define TK_VGROUPS 66 +#define TK_SINGLE_STABLE 67 +#define TK_STREAM_MODE 68 +#define TK_RETENTIONS 69 +#define TK_FILE_FACTOR 70 +#define TK_NK_FLOAT 71 +#define TK_TABLE 72 +#define TK_NK_LP 73 +#define TK_NK_RP 74 +#define TK_STABLE 75 +#define TK_TABLES 76 +#define TK_STABLES 77 +#define TK_ADD 78 +#define TK_COLUMN 79 +#define TK_MODIFY 80 +#define TK_RENAME 81 +#define TK_TAG 82 +#define TK_SET 83 +#define TK_NK_EQ 84 +#define TK_USING 85 +#define TK_TAGS 86 +#define TK_NK_DOT 87 +#define TK_NK_COMMA 88 +#define TK_COMMENT 89 +#define TK_BOOL 90 +#define TK_TINYINT 91 +#define TK_SMALLINT 92 +#define TK_INT 93 +#define TK_INTEGER 94 +#define TK_BIGINT 95 +#define TK_FLOAT 96 +#define TK_DOUBLE 97 +#define TK_BINARY 98 +#define TK_TIMESTAMP 99 +#define TK_NCHAR 100 +#define TK_UNSIGNED 101 +#define TK_JSON 102 +#define TK_VARCHAR 103 +#define TK_MEDIUMBLOB 104 +#define TK_BLOB 105 +#define TK_VARBINARY 106 +#define TK_DECIMAL 107 +#define TK_SMA 108 +#define TK_ROLLUP 109 +#define TK_INDEX 110 +#define TK_FULLTEXT 111 +#define TK_FUNCTION 112 +#define TK_INTERVAL 113 +#define TK_TOPIC 114 +#define TK_AS 115 +#define TK_MNODES 116 +#define TK_NK_BOOL 117 +#define TK_NK_VARIABLE 118 +#define TK_BETWEEN 119 +#define TK_IS 120 +#define TK_NULL 121 +#define TK_NK_LT 122 +#define TK_NK_GT 123 +#define TK_NK_LE 124 +#define TK_NK_GE 125 +#define TK_NK_NE 126 +#define TK_LIKE 127 +#define TK_MATCH 128 +#define TK_NMATCH 129 +#define TK_IN 130 +#define TK_FROM 131 +#define TK_JOIN 132 +#define TK_INNER 133 +#define TK_SELECT 134 +#define TK_DISTINCT 135 +#define TK_WHERE 136 +#define TK_PARTITION 137 +#define TK_BY 138 +#define TK_SESSION 139 +#define TK_STATE_WINDOW 140 +#define TK_SLIDING 141 +#define TK_FILL 142 +#define TK_VALUE 143 +#define TK_NONE 144 +#define TK_PREV 145 +#define TK_LINEAR 146 +#define TK_NEXT 147 +#define TK_GROUP 148 +#define TK_HAVING 149 +#define TK_ORDER 150 +#define TK_SLIMIT 151 +#define TK_SOFFSET 152 +#define TK_LIMIT 153 +#define TK_OFFSET 154 +#define TK_ASC 155 +#define TK_DESC 156 +#define TK_NULLS 157 +#define TK_FIRST 158 +#define TK_LAST 159 #define TK_NK_SPACE 300 #define TK_NK_COMMENT 301 diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 5aa22bdcad..a936ea3b54 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -27,7 +27,7 @@ extern "C" { typedef int32_t VarDataOffsetT; typedef uint32_t TDRowLenT; typedef uint8_t TDRowValT; -typedef uint16_t col_id_t; +typedef int16_t col_id_t; typedef int8_t col_type_t; #pragma pack(push, 1) diff --git a/include/dnode/bnode/bnode.h b/include/dnode/bnode/bnode.h index 7f7b8a8d53..84c2ea55f7 100644 --- a/include/dnode/bnode/bnode.h +++ b/include/dnode/bnode/bnode.h @@ -21,24 +21,19 @@ extern "C" { #endif /* ------------------------ TYPES EXPOSED ------------------------ */ -typedef struct SDnode SDnode; -typedef struct SBnode SBnode; -typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg); -typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg); -typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg); +typedef struct SMgmtWrapper SMgmtWrapper; +typedef struct SBnode SBnode; typedef struct { - int64_t numOfErrors; } SBnodeLoad; typedef struct { - int32_t sver; - int32_t dnodeId; - int64_t clusterId; - SDnode *pDnode; - SendReqToDnodeFp sendReqToDnodeFp; - SendReqToMnodeFp sendReqToMnodeFp; - SendRedirectRspFp sendRedirectRspFp; + int32_t dnodeId; + int64_t clusterId; + SMgmtWrapper *pWrapper; + SendReqFp sendReqFp; + SendMnodeReqFp sendMnodeReqFp; + SendRspFp sendRspFp; } SBnodeOpt; /* ------------------------ SBnode ------------------------ */ @@ -76,13 +71,6 @@ int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad); */ int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs); -/** - * @brief Drop a bnode. - * - * @param path Path of the bnode. - */ -void bndDestroy(const char *path); - #ifdef __cplusplus } #endif diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index 068e9a6917..8d19ce23df 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -33,8 +33,7 @@ typedef struct SDnode SDnode; int32_t dndInit(); /** - * @brief clear the environment - * + * @brief Clear the environment */ void dndCleanup(); @@ -42,22 +41,24 @@ void dndCleanup(); typedef struct { int32_t numOfSupportVnodes; uint16_t serverPort; - char dataDir[TSDB_FILENAME_LEN]; + char dataDir[PATH_MAX]; char localEp[TSDB_EP_LEN]; char localFqdn[TSDB_FQDN_LEN]; char firstEp[TSDB_EP_LEN]; char secondEp[TSDB_EP_LEN]; SDiskCfg *pDisks; int32_t numOfDisks; -} SDnodeObjCfg; +} SDnodeOpt; + +typedef enum { DND_EVENT_START, DND_EVENT_STOP = 1, DND_EVENT_RELOAD } EDndEvent; /** * @brief Initialize and start the dnode. * - * @param pCfg Config of the dnode. + * @param pOption Option of the dnode. * @return SDnode* The dnode object. */ -SDnode *dndCreate(SDnodeObjCfg *pCfg); +SDnode *dndCreate(const SDnodeOpt *pOption); /** * @brief Stop and cleanup the dnode. @@ -66,6 +67,21 @@ SDnode *dndCreate(SDnodeObjCfg *pCfg); */ void dndClose(SDnode *pDnode); +/** + * @brief Run dnode until specific event is receive. + * + * @param pDnode The dnode object to run. + */ +int32_t dndRun(SDnode *pDnode); + +/** + * @brief Handle event in the dnode. + * + * @param pDnode The dnode object to close. + * @param event The event to handle. + */ +void dndHandleEvent(SDnode *pDnode, EDndEvent event); + #ifdef __cplusplus } #endif diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index d295e772e8..d421b2e45d 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -23,27 +23,21 @@ extern "C" { #endif /* ------------------------ TYPES EXPOSED ------------------------ */ -typedef struct SDnode SDnode; -typedef struct SMnode SMnode; -typedef struct SMnodeMsg SMnodeMsg; -typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); -typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); -typedef int32_t (*PutReqToMWriteQFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); -typedef int32_t (*PutReqToMReadQFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); -typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); +typedef struct SMgmtWrapper SMgmtWrapper; +typedef struct SMnode SMnode; typedef struct { - int32_t dnodeId; - int64_t clusterId; - int8_t replica; - int8_t selfIndex; - SReplica replicas[TSDB_MAX_REPLICA]; - SDnode *pDnode; - PutReqToMWriteQFp putReqToMWriteQFp; - PutReqToMReadQFp putReqToMReadQFp; - SendReqToDnodeFp sendReqToDnodeFp; - SendReqToMnodeFp sendReqToMnodeFp; - SendRedirectRspFp sendRedirectRspFp; + int32_t dnodeId; + int64_t clusterId; + int8_t replica; + int8_t selfIndex; + SReplica replicas[TSDB_MAX_REPLICA]; + SMgmtWrapper *pWrapper; + PutToQueueFp putToWriteQFp; + PutToQueueFp putToReadQFp; + SendReqFp sendReqFp; + SendMnodeReqFp sendMnodeReqFp; + SendRspFp sendRspFp; } SMnodeOpt; /* ------------------------ SMnode ------------------------ */ @@ -73,11 +67,11 @@ void mndClose(SMnode *pMnode); int32_t mndAlter(SMnode *pMnode, const SMnodeOpt *pOption); /** - * @brief Drop a mnode. + * @brief Start mnode * - * @param path Path of the mnode. + * @param pMnode The mnode object. */ -void mndDestroy(const char *path); +int32_t mndStart(SMnode *pMnode); /** * @brief Get mnode monitor info. @@ -104,37 +98,13 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr */ int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey); -/** - * @brief Initialize mnode msg. - * - * @param pMnode The mnode object. - * @param pMsg The request rpc msg. - * @return int32_t The created mnode msg. - */ -SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg); - -/** - * @brief Cleanup mnode msg. - * - * @param pMsg The request msg. - */ -void mndCleanupMsg(SMnodeMsg *pMsg); - -/** - * @brief Cleanup mnode msg. - * - * @param pMsg The request msg. - * @param code The error code. - */ -void mndSendRsp(SMnodeMsg *pMsg, int32_t code); - /** * @brief Process the read, write, sync request. * * @param pMsg The request msg. * @return int32_t 0 for success, -1 for failure. */ -void mndProcessMsg(SMnodeMsg *pMsg); +int32_t mndProcessMsg(SNodeMsg *pMsg); #ifdef __cplusplus } diff --git a/include/dnode/qnode/qnode.h b/include/dnode/qnode/qnode.h index 3de2986047..df735cf1b3 100644 --- a/include/dnode/qnode/qnode.h +++ b/include/dnode/qnode/qnode.h @@ -21,11 +21,8 @@ extern "C" { #endif /* ------------------------ TYPES EXPOSED ------------------------ */ -typedef struct SDnode SDnode; -typedef struct SQnode SQnode; -typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg); -typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg); -typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg); +typedef struct SMgmtWrapper SMgmtWrapper; +typedef struct SQnode SQnode; typedef struct { int64_t numOfStartTask; @@ -39,13 +36,12 @@ typedef struct { } SQnodeLoad; typedef struct { - int32_t sver; - int32_t dnodeId; - int64_t clusterId; - SDnode *pDnode; - SendReqToDnodeFp sendReqToDnodeFp; - SendReqToMnodeFp sendReqToMnodeFp; - SendRedirectRspFp sendRedirectRspFp; + int32_t dnodeId; + int64_t clusterId; + SMgmtWrapper *pWrapper; + SendReqFp sendReqFp; + SendMnodeReqFp sendMnodeReqFp; + SendRspFp sendRspFp; } SQnodeOpt; /* ------------------------ SQnode ------------------------ */ diff --git a/include/dnode/snode/snode.h b/include/dnode/snode/snode.h index b25f8a8666..3a93d16733 100644 --- a/include/dnode/snode/snode.h +++ b/include/dnode/snode/snode.h @@ -25,24 +25,20 @@ extern "C" { #endif /* ------------------------ TYPES EXPOSED ------------------------ */ -typedef struct SDnode SDnode; -typedef struct SSnode SSnode; -typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg); -typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg); -typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg); +typedef struct SMgmtWrapper SMgmtWrapper; +typedef struct SSnode SSnode; typedef struct { - int64_t numOfErrors; + int32_t reserved; } SSnodeLoad; typedef struct { - int32_t sver; - int32_t dnodeId; - int64_t clusterId; - SDnode *pDnode; - SendReqToDnodeFp sendReqToDnodeFp; - SendReqToMnodeFp sendReqToMnodeFp; - SendRedirectRspFp sendRedirectRspFp; + int32_t dnodeId; + int64_t clusterId; + SMgmtWrapper *pWrapper; + SendReqFp sendReqFp; + SendMnodeReqFp sendMnodeReqFp; + SendRspFp sendRspFp; } SSnodeOpt; /* ------------------------ SSnode ------------------------ */ @@ -77,20 +73,9 @@ int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad); * @param pSnode The snode object. * @param pMsg The request message * @param pRsp The response message - * @return int32_t 0 for success, -1 for failure */ -// int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp); - -int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg); - -int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg); - -/** - * @brief Drop a snode. - * - * @param path Path of the snode. - */ -void sndDestroy(const char *path); +void sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg); +void sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg); #ifdef __cplusplus } diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 5d7174fc08..5dbfd6c2c2 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -127,6 +127,18 @@ typedef struct SDropSuperTableStmt { bool ignoreNotExists; } SDropSuperTableStmt; +typedef struct SAlterTableStmt { + ENodeType type; + char dbName[TSDB_DB_NAME_LEN]; + char tableName[TSDB_TABLE_NAME_LEN]; + int8_t alterType; + char colName[TSDB_COL_NAME_LEN]; + char newColName[TSDB_COL_NAME_LEN]; + STableOptions* pOptions; + SDataType dataType; + SValueNode* pVal; +} SAlterTableStmt; + typedef struct SCreateUserStmt { ENodeType type; char useName[TSDB_USER_LEN]; @@ -158,6 +170,13 @@ typedef struct SDropDnodeStmt { int32_t port; } SDropDnodeStmt; +typedef struct SAlterDnodeStmt { + ENodeType type; + int32_t dnodeId; + char config[TSDB_DNODE_CONFIG_LEN]; + char value[TSDB_DNODE_VALUE_LEN]; +} SAlterDnodeStmt; + typedef struct SShowStmt { ENodeType type; char dbName[TSDB_DB_NAME_LEN]; @@ -215,6 +234,12 @@ typedef struct SDropTopicStmt { bool ignoreNotExists; } SDropTopicStmt; +typedef struct SAlterLocalStmt { + ENodeType type; + char config[TSDB_DNODE_CONFIG_LEN]; + char value[TSDB_DNODE_VALUE_LEN]; +} SAlterLocalStmt; + #ifdef __cplusplus } #endif diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 4ef70fc7ab..d7e67be9e0 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -85,6 +85,7 @@ typedef enum ENodeType { QUERY_NODE_DROP_TABLE_CLAUSE, QUERY_NODE_DROP_TABLE_STMT, QUERY_NODE_DROP_SUPER_TABLE_STMT, + QUERY_NODE_ALTER_TABLE_STMT, QUERY_NODE_SHOW_TABLES_STMT, // temp QUERY_NODE_SHOW_STABLES_STMT, QUERY_NODE_CREATE_USER_STMT, @@ -94,6 +95,7 @@ typedef enum ENodeType { QUERY_NODE_USE_DATABASE_STMT, QUERY_NODE_CREATE_DNODE_STMT, QUERY_NODE_DROP_DNODE_STMT, + QUERY_NODE_ALTER_DNODE_STMT, QUERY_NODE_SHOW_DNODES_STMT, QUERY_NODE_SHOW_VGROUPS_STMT, QUERY_NODE_SHOW_MNODES_STMT, @@ -104,6 +106,7 @@ typedef enum ENodeType { QUERY_NODE_DROP_QNODE_STMT, QUERY_NODE_CREATE_TOPIC_STMT, QUERY_NODE_DROP_TOPIC_STMT, + QUERY_NODE_ALTER_LOCAL_STMT, // logic plan node QUERY_NODE_LOGIC_PLAN_SCAN, @@ -163,6 +166,7 @@ SNodeList* nodesMakeList(); int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode); int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode); int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc); +int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc); SListCell* nodesListErase(SNodeList* pList, SListCell* pCell); SNodeptr nodesListGetNode(SNodeList* pList, int32_t index); void nodesDestroyList(SNodeList* pList); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 1d89b07539..cbc1f6c1f8 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -26,7 +26,6 @@ extern "C" { typedef struct SLogicNode { ENodeType type; - int32_t id; SNodeList* pTargets; // SColumnNode SNode* pConditions; SNodeList* pChildren; @@ -37,6 +36,7 @@ typedef enum EScanType { SCAN_TYPE_TAG, SCAN_TYPE_TABLE, SCAN_TYPE_STABLE, + SCAN_TYPE_TOPIC, SCAN_TYPE_STREAM } EScanType; @@ -155,7 +155,7 @@ typedef struct SPhysiNode { } SPhysiNode; typedef struct SScanPhysiNode { - SPhysiNode node; + SPhysiNode node; SNodeList* pScanCols; uint64_t uid; // unique id of the table int8_t tableType; @@ -167,6 +167,7 @@ typedef struct SScanPhysiNode { typedef SScanPhysiNode SSystemTableScanPhysiNode; typedef SScanPhysiNode STagScanPhysiNode; +typedef SScanPhysiNode SStreamScanPhysiNode; typedef struct STableScanPhysiNode { SScanPhysiNode scan; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index fe44d8666b..2c733454dc 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -23,6 +23,10 @@ extern "C" { #include "nodes.h" #include "tmsg.h" +#define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags) +#define TABLE_META_SIZE(pMeta) (NULL == (pMeta) ? 0 : (sizeof(STableMeta) + TABLE_TOTAL_COL_NUM((pMeta)) * sizeof(SSchema))) +#define VGROUPS_INFO_SIZE(pInfo) (NULL == (pInfo) ? 0 : (sizeof(SVgroupsInfo) + (pInfo)->numOfVgroups * sizeof(SVgroupInfo))) + typedef struct SRawExprNode { ENodeType nodeType; char* p; diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 77838c705f..74a15e2d18 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -26,6 +26,7 @@ typedef struct SParseContext { uint64_t requestId; int32_t acctId; const char *db; + bool topicQuery; void *pTransporter; SEpSet mgmtEpSet; const char *pSql; // sql string diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 7eb9d038a5..4ba04d1713 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -26,6 +26,7 @@ typedef struct SPlanContext { uint64_t queryId; int32_t acctId; SNode* pAstRoot; + bool topicQuery; bool streamQuery; } SPlanContext; diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index dd3103edf1..8954cbaa0e 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -49,10 +49,10 @@ typedef struct { } SQWorkerStat; typedef int32_t (*putReqToQueryQFp)(void *, struct SRpcMsg *); -typedef int32_t (*sendReqToDnodeFp)(void *, struct SEpSet *, struct SRpcMsg *); +typedef int32_t (*sendReqFp)(void *, struct SEpSet *, struct SRpcMsg *); int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, - putReqToQueryQFp fp1, sendReqToDnodeFp fp2); + putReqToQueryQFp fp1, sendReqFp fp2); int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 6bc28e3ea0..02029a996c 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -38,16 +38,24 @@ typedef struct SRpcConnInfo { typedef struct SRpcMsg { tmsg_t msgType; - tmsg_t expectMsgType; void * pCont; int contLen; int32_t code; - void * handle; // rpc handle returned to app - void * ahandle; // app handle set by client - int noResp; // has response or not(default 0 indicate resp); + void * handle; // rpc handle returned to app + void * ahandle; // app handle set by client + int noResp; // has response or not(default 0, 0: resp, 1: no resp); + int persistHandle; // persist handle or not } SRpcMsg; +typedef struct { + char user[TSDB_USER_LEN]; + SRpcMsg rpcMsg; + int32_t rspLen; + void *pRsp; + void *pNode; +} SNodeMsg; + typedef struct SRpcInit { uint16_t localPort; // local port char * label; // for debug purpose @@ -69,18 +77,19 @@ typedef struct SRpcInit { // call back to retrieve the client auth info, for server app only int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); - // call back to keep conn or not - bool (*pfp)(void *parent, tmsg_t msgType); - - // to support Send messages multiple times on a link - void *(*mfp)(void *parent, tmsg_t msgType); - - // call back to handle except when query/fetch in progress - bool (*efp)(void *parent, tmsg_t msgType); - void *parent; } SRpcInit; +typedef struct { + void * val; + int32_t len; + void (*free)(void *arg); +} SRpcCtxVal; + +typedef struct { + SHashObj *args; +} SRpcCtx; + int32_t rpcInit(); void rpcCleanup(); void * rpcOpen(const SRpcInit *pRpc); @@ -89,16 +98,17 @@ void * rpcMallocCont(int contLen); void rpcFreeCont(void *pCont); void * rpcReallocCont(void *ptr, int contLen); void rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid); -void rpcSendResponse(const SRpcMsg *pMsg); -void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet); -int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); -void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); -int rpcReportProgress(void *pConn, char *pCont, int contLen); -void rpcCancelRequest(int64_t rid); +void rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx); +void rpcSendResponse(const SRpcMsg *pMsg); +void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet); +int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); +void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); +int rpcReportProgress(void *pConn, char *pCont, int contLen); +void rpcCancelRequest(int64_t rid); +void rpcRegisterBrokenLinkArg(SRpcMsg *msg); // just release client conn to rpc instance, no close sock -void rpcReleaseHandle(void *handle, int8_t type); - +void rpcReleaseHandle(void *handle, int8_t type); // void rpcRefHandle(void *handle, int8_t type); void rpcUnrefHandle(void *handle, int8_t type); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 32fcd95948..7342af4d18 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -127,7 +127,7 @@ typedef struct SWal { int64_t lastRollSeq; // ctl int64_t refId; - pthread_mutex_t mutex; + TdThreadMutex mutex; // path char path[WAL_PATH_LEN]; // reusable write head diff --git a/include/os/os.h b/include/os/os.h index 9118a92e11..b05bfab6d0 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -22,7 +22,6 @@ extern "C" { #include #include -#include #include #include @@ -83,6 +82,7 @@ extern "C" { #include "osMath.h" #include "osMemory.h" #include "osRand.h" +#include "osThread.h" #include "osSemaphore.h" #include "osSignal.h" #include "osSleep.h" @@ -90,7 +90,6 @@ extern "C" { #include "osString.h" #include "osSysinfo.h" #include "osSystem.h" -#include "osThread.h" #include "osTime.h" #include "osTimer.h" #include "osTimezone.h" diff --git a/include/os/osEnv.h b/include/os/osEnv.h index 1426ba87f6..ebf4c360dd 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -45,6 +45,7 @@ extern SDiskSpace tsTempSpace; void osInit(); void osUpdate(); +void osCleanup(); bool osLogSpaceAvailable(); void osSetTimezone(const char *timezone); diff --git a/include/os/osSemaphore.h b/include/os/osSemaphore.h index 4bac81754d..594daf1bf3 100644 --- a/include/os/osSemaphore.h +++ b/include/os/osSemaphore.h @@ -20,7 +20,6 @@ extern "C" { #endif -#include #include #if defined (_TD_DARWIN_64) @@ -38,25 +37,25 @@ extern "C" { #endif #if defined (_TD_DARWIN_64) -// #define pthread_rwlock_t pthread_mutex_t -// #define pthread_rwlock_init(lock, NULL) pthread_mutex_init(lock, NULL) -// #define pthread_rwlock_destroy(lock) pthread_mutex_destroy(lock) -// #define pthread_rwlock_wrlock(lock) pthread_mutex_lock(lock) -// #define pthread_rwlock_rdlock(lock) pthread_mutex_lock(lock) -// #define pthread_rwlock_unlock(lock) pthread_mutex_unlock(lock) +// #define TdThreadRwlock TdThreadMutex +// #define taosThreadRwlockInit(lock, NULL) taosThreadMutexInit(lock, NULL) +// #define taosThreadRwlockDestroy(lock) taosThreadMutexDestroy(lock) +// #define taosThreadRwlockWrlock(lock) taosThreadMutexLock(lock) +// #define taosThreadRwlockRdlock(lock) taosThreadMutexLock(lock) +// #define taosThreadRwlockUnlock(lock) taosThreadMutexUnlock(lock) - #define pthread_spinlock_t pthread_mutex_t - #define pthread_spin_init(lock, NULL) pthread_mutex_init(lock, NULL) - #define pthread_spin_destroy(lock) pthread_mutex_destroy(lock) - #define pthread_spin_lock(lock) pthread_mutex_lock(lock) - #define pthread_spin_unlock(lock) pthread_mutex_unlock(lock) + #define TdThreadSpinlock TdThreadMutex + #define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL) + #define taosThreadSpinDestroy(lock) taosThreadMutexDestroy(lock) + #define taosThreadSpinLock(lock) taosThreadMutexLock(lock) + #define taosThreadSpinUnlock(lock) taosThreadMutexUnlock(lock) #endif -bool taosCheckPthreadValid(pthread_t thread); +bool taosCheckPthreadValid(TdThread thread); int64_t taosGetSelfPthreadId(); -int64_t taosGetPthreadId(pthread_t thread); -void taosResetPthread(pthread_t* thread); -bool taosComparePthread(pthread_t first, pthread_t second); +int64_t taosGetPthreadId(TdThread thread); +void taosResetPthread(TdThread* thread); +bool taosComparePthread(TdThread first, TdThread second); int32_t taosGetPId(); int32_t taosGetAppName(char* name, int32_t* len); diff --git a/include/os/osString.h b/include/os/osString.h index 9c6d523ab2..5846d6b7ed 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -35,7 +35,7 @@ typedef int32_t TdUcs4; #define wctomb WCTOMB_FUNC_TAOS_FORBID #define wcstombs WCSTOMBS_FUNC_TAOS_FORBID #define wcsncpy WCSNCPY_FUNC_TAOS_FORBID - #define wchar_t WCHAR_T_FUNC_TAOS_FORBID + #define wchar_t WCHAR_T_TYPE_TAOS_FORBID #endif #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) diff --git a/include/os/osThread.h b/include/os/osThread.h index cccc13755d..504814c1c6 100644 --- a/include/os/osThread.h +++ b/include/os/osThread.h @@ -22,6 +22,94 @@ extern "C" { #endif +typedef pthread_t TdThread; +typedef pthread_spinlock_t TdThreadSpinlock; +typedef pthread_mutex_t TdThreadMutex; +typedef pthread_mutexattr_t TdThreadMutexAttr; +typedef pthread_rwlock_t TdThreadRwlock; +typedef pthread_attr_t TdThreadAttr; +typedef pthread_once_t TdThreadOnce; +typedef pthread_rwlockattr_t TdThreadRwlockAttr; +typedef pthread_cond_t TdThreadCond; +typedef pthread_condattr_t TdThreadCondAttr; + +#define taosThreadCleanupPush pthread_cleanup_push +#define taosThreadCleanupPop pthread_cleanup_pop + +// If the error is in a third-party library, place this header file under the third-party library header file. +#ifndef ALLOW_FORBID_FUNC + #define pthread_t PTHREAD_T_TYPE_TAOS_FORBID + #define pthread_spinlock_t PTHREAD_SPINLOCK_T_TYPE_TAOS_FORBID + #define pthread_mutex_t PTHREAD_MUTEX_T_TYPE_TAOS_FORBID + #define pthread_mutexattr_t PTHREAD_MUTEXATTR_T_TYPE_TAOS_FORBID + #define pthread_rwlock_t PTHREAD_RWLOCK_T_TYPE_TAOS_FORBID + #define pthread_attr_t PTHREAD_ATTR_T_TYPE_TAOS_FORBID + #define pthread_once_t PTHREAD_ONCE_T_TYPE_TAOS_FORBID + #define pthread_rwlockattr_t PTHREAD_RWLOCKATTR_T_TYPE_TAOS_FORBID + #define pthread_cond_t PTHREAD_COND_T_TYPE_TAOS_FORBID + #define pthread_condattr_t PTHREAD_CONDATTR_T_TYPE_TAOS_FORBID + #define pthread_spin_init PTHREAD_SPIN_INIT_FUNC_TAOS_FORBID + #define pthread_mutex_init PTHREAD_MUTEX_INIT_FUNC_TAOS_FORBID + #define pthread_spin_destroy PTHREAD_SPIN_DESTROY_FUNC_TAOS_FORBID + #define pthread_mutex_destroy PTHREAD_MUTEX_DESTROY_FUNC_TAOS_FORBID + #define pthread_spin_lock PTHREAD_SPIN_LOCK_FUNC_TAOS_FORBID + #define pthread_mutex_lock PTHREAD_MUTEX_LOCK_FUNC_TAOS_FORBID + #define pthread_spin_unlock PTHREAD_SPIN_UNLOCK_FUNC_TAOS_FORBID + #define pthread_mutex_unlock PTHREAD_MUTEX_UNLOCK_FUNC_TAOS_FORBID + #define pthread_rwlock_rdlock PTHREAD_RWLOCK_RDLOCK_FUNC_TAOS_FORBID + #define pthread_rwlock_wrlock PTHREAD_RWLOCK_WRLOCK_FUNC_TAOS_FORBID + #define pthread_rwlock_unlock PTHREAD_RWLOCK_UNLOCK_FUNC_TAOS_FORBID + #define pthread_testcancel PTHREAD_TESTCANCEL_FUNC_TAOS_FORBID + #define pthread_attr_init PTHREAD_ATTR_INIT_FUNC_TAOS_FORBID + #define pthread_create PTHREAD_CREATE_FUNC_TAOS_FORBID + #define pthread_once PTHREAD_ONCE_FUNC_TAOS_FORBID + #define pthread_attr_setdetachstate PTHREAD_ATTR_SETDETACHSTATE_FUNC_TAOS_FORBID + #define pthread_attr_destroy PTHREAD_ATTR_DESTROY_FUNC_TAOS_FORBID + #define pthread_join PTHREAD_JOIN_FUNC_TAOS_FORBID + #define pthread_rwlock_init PTHREAD_RWLOCK_INIT_FUNC_TAOS_FORBID + #define pthread_rwlock_destroy PTHREAD_RWLOCK_DESTROY_FUNC_TAOS_FORBID + #define pthread_cond_signal PTHREAD_COND_SIGNAL_FUNC_TAOS_FORBID + #define pthread_cond_init PTHREAD_COND_INIT_FUNC_TAOS_FORBID + #define pthread_cond_broadcast PTHREAD_COND_BROADCAST_FUNC_TAOS_FORBID + #define pthread_cond_destroy PTHREAD_COND_DESTROY_FUNC_TAOS_FORBID + #define pthread_cond_wait PTHREAD_COND_WAIT_FUNC_TAOS_FORBID + #define pthread_self PTHREAD_SELF_FUNC_TAOS_FORBID + #define pthread_equal PTHREAD_EQUAL_FUNC_TAOS_FORBID + #define pthread_sigmask PTHREAD_SIGMASK_FUNC_TAOS_FORBID + #define pthread_cancel PTHREAD_CANCEL_FUNC_TAOS_FORBID + #define pthread_kill PTHREAD_KILL_FUNC_TAOS_FORBID +#endif + +int32_t taosThreadSpinInit(TdThreadSpinlock *lock, int pshared); +int32_t taosThreadMutexInit(TdThreadMutex *mutex, const TdThreadMutexAttr *attr); +int32_t taosThreadSpinDestroy(TdThreadSpinlock *lock); +int32_t taosThreadMutexDestroy(TdThreadMutex * mutex); +int32_t taosThreadSpinLock(TdThreadSpinlock *lock); +int32_t taosThreadMutexLock(TdThreadMutex *mutex); +int32_t taosThreadRwlockRdlock(TdThreadRwlock *rwlock); +int32_t taosThreadSpinUnlock(TdThreadSpinlock *lock); +int32_t taosThreadMutexUnlock(TdThreadMutex *mutex); +int32_t taosThreadRwlockWrlock(TdThreadRwlock *rwlock); +int32_t taosThreadRwlockUnlock(TdThreadRwlock *rwlock); +void taosThreadTestCancel(void); +int32_t taosThreadAttrInit(TdThreadAttr *attr); +int32_t taosThreadCreate(TdThread *tid, const TdThreadAttr *attr, void*(*start)(void*), void *arg); +int32_t taosThreadOnce(TdThreadOnce *onceControl, void(*initRoutine)(void)); +int32_t taosThreadAttrSetDetachState(TdThreadAttr *attr, int32_t detachState); +int32_t taosThreadAttrDestroy(TdThreadAttr *attr); +int32_t taosThreadJoin(TdThread thread, void **pValue); +int32_t taosThreadRwlockInit(TdThreadRwlock *rwlock, const TdThreadRwlockAttr *attr); +int32_t taosThreadRwlockDestroy(TdThreadRwlock *rwlock); +int32_t taosThreadCondSignal(TdThreadCond *cond); +int32_t taosThreadCondInit(TdThreadCond *cond, const TdThreadCondAttr *attr); +int32_t taosThreadCondBroadcast(TdThreadCond *cond); +int32_t taosThreadCondDestroy(TdThreadCond *cond); +int32_t taosThreadCondWait(TdThreadCond *cond, TdThreadMutex *mutex); +TdThread taosThreadSelf(void); +int32_t taosThreadEqual(TdThread t1, TdThread t2); +int32_t taosThreadSigmask(int how, sigset_t const *set, sigset_t *oset); +int32_t taosThreadCancel(TdThread thread); +int32_t taosThreadKill(TdThread thread, int sig); #ifdef __cplusplus } #endif diff --git a/include/os/osTimezone.h b/include/os/osTimezone.h index ff015ef0b1..fa0e70f209 100644 --- a/include/os/osTimezone.h +++ b/include/os/osTimezone.h @@ -20,6 +20,11 @@ extern "C" { #endif +// If the error is in a third-party library, place this header file under the third-party library header file. +#ifndef ALLOW_FORBID_FUNC + #define tzset TZSET_FUNC_TAOS_FORBID +#endif + void taosGetSystemTimezone(char *outTimezone); void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index fe06a1c4a2..9f95848865 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -75,6 +75,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x010B) #define TSDB_CODE_CFG_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x010C) #define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x010D) +#define TSDB_CODE_OUT_OF_SHM_MEM TAOS_DEF_ERROR_CODE(0, 0x010E) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0110) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0111) #define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0112) @@ -277,34 +278,14 @@ int32_t* taosGetErrno(); #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) #define TSDB_CODE_DND_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0401) #define TSDB_CODE_DND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0402) -#define TSDB_CODE_DND_DNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0410) -#define TSDB_CODE_DND_DNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0411) -#define TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0420) -#define TSDB_CODE_DND_MNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0421) -#define TSDB_CODE_DND_MNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0422) -#define TSDB_CODE_DND_MNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0423) -#define TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0424) -#define TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0430) -#define TSDB_CODE_DND_QNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0431) -#define TSDB_CODE_DND_QNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0432) -#define TSDB_CODE_DND_QNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0433) -#define TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0434) -#define TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0440) -#define TSDB_CODE_DND_SNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0441) -#define TSDB_CODE_DND_SNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0442) -#define TSDB_CODE_DND_SNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0443) -#define TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0444) -#define TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0450) -#define TSDB_CODE_DND_BNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0451) -#define TSDB_CODE_DND_BNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0452) -#define TSDB_CODE_DND_BNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0453) -#define TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0454) -#define TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0460) -#define TSDB_CODE_DND_VNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0461) -#define TSDB_CODE_DND_VNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0462) -#define TSDB_CODE_DND_VNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0463) -#define TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0464) -#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0465) +#define TSDB_CODE_NODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0403) +#define TSDB_CODE_NODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0404) +#define TSDB_CODE_NODE_PARSE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0405) +#define TSDB_CODE_NODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0406) +#define TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0410) +#define TSDB_CODE_DND_VNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0411) +#define TSDB_CODE_DND_VNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0412) +#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0413) // vnode #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) @@ -482,6 +463,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_PASSWD_EMPTY TAOS_DEF_ERROR_CODE(0, 0x2611) #define TSDB_CODE_PAR_INVALID_PORT TAOS_DEF_ERROR_CODE(0, 0x2612) #define TSDB_CODE_PAR_INVALID_ENDPOINT TAOS_DEF_ERROR_CODE(0, 0x2613) +#define TSDB_CODE_PAR_EXPRIE_STATEMENT TAOS_DEF_ERROR_CODE(0, 0x2614) #ifdef __cplusplus } diff --git a/include/util/tidpool.h b/include/util/tidpool.h index 8596b439e3..c97f4c5f38 100644 --- a/include/util/tidpool.h +++ b/include/util/tidpool.h @@ -27,7 +27,7 @@ typedef struct { int32_t numOfFree; int32_t freeSlot; bool *freeList; - pthread_mutex_t mutex; + TdThreadMutex mutex; } id_pool_t; void *taosInitIdPool(int32_t maxId); diff --git a/include/util/tjson.h b/include/util/tjson.h index 1218caae35..6b2221f704 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -22,6 +22,14 @@ extern "C" { #endif +#define tjsonGetNumberValue(pJson, pName, val) \ + ({ \ + uint64_t _tmp = 0; \ + int32_t _code = tjsonGetUBigIntValue(pJson, pName, &_tmp); \ + val = _tmp; \ + _code; \ + }) + typedef void SJson; SJson* tjsonCreateObject(); @@ -44,6 +52,7 @@ int32_t tjsonGetIntValue(const SJson* pJson, const char* pName, int32_t* pVal); int32_t tjsonGetSmallIntValue(const SJson* pJson, const char* pName, int16_t* pVal); int32_t tjsonGetTinyIntValue(const SJson* pJson, const char* pName, int8_t* pVal); int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pVal); +int32_t tjsonGetUIntValue(const SJson* pJson, const char* pName, uint32_t* pVal); int32_t tjsonGetUTinyIntValue(const SJson* pJson, const char* pName, uint8_t* pVal); int32_t tjsonGetBoolValue(const SJson* pJson, const char* pName, bool* pVal); int32_t tjsonGetDoubleValue(const SJson* pJson, const char* pName, double* pVal); @@ -60,6 +69,7 @@ int32_t tjsonAddArray(SJson* pJson, const char* pName, FToJson func, const void* typedef int32_t (*FToObject)(const SJson* pJson, void* pObj); int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, void* pObj); +int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, void** pObj, int32_t objSize); int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void* pArray, int32_t itemSize); char* tjsonToString(const SJson* pJson); diff --git a/include/util/tprocess.h b/include/util/tprocess.h new file mode 100644 index 0000000000..4ce536fd96 --- /dev/null +++ b/include/util/tprocess.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_PROCESS_H_ +#define _TD_UTIL_PROCESS_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SProcQueue SProcQueue; +typedef struct SProcObj SProcObj; +typedef void *(*ProcMallocFp)(int32_t contLen); +typedef void *(*ProcFreeFp)(void *pCont); +typedef void *(*ProcConsumeFp)(void *pParent, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen); + +typedef struct { + int32_t childQueueSize; + ProcConsumeFp childConsumeFp; + ProcMallocFp childMallocHeadFp; + ProcFreeFp childFreeHeadFp; + ProcMallocFp childMallocBodyFp; + ProcFreeFp childFreeBodyFp; + int32_t parentQueueSize; + ProcConsumeFp parentConsumeFp; + ProcMallocFp parentdMallocHeadFp; + ProcFreeFp parentFreeHeadFp; + ProcMallocFp parentMallocBodyFp; + ProcFreeFp parentFreeBodyFp; + bool testFlag; + void *pParent; + const char *name; +} SProcCfg; + +SProcObj *taosProcInit(const SProcCfg *pCfg); +void taosProcCleanup(SProcObj *pProc); +int32_t taosProcRun(SProcObj *pProc); +void taosProcStop(SProcObj *pProc); +bool taosProcIsChild(SProcObj *pProc); + +int32_t taosProcPutToChildQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen); +int32_t taosProcPutToParentQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_PROCESS_H_*/ diff --git a/include/util/tskiplist.h b/include/util/tskiplist.h index 64cab08cfe..a2382ad541 100644 --- a/include/util/tskiplist.h +++ b/include/util/tskiplist.h @@ -57,7 +57,7 @@ typedef struct SSkipListNode { * @date 2017/11/12 * the simple version of skip list. * - * for multi-thread safe purpose, we employ pthread_rwlock_t to guarantee to generate + * for multi-thread safe purpose, we employ TdThreadRwlock to guarantee to generate * deterministic result. Later, we will remove the lock in SkipList to further enhance the performance. * In this case, one should use the concurrent skip list (by using michael-scott algorithm) instead of * this simple version in a multi-thread environment, to achieve higher performance of read/write operations. @@ -106,7 +106,7 @@ typedef struct SSkipList { uint32_t seed; __compar_fn_t comparFn; __sl_key_fn_t keyFn; - pthread_rwlock_t *lock; + TdThreadRwlock *lock; uint16_t len; uint8_t maxLevel; uint8_t flags; diff --git a/include/util/tthread.h b/include/util/tthread.h index 4941206944..2215add062 100644 --- a/include/util/tthread.h +++ b/include/util/tthread.h @@ -22,9 +22,11 @@ extern "C" { #endif -pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param); -bool taosDestoryThread(pthread_t* pthread); -bool taosThreadRunning(pthread_t* pthread); +TdThread* taosCreateThread(void* (*__start_routine)(void*), void* param); +bool taosDestoryThread(TdThread* pthread); +bool taosThreadRunning(TdThread* pthread); + +typedef void *(*ThreadFp)(void *param); #ifdef __cplusplus } diff --git a/include/util/tworker.h b/include/util/tworker.h index e6f6bc077c..dabd1ac9b3 100644 --- a/include/util/tworker.h +++ b/include/util/tworker.h @@ -27,7 +27,7 @@ typedef struct SWWorkerPool SWWorkerPool; typedef struct SQWorker { int32_t id; // worker ID - pthread_t thread; // thread + TdThread thread; // thread SQWorkerPool *pool; } SQWorker, SFWorker; @@ -38,12 +38,12 @@ typedef struct SQWorkerPool { STaosQset *qset; const char *name; SQWorker *workers; - pthread_mutex_t mutex; + TdThreadMutex mutex; } SQWorkerPool, SFWorkerPool; typedef struct SWWorker { int32_t id; // worker id - pthread_t thread; // thread + TdThread thread; // thread STaosQall *qall; STaosQset *qset; // queue set SWWorkerPool *pool; @@ -54,7 +54,7 @@ typedef struct SWWorkerPool { int32_t nextId; // from 0 to max-1, cyclic const char *name; SWWorker *workers; - pthread_mutex_t mutex; + TdThreadMutex mutex; } SWWorkerPool; int32_t tQWorkerInit(SQWorkerPool *pool); diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index b672c9bb47..d713b11a92 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -77,8 +77,8 @@ typedef struct { int8_t inited; // ctl int8_t threadStop; - pthread_t thread; - pthread_mutex_t lock; // used when app init and cleanup + TdThread thread; + TdThreadMutex lock; // used when app init and cleanup SArray* appHbMgrs; // SArray one for each cluster FHbReqHandle reqHandle[HEARTBEAT_TYPE_MAX]; FHbRspHandle rspHandle[HEARTBEAT_TYPE_MAX]; @@ -125,7 +125,7 @@ typedef struct SAppInfo { int32_t pid; int32_t numOfThreads; SHashObj* pInstMap; - pthread_mutex_t mutex; + TdThreadMutex mutex; } SAppInfo; typedef struct STscObj { @@ -137,7 +137,7 @@ typedef struct STscObj { uint32_t connId; int32_t connType; uint64_t id; // ref ID returned by taosAddRef - pthread_mutex_t mutex; // used to protect the operation on db + TdThreadMutex mutex; // used to protect the operation on db int32_t numOfReqs; // number of sqlObj bound to this connection SAppInstInfo* pAppInfo; } STscObj; @@ -179,6 +179,7 @@ typedef struct SRequestObj { uint64_t requestId; int32_t type; // request type STscObj* pTscObj; + char* pDb; char* sqlstr; // sql string int32_t sqlLen; int64_t self; @@ -229,7 +230,7 @@ void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest); -int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery); +int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery); int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList); // --- heartbeat diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 525c5f9fb8..07f925bce8 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -33,7 +33,7 @@ SAppInfo appInfo; int32_t clientReqRefPool = -1; int32_t clientConnRefPool = -1; -static pthread_once_t tscinit = PTHREAD_ONCE_INIT; +static TdThreadOnce tscinit = PTHREAD_ONCE_INIT; volatile int32_t tscInitRes = 0; static void registerRequest(SRequestObj *pRequest) { @@ -90,7 +90,6 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.label = "TSC"; rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; - rpcInit.pfp = persistConnForSpecificMsg; rpcInit.sessions = tsMaxConnections; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; @@ -115,7 +114,7 @@ void destroyTscObj(void *pObj) { hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey); atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); - pthread_mutex_destroy(&pTscObj->mutex); + taosThreadMutexDestroy(&pTscObj->mutex); tfree(pTscObj); } @@ -134,7 +133,7 @@ void *createTscObj(const char *user, const char *auth, const char *db, SAppInstI tstrncpy(pObj->db, db, tListLen(pObj->db)); } - pthread_mutex_init(&pObj->mutex, NULL); + taosThreadMutexInit(&pObj->mutex, NULL); pObj->id = taosAddRef(clientConnRefPool, pObj); tscDebug("connObj created, 0x%" PRIx64, pObj->id); @@ -150,6 +149,7 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty return NULL; } + pRequest->pDb = getDbOfConnection(pObj); pRequest->requestId = generateRequestId(); pRequest->metric.start = taosGetTimestampMs(); @@ -180,6 +180,7 @@ static void doDestroyRequest(void *p) { tfree(pRequest->msgBuf); tfree(pRequest->sqlstr); tfree(pRequest->pInfo); + tfree(pRequest->pDb); doFreeReqResultInfo(&pRequest->body.resInfo); qDestroyQueryPlan(pRequest->body.pDag); @@ -241,7 +242,7 @@ void taos_init_imp(void) { // transDestroyBuffer(&conn->readBuf); taosGetAppName(appInfo.appName, NULL); - pthread_mutex_init(&appInfo.mutex, NULL); + taosThreadMutexInit(&appInfo.mutex, NULL); appInfo.pid = taosGetPId(); appInfo.startTime = taosGetTimestampMs(); @@ -250,7 +251,7 @@ void taos_init_imp(void) { } int taos_init() { - pthread_once(&tscinit, taos_init_imp); + taosThreadOnce(&tscinit, taos_init_imp); return tscInitRes; } @@ -506,9 +507,9 @@ static setConfRet taos_set_config_imp(const char *config){ } setConfRet taos_set_config(const char *config){ - pthread_mutex_lock(&setConfMutex); + taosThreadMutexLock(&setConfMutex); setConfRet ret = taos_set_config_imp(config); - pthread_mutex_unlock(&setConfMutex); + taosThreadMutexUnlock(&setConfMutex); return ret; } #endif diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 7163444719..479281eec6 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -372,7 +372,7 @@ static void *hbThreadFunc(void *param) { break; } - pthread_mutex_lock(&clientHbMgr.lock); + taosThreadMutexLock(&clientHbMgr.lock); int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for (int i = 0; i < sz; i++) { @@ -423,7 +423,7 @@ static void *hbThreadFunc(void *param) { atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); } - pthread_mutex_unlock(&clientHbMgr.lock); + taosThreadMutexUnlock(&clientHbMgr.lock); taosMsleep(HEARTBEAT_INTERVAL); } @@ -431,15 +431,15 @@ static void *hbThreadFunc(void *param) { } static int32_t hbCreateThread() { - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); -// if (pthread_create(&clientHbMgr.thread, &thAttr, hbThreadFunc, NULL) != 0) { +// if (taosThreadCreate(&clientHbMgr.thread, &thAttr, hbThreadFunc, NULL) != 0) { // terrno = TAOS_SYSTEM_ERROR(errno); // return -1; // } -// pthread_attr_destroy(&thAttr); +// taosThreadAttrDestroy(&thAttr); return 0; } @@ -492,15 +492,15 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { return NULL; } - pthread_mutex_lock(&clientHbMgr.lock); + taosThreadMutexLock(&clientHbMgr.lock); taosArrayPush(clientHbMgr.appHbMgrs, &pAppHbMgr); - pthread_mutex_unlock(&clientHbMgr.lock); + taosThreadMutexUnlock(&clientHbMgr.lock); return pAppHbMgr; } void appHbMgrCleanup(void) { - pthread_mutex_lock(&clientHbMgr.lock); + taosThreadMutexLock(&clientHbMgr.lock); int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for (int i = 0; i < sz; i++) { @@ -511,7 +511,7 @@ void appHbMgrCleanup(void) { pTarget->connInfo = NULL; } - pthread_mutex_unlock(&clientHbMgr.lock); + taosThreadMutexUnlock(&clientHbMgr.lock); } int hbMgrInit() { @@ -520,7 +520,7 @@ int hbMgrInit() { if (old == 1) return 0; clientHbMgr.appHbMgrs = taosArrayInit(0, sizeof(void *)); - pthread_mutex_init(&clientHbMgr.lock, NULL); + taosThreadMutexInit(&clientHbMgr.lock, NULL); // init handle funcs hbMgrInitHandle(); @@ -539,10 +539,10 @@ void hbMgrCleanUp() { int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 1, 0); if (old == 0) return; - pthread_mutex_lock(&clientHbMgr.lock); + taosThreadMutexLock(&clientHbMgr.lock); appHbMgrCleanup(); taosArrayDestroy(clientHbMgr.appHbMgrs); - pthread_mutex_unlock(&clientHbMgr.lock); + taosThreadMutexUnlock(&clientHbMgr.lock); clientHbMgr.appHbMgrs = NULL; #endif diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index fc97f4113c..e73854dd6d 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -95,7 +95,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, char* key = getClusterKey(user, secretEncrypt, ip, port); SAppInstInfo** pInst = NULL; - pthread_mutex_lock(&appInfo.mutex); + taosThreadMutexLock(&appInfo.mutex); pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); SAppInstInfo* p = NULL; @@ -109,7 +109,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, pInst = &p; } - pthread_mutex_unlock(&appInfo.mutex); + taosThreadMutexUnlock(&appInfo.mutex); tfree(key); return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst); @@ -137,13 +137,14 @@ int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj* return TSDB_CODE_SUCCESS; } -int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery) { +int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) { STscObj* pTscObj = pRequest->pTscObj; SParseContext cxt = { .requestId = pRequest->requestId, .acctId = pTscObj->acctId, - .db = getDbOfConnection(pTscObj), + .db = pRequest->pDb, + .topicQuery = topicQuery, .pSql = pRequest->sqlstr, .sqlLen = pRequest->sqlLen, .pMsg = pRequest->msgBuf, @@ -154,7 +155,6 @@ int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery) { cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog); if (code != TSDB_CODE_SUCCESS) { - tfree(cxt.db); return code; } @@ -163,7 +163,6 @@ int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery) { setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols); } - tfree(cxt.db); return code; } @@ -249,7 +248,7 @@ TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) { terrno = TSDB_CODE_SUCCESS; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); - CHECK_CODE_GOTO(parseSql(pRequest, &pQuery), _return); + CHECK_CODE_GOTO(parseSql(pRequest, false, &pQuery), _return); if (pQuery->directRpc) { CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return); @@ -592,21 +591,21 @@ void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t char* getDbOfConnection(STscObj* pObj) { char* p = NULL; - pthread_mutex_lock(&pObj->mutex); + taosThreadMutexLock(&pObj->mutex); size_t len = strlen(pObj->db); if (len > 0) { p = strndup(pObj->db, tListLen(pObj->db)); } - pthread_mutex_unlock(&pObj->mutex); + taosThreadMutexUnlock(&pObj->mutex); return p; } void setConnectionDB(STscObj* pTscObj, const char* db) { assert(db != NULL && pTscObj != NULL); - pthread_mutex_lock(&pTscObj->mutex); + taosThreadMutexLock(&pTscObj->mutex); tstrncpy(pTscObj->db, db, tListLen(pTscObj->db)); - pthread_mutex_unlock(&pTscObj->mutex); + taosThreadMutexUnlock(&pTscObj->mutex); } void setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) { diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index ee13527b96..a8fc394720 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -456,11 +456,99 @@ _return: void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { conf->commit_cb = cb; } +TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbName, const char* sql) { + STscObj* pTscObj = (STscObj*)taos; + SRequestObj* pRequest = NULL; + SQuery* pQueryNode = NULL; + char* astStr = NULL; + int32_t sqlLen; + + terrno = TSDB_CODE_SUCCESS; + if (taos == NULL || streamName == NULL || sql == NULL) { + tscError("invalid parameters for creating stream, connObj:%p, stream name:%s, sql:%s", taos, streamName, sql); + terrno = TSDB_CODE_TSC_INVALID_INPUT; + goto _return; + } + sqlLen = strlen(sql); + + if (strlen(streamName) >= TSDB_TABLE_NAME_LEN) { + tscError("stream name too long, max length:%d", TSDB_TABLE_NAME_LEN - 1); + terrno = TSDB_CODE_TSC_INVALID_INPUT; + goto _return; + } + + if (sqlLen > TSDB_MAX_ALLOWED_SQL_LEN) { + tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); + terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; + goto _return; + } + + tscDebug("start to create stream: %s", streamName); + + CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); + CHECK_CODE_GOTO(parseSql(pRequest, false, &pQueryNode), _return); + + // todo check for invalid sql statement and return with error code + + CHECK_CODE_GOTO(nodesNodeToString(pQueryNode->pRoot, false, &astStr, NULL), _return); + + /*printf("%s\n", pStr);*/ + + SName name = {.acctId = pTscObj->acctId, .type = TSDB_TABLE_NAME_T}; + strcpy(name.dbname, pRequest->pDb); + strcpy(name.tname, streamName); + + SCMCreateStreamReq req = { + .igExists = 1, + .ast = astStr, + .sql = (char*)sql, + }; + tNameExtractFullName(&name, req.name); + strcpy(req.outputTbName, tbName); + + int tlen = tSerializeSCMCreateStreamReq(NULL, 0, &req); + void* buf = malloc(tlen); + if (buf == NULL) { + goto _return; + } + + tSerializeSCMCreateStreamReq(buf, tlen, &req); + /*printf("formatted: %s\n", dagStr);*/ + + pRequest->body.requestMsg = (SDataBuf){ + .pData = buf, + .len = tlen, + .handle = NULL, + }; + pRequest->type = TDMT_MND_CREATE_STREAM; + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + SEpSet epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + + int64_t transporterId = 0; + asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); + + tsem_wait(&pRequest->body.rspSem); + +_return: + tfree(astStr); + qDestroyQuery(pQueryNode); + /*if (sendInfo != NULL) {*/ + /*destroySendMsgInfo(sendInfo);*/ + /*}*/ + + if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) { + pRequest->code = terrno; + } + + return pRequest; +} + TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) { STscObj* pTscObj = (STscObj*)taos; SRequestObj* pRequest = NULL; SQuery* pQueryNode = NULL; - char* pStr = NULL; + char* astStr = NULL; terrno = TSDB_CODE_SUCCESS; if (taos == NULL || topicName == NULL || sql == NULL) { @@ -481,39 +569,25 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i goto _return; } - tscDebug("start to create topic, %s", topicName); -#if 0 - CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); - CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return); + tscDebug("start to create topic: %s", topicName); - pQueryNode->streamQuery = true; + CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); + CHECK_CODE_GOTO(parseSql(pRequest, true, &pQueryNode), _return); // todo check for invalid sql statement and return with error code - SSchema* schema = NULL; - int32_t numOfCols = 0; - CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag, NULL), _return); - - pStr = qQueryPlanToString(pRequest->body.pDag); - if (pStr == NULL) { - goto _return; - } + CHECK_CODE_GOTO(nodesNodeToString(pQueryNode->pRoot, false, &astStr, NULL), _return); /*printf("%s\n", pStr);*/ - // The topic should be related to a database that the queried table is belonged to. - SName name = {0}; - char dbName[TSDB_DB_FNAME_LEN] = {0}; - // tNameGetFullDbName(&((SQueryStmtInfo*)pQueryNode)->pTableMetaInfo[0]->name, dbName); - - tNameFromString(&name, dbName, T_NAME_ACCT | T_NAME_DB); - tNameFromString(&name, topicName, T_NAME_TABLE); + SName name = {.acctId = pTscObj->acctId, .type = TSDB_TABLE_NAME_T}; + strcpy(name.dbname, pRequest->pDb); + strcpy(name.tname, topicName); SCMCreateTopicReq req = { .igExists = 1, - .physicalPlan = (char*)pStr, + .ast = astStr, .sql = (char*)sql, - .logicalPlan = (char*)"no logic plan", }; tNameExtractFullName(&name, req.name); @@ -526,7 +600,11 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i tSerializeSCMCreateTopicReq(buf, tlen, &req); /*printf("formatted: %s\n", dagStr);*/ - pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen, .handle = NULL}; + pRequest->body.requestMsg = (SDataBuf){ + .pData = buf, + .len = tlen, + .handle = NULL, + }; pRequest->type = TDMT_MND_CREATE_TOPIC; SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); @@ -536,8 +614,9 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); tsem_wait(&pRequest->body.rspSem); -#endif + _return: + tfree(astStr); qDestroyQuery(pQueryNode); /*if (sendInfo != NULL) {*/ /*destroySendMsgInfo(sendInfo);*/ diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 4b649d5f62..31328cc4b2 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1072,6 +1072,10 @@ void blockDataClearup(SSDataBlock* pDataBlock) { } int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) { + if (0 == numOfRows) { + return TSDB_CODE_SUCCESS; + } + if (IS_VAR_DATA_TYPE(pColumn->info.type)) { char* tmp = realloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); if (tmp == NULL) { @@ -1092,7 +1096,7 @@ int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRo pColumn->nullbitmap = tmp; memset(pColumn->nullbitmap, 0, BitmapLen(numOfRows)); - + assert(pColumn->info.bytes); tmp = realloc(pColumn->pData, numOfRows * pColumn->info.bytes); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -1137,7 +1141,7 @@ void* blockDataDestroy(SSDataBlock* pBlock) { taosArrayDestroy(pBlock->pDataBlock); tfree(pBlock->pBlockAgg); - tfree(pBlock); + // tfree(pBlock); return NULL; } @@ -1190,7 +1194,7 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { } int32_t len = colDataGetLength(pColData, rows); - taosEncodeFixedI32(buf, len); + tlen += taosEncodeFixedI32(buf, len); tlen += taosEncodeBinary(buf, pColData->pData, len); } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 9a0309e024..841824a7c7 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -45,6 +45,7 @@ float tsRatioOfQueryCores = 1.0f; int32_t tsMaxBinaryDisplayWidth = 30; bool tsEnableSlaveQuery = 1; bool tsPrintAuth = 0; +int32_t tsMultiProcess = 0; // monitor bool tsEnableMonitor = 1; @@ -309,7 +310,6 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "os release", info.release, 1) != 0) return -1; if (cfgAddString(pCfg, "os version", info.version, 1) != 0) return -1; if (cfgAddString(pCfg, "os machine", info.machine, 1) != 0) return -1; - if (cfgAddString(pCfg, "os sysname", info.sysname, 1) != 0) return -1; if (cfgAddString(pCfg, "version", version, 1) != 0) return -1; if (cfgAddString(pCfg, "compatible_version", compatible_version, 1) != 0) return -1; @@ -340,6 +340,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0) != 0) return -1; if (cfgAddBool(pCfg, "slaveQuery", tsEnableSlaveQuery, 0) != 0) return -1; if (cfgAddBool(pCfg, "deadLockKillQuery", tsDeadLockKillQuery, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "multiProcess", tsMultiProcess, 0, 2, 0) != 0) return -1; if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1; if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 360000, 0) != 0) return -1; @@ -403,7 +404,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { return -1; } - tsNumOfThreadsPerCore = cfgGetItem(pCfg, "maxTmrCtrl")->fval; + tsNumOfThreadsPerCore = cfgGetItem(pCfg, "numOfThreadsPerCore")->fval; tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32; tsRpcTimer = cfgGetItem(pCfg, "rpcTimer")->i32; tsRpcMaxTime = cfgGetItem(pCfg, "rpcMaxTime")->i32; @@ -457,6 +458,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsPrintAuth = cfgGetItem(pCfg, "printAuth")->bval; tsEnableSlaveQuery = cfgGetItem(pCfg, "slaveQuery")->bval; tsDeadLockKillQuery = cfgGetItem(pCfg, "deadLockKillQuery")->bval; + tsMultiProcess = cfgGetItem(pCfg, "multiProcess")->i32; tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval; tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index d9242cf0ac..a2382d1dd1 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -757,6 +757,8 @@ int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) { return 0; } +void tFreeSStatusRsp(SStatusRsp *pRsp) { taosArrayDestroy(pRsp->pDnodeEps); } + int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -2467,7 +2469,7 @@ int32_t tEncodeSMqCMCommitOffsetReq(SCoder *encoder, const SMqCMCommitOffsetReq int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) { if (tStartDecode(decoder) < 0) return -1; if (tDecodeI32(decoder, &pReq->num) < 0) return -1; - TCODER_MALLOC(pReq->offsets, SMqOffset*, pReq->num * sizeof(SMqOffset), decoder); + TCODER_MALLOC(pReq->offsets, SMqOffset *, pReq->num * sizeof(SMqOffset), decoder); if (pReq->offsets == NULL) return -1; for (int32_t i = 0; i < pReq->num; i++) { tDecodeSMqOffset(decoder, &pReq->offsets[i]); @@ -2639,27 +2641,36 @@ int32_t tSerializeSVDropTSmaReq(void **buf, SVDropTSmaReq *pReq) { int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pReq->ver); + tlen += taosEncodeFixedI64(buf, pReq->indexUid); tlen += taosEncodeString(buf, pReq->indexName); return tlen; } void *tDeserializeSVDropTSmaReq(void *buf, SVDropTSmaReq *pReq) { buf = taosDecodeFixedI64(buf, &(pReq->ver)); + buf = taosDecodeFixedI64(buf, &(pReq->indexUid)); buf = taosDecodeStringTo(buf, pReq->indexName); return buf; } int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateStreamReq *pReq) { + int32_t sqlLen = 0; + int32_t astLen = 0; + if (pReq->sql != NULL) sqlLen = (int32_t)strlen(pReq->sql); + if (pReq->ast != NULL) astLen = (int32_t)strlen(pReq->ast); + SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); if (tStartEncode(&encoder) < 0) return -1; if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->outputTbName) < 0) return -1; if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->physicalPlan) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->logicalPlan) < 0) return -1; + if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1; + if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1; + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2668,15 +2679,30 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS } int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStreamReq *pReq) { + int32_t sqlLen = 0; + int32_t astLen = 0; + SCoder decoder = {0}; tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); if (tStartDecode(&decoder) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->outputTbName) < 0) return -1; if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; - if (tDecodeCStrAlloc(&decoder, &pReq->sql) < 0) return -1; - if (tDecodeCStrAlloc(&decoder, &pReq->physicalPlan) < 0) return -1; - if (tDecodeCStrAlloc(&decoder, &pReq->logicalPlan) < 0) return -1; + if (tDecodeI32(&decoder, &sqlLen) < 0) return -1; + if (tDecodeI32(&decoder, &astLen) < 0) return -1; + + if (sqlLen > 0) { + pReq->sql = calloc(1, sqlLen + 1); + if (pReq->sql == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1; + } + + if (astLen > 0) { + pReq->ast = calloc(1, astLen + 1); + if (pReq->ast == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1; + } tEndDecode(&decoder); tCoderClear(&decoder); @@ -2685,8 +2711,7 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) { tfree(pReq->sql); - tfree(pReq->physicalPlan); - tfree(pReq->logicalPlan); + tfree(pReq->ast); } int32_t tEncodeSStreamTask(SCoder *pEncoder, const SStreamTask *pTask) { @@ -2695,6 +2720,7 @@ int32_t tEncodeSStreamTask(SCoder *pEncoder, const SStreamTask *pTask) { if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->level) < 0) return -1; if (tEncodeI8(pEncoder, pTask->status) < 0) return -1; + if (tEncodeSEpSet(pEncoder, &pTask->NextOpEp) < 0) return -1; if (tEncodeCStr(pEncoder, pTask->qmsg) < 0) return -1; tEndEncode(pEncoder); return pEncoder->pos; @@ -2706,6 +2732,7 @@ int32_t tDecodeSStreamTask(SCoder *pDecoder, SStreamTask *pTask) { if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->level) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1; + if (tDecodeSEpSet(pDecoder, &pTask->NextOpEp) < 0) return -1; if (tDecodeCStrAlloc(pDecoder, &pTask->qmsg) < 0) return -1; tEndDecode(pDecoder); return 0; diff --git a/source/dnode/bnode/src/bnode.c b/source/dnode/bnode/src/bnode.c index 9570bc72a0..2e5c96ecdd 100644 --- a/source/dnode/bnode/src/bnode.c +++ b/source/dnode/bnode/src/bnode.c @@ -25,5 +25,3 @@ void bndClose(SBnode *pBnode) { free(pBnode); } int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad) { return 0; } int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs) { return 0; } - -void bndDestroy(const char *path) {} diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 64e8980219..35ea166983 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -1,2 +1,30 @@ -add_subdirectory(daemon) -add_subdirectory(impl) \ No newline at end of file +aux_source_directory(src DNODE_SRC) +aux_source_directory(dnode/src DNODE_SRC) +aux_source_directory(qnode/src DNODE_SRC) +aux_source_directory(bnode/src DNODE_SRC) +aux_source_directory(snode/src DNODE_SRC) +aux_source_directory(vnode/src DNODE_SRC) +aux_source_directory(mnode/src DNODE_SRC) +aux_source_directory(container/src DNODE_SRC) + +add_library(dnode STATIC ${DNODE_SRC}) +target_link_libraries( + dnode cjson mnode vnode qnode snode bnode wal sync taos tfs monitor +) +target_include_directories( + dnode + PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/dnode/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/qnode/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/bnode/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/snode/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/vnode/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/mnode/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/container/inc" +) + +add_subdirectory(main) + +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) diff --git a/source/dnode/mgmt/impl/inc/dndBnode.h b/source/dnode/mgmt/bnode/inc/bm.h similarity index 71% rename from source/dnode/mgmt/impl/inc/dndBnode.h rename to source/dnode/mgmt/bnode/inc/bm.h index 080cd2e487..79cf76d113 100644 --- a/source/dnode/mgmt/impl/inc/dndBnode.h +++ b/source/dnode/mgmt/bnode/inc/bm.h @@ -16,17 +16,13 @@ #ifndef _TD_DND_BNODE_H_ #define _TD_DND_BNODE_H_ +#include "dnd.h" + #ifdef __cplusplus extern "C" { #endif -#include "dndEnv.h" -int32_t dndInitBnode(SDnode *pDnode); -void dndCleanupBnode(SDnode *pDnode); - -void dndProcessBnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); -int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); +void bmGetMgmtFp(SMgmtWrapper *pWrapper); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/bnode/inc/bmInt.h b/source/dnode/mgmt/bnode/inc/bmInt.h new file mode 100644 index 0000000000..ddbf7f4c4e --- /dev/null +++ b/source/dnode/mgmt/bnode/inc/bmInt.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_BNODE_INT_H_ +#define _TD_DND_BNODE_INT_H_ + +#include "bm.h" +#include "bnode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SBnodeMgmt { + SBnode *pBnode; + SDnode *pDnode; + SMgmtWrapper *pWrapper; + const char *path; + SDnodeWorker writeWorker; +} SBnodeMgmt; + +// bmInt.c +int32_t bmOpen(SMgmtWrapper *pWrapper); +int32_t bmDrop(SMgmtWrapper *pWrapper); + +// bmMsg.c +void bmInitMsgHandles(SMgmtWrapper *pWrapper); +int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); + +// bmWorker.c +int32_t bmStartWorker(SBnodeMgmt *pMgmt); +void bmStopWorker(SBnodeMgmt *pMgmt); +int32_t bmProcessWriteMsg(SBnodeMgmt *pMgmt, SNodeMsg *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_BNODE_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/bnode/src/bmInt.c b/source/dnode/mgmt/bnode/src/bmInt.c new file mode 100644 index 0000000000..9964b88090 --- /dev/null +++ b/source/dnode/mgmt/bnode/src/bmInt.c @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "bmInt.h" + +static int32_t bmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); } + +static void bmInitOption(SBnodeMgmt *pMgmt, SBnodeOpt *pOption) { + SDnode *pDnode = pMgmt->pDnode; + pOption->pWrapper = pMgmt->pWrapper; + pOption->sendReqFp = dndSendReqToDnode; + pOption->sendMnodeReqFp = dndSendReqToMnode; + pOption->sendRspFp = dndSendRsp; + pOption->dnodeId = pDnode->dnodeId; + pOption->clusterId = pDnode->clusterId; +} + +static int32_t bmOpenImp(SBnodeMgmt *pMgmt) { + SBnodeOpt option = {0}; + bmInitOption(pMgmt, &option); + + pMgmt->pBnode = bndOpen(pMgmt->path, &option); + if (pMgmt->pBnode == NULL) { + dError("failed to open bnode since %s", terrstr()); + return -1; + } + + if (bmStartWorker(pMgmt) != 0) { + dError("failed to start bnode worker since %s", terrstr()); + return -1; + } + + bool deployed = true; + if (dndWriteFile(pMgmt->pWrapper, deployed) != 0) { + dError("failed to write bnode file since %s", terrstr()); + return -1; + } + + return 0; +} + +static void bmCloseImp(SBnodeMgmt *pMgmt) { + if (pMgmt->pBnode != NULL) { + bmStopWorker(pMgmt); + bndClose(pMgmt->pBnode); + pMgmt->pBnode = NULL; + } +} + +int32_t bmDrop(SMgmtWrapper *pWrapper) { + SBnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return 0; + + dInfo("bnode-mgmt start to drop"); + bool deployed = false; + if (dndWriteFile(pWrapper, deployed) != 0) { + dError("failed to drop bnode since %s", terrstr()); + return -1; + } + + bmCloseImp(pMgmt); + taosRemoveDir(pMgmt->path); + pWrapper->pMgmt = NULL; + free(pMgmt); + dInfo("bnode-mgmt is dropped"); + return 0; +} + +static void bmClose(SMgmtWrapper *pWrapper) { + SBnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return; + + dInfo("bnode-mgmt start to cleanup"); + bmCloseImp(pMgmt); + pWrapper->pMgmt = NULL; + free(pMgmt); + dInfo("bnode-mgmt is cleaned up"); +} + +int32_t bmOpen(SMgmtWrapper *pWrapper) { + dInfo("bnode-mgmt start to init"); + SBnodeMgmt *pMgmt = calloc(1, sizeof(SBnodeMgmt)); + if (pMgmt == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pMgmt->path = pWrapper->path; + pMgmt->pDnode = pWrapper->pDnode; + pMgmt->pWrapper = pWrapper; + pWrapper->pMgmt = pMgmt; + + int32_t code = bmOpenImp(pMgmt); + if (code != 0) { + dError("failed to init bnode-mgmt since %s", terrstr()); + bmClose(pWrapper); + } else { + dInfo("bnode-mgmt is initialized"); + } + + return code; +} + +void bmGetMgmtFp(SMgmtWrapper *pWrapper) { + SMgmtFp mgmtFp = {0}; + mgmtFp.openFp = bmOpen; + mgmtFp.closeFp = bmClose; + mgmtFp.createMsgFp = bmProcessCreateReq; + mgmtFp.dropMsgFp = bmProcessDropReq; + mgmtFp.requiredFp = bmRequire; + + bmInitMsgHandles(pWrapper); + pWrapper->name = "bnode"; + pWrapper->fp = mgmtFp; +} diff --git a/source/dnode/mgmt/bnode/src/bmMsg.c b/source/dnode/mgmt/bnode/src/bmMsg.c new file mode 100644 index 0000000000..c01d260c3f --- /dev/null +++ b/source/dnode/mgmt/bnode/src/bmMsg.c @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "bmInt.h" + +int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDCreateBnodeReq createReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (createReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to create bnode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pDnode->dnodeId); + return -1; + } else { + return bmOpen(pWrapper); + } +} + +int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDDropBnodeReq dropReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (dropReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to drop bnode since %s", terrstr()); + return -1; + } else { + return bmDrop(pWrapper); + } +} + +void bmInitMsgHandles(SMgmtWrapper *pWrapper) {} diff --git a/source/dnode/mgmt/bnode/src/bmWorker.c b/source/dnode/mgmt/bnode/src/bmWorker.c new file mode 100644 index 0000000000..c8ad137842 --- /dev/null +++ b/source/dnode/mgmt/bnode/src/bmWorker.c @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "bmInt.h" + +static void bmSendErrorRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { + SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; + dndSendRsp(pWrapper, &rpcRsp); + + dTrace("msg:%p, is freed", pMsg); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); +} + +static void bmSendErrorRsps(SMgmtWrapper *pWrapper, STaosQall *qall, int32_t numOfMsgs, int32_t code) { + for (int32_t i = 0; i < numOfMsgs; ++i) { + SNodeMsg *pMsg = NULL; + taosGetQitem(qall, (void **)&pMsg); + bmSendErrorRsp(pWrapper, pMsg, code); + } +} + +static void bmProcessQueue(SBnodeMgmt *pMgmt, STaosQall *qall, int32_t numOfMsgs) { + SMgmtWrapper *pWrapper = pMgmt->pWrapper; + + SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SNodeMsg *)); + if (pArray == NULL) { + bmSendErrorRsps(pWrapper, qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY); + return; + } + + for (int32_t i = 0; i < numOfMsgs; ++i) { + SNodeMsg *pMsg = NULL; + taosGetQitem(qall, (void **)&pMsg); + dTrace("msg:%p, will be processed in bnode queue", pMsg); + if (taosArrayPush(pArray, &pMsg) == NULL) { + bmSendErrorRsp(pWrapper, pMsg, TSDB_CODE_OUT_OF_MEMORY); + } + } + + bndProcessWMsgs(pMgmt->pBnode, pArray); + + for (size_t i = 0; i < numOfMsgs; i++) { + SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); + dTrace("msg:%p, is freed", pMsg); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); + } + taosArrayDestroy(pArray); +} + +int32_t bmProcessWriteMsg(SBnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SDnodeWorker *pWorker = &pMgmt->writeWorker; + + dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + return dndWriteMsgToWorker(pWorker, pMsg); +} + +int32_t bmStartWorker(SBnodeMgmt *pMgmt) { + if (dndInitWorker(pMgmt, &pMgmt->writeWorker, DND_WORKER_MULTI, "bnode-write", 0, 1, bmProcessQueue) != 0) { + dError("failed to start bnode write worker since %s", terrstr()); + return -1; + } + + return 0; +} + +void bmStopWorker(SBnodeMgmt *pMgmt) { dndCleanupWorker(&pMgmt->writeWorker); } diff --git a/source/dnode/mgmt/container/inc/dnd.h b/source/dnode/mgmt/container/inc/dnd.h new file mode 100644 index 0000000000..29c05d6d01 --- /dev/null +++ b/source/dnode/mgmt/container/inc/dnd.h @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_H_ +#define _TD_DND_H_ + +#include "os.h" + +#include "cJSON.h" +#include "monitor.h" +#include "tcache.h" +#include "tcrc32c.h" +#include "tdatablock.h" +#include "tglobal.h" +#include "thash.h" +#include "tlockfree.h" +#include "tlog.h" +#include "tmsg.h" +#include "tprocess.h" +#include "tqueue.h" +#include "trpc.h" +#include "tthread.h" +#include "ttime.h" +#include "tworker.h" + +#include "dnode.h" +#include "tfs.h" +#include "wal.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} +#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} +#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} +#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} +#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} + +typedef enum { DNODE, VNODES, QNODE, SNODE, MNODE, BNODE, NODE_MAX } ENodeType; +typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EDndStatus; +typedef enum { DND_ENV_INIT, DND_ENV_READY, DND_ENV_CLEANUP } EEnvStatus; +typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType; +typedef enum { PROC_SINGLE, PROC_CHILD, PROC_PARENT } EProcType; + +typedef struct SMgmtFp SMgmtFp; +typedef struct SMgmtWrapper SMgmtWrapper; +typedef struct SMsgHandle SMsgHandle; +typedef struct SDnodeMgmt SDnodeMgmt; +typedef struct SVnodesMgmt SVnodesMgmt; +typedef struct SMnodeMgmt SMnodeMgmt; +typedef struct SQnodeMgmt SQnodeMgmt; +typedef struct SSnodeMgmt SSnodeMgmt; +typedef struct SBnodeMgmt SBnodeMgmt; + +typedef int32_t (*NodeMsgFp)(void *pMgmt, SNodeMsg *pMsg); +typedef int32_t (*OpenNodeFp)(SMgmtWrapper *pWrapper); +typedef void (*CloseNodeFp)(SMgmtWrapper *pWrapper); +typedef int32_t (*StartNodeFp)(SMgmtWrapper *pWrapper); +typedef int32_t (*CreateNodeFp)(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +typedef int32_t (*DropNodeFp)(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +typedef int32_t (*RequireNodeFp)(SMgmtWrapper *pWrapper, bool *required); + +typedef struct { + EWorkerType type; + const char *name; + int32_t minNum; + int32_t maxNum; + void *queueFp; + void *param; + STaosQueue *queue; + union { + SQWorkerPool pool; + SWWorkerPool mpool; + }; +} SDnodeWorker; + +typedef struct SMsgHandle { + NodeMsgFp msgFp; + SMgmtWrapper *pWrapper; +} SMsgHandle; + +typedef struct SMgmtFp { + OpenNodeFp openFp; + CloseNodeFp closeFp; + StartNodeFp startFp; + CreateNodeFp createMsgFp; + DropNodeFp dropMsgFp; + RequireNodeFp requiredFp; +} SMgmtFp; + +typedef struct SMgmtWrapper { + const char *name; + char *path; + int32_t refCount; + SRWLatch latch; + bool deployed; + bool required; + EProcType procType; + SProcObj *pProc; + void *pMgmt; + SDnode *pDnode; + NodeMsgFp msgFps[TDMT_MAX]; + SMgmtFp fp; +} SMgmtWrapper; + +typedef struct { + void *serverRpc; + void *clientRpc; + SMsgHandle msgHandles[TDMT_MAX]; +} STransMgmt; + +typedef struct SDnode { + int64_t clusterId; + int32_t dnodeId; + int32_t numOfSupportVnodes; + int64_t rebootTime; + char *localEp; + char *localFqdn; + char *firstEp; + char *secondEp; + char *dataDir; + SDiskCfg *pDisks; + int32_t numOfDisks; + uint16_t serverPort; + bool dropped; + EDndStatus status; + EDndEvent event; + EProcType procType; + SStartupReq startup; + TdFilePtr pLockFile; + STransMgmt trans; + SMgmtWrapper wrappers[NODE_MAX]; +} SDnode; + +EDndStatus dndGetStatus(SDnode *pDnode); +void dndSetStatus(SDnode *pDnode, EDndStatus stat); +SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); +void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp); +void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc); +void dndSendMonitorReport(SDnode *pDnode); + +int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); +int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pMsg); +void dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp); + +int32_t dndInitWorker(void *param, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum, + int32_t maxNum, void *queueFp); +void dndCleanupWorker(SDnodeWorker *pWorker); +int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pMsg); + +int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg); + +int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed); +int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/container/inc/dndInt.h b/source/dnode/mgmt/container/inc/dndInt.h new file mode 100644 index 0000000000..9ae70874fe --- /dev/null +++ b/source/dnode/mgmt/container/inc/dndInt.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_INT_H_ +#define _TD_DND_INT_H_ + +#include "dnd.h" + +#include "bm.h" +#include "dm.h" +#include "mm.h" +#include "qm.h" +#include "sm.h" +#include "vm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// dndInt.c +int32_t dndInit(); +void dndCleanup(); +const char *dndStatStr(EDndStatus stat); +void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup); +TdFilePtr dndCheckRunning(char *dataDir); +void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); + +// dndMsg.c +void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, SEpSet *pEpSet); + +// dndExec.c +int32_t dndOpenNode(SMgmtWrapper *pWrapper); +void dndCloseNode(SMgmtWrapper *pWrapper); +int32_t dndRun(SDnode *pDnode); + +// dndObj.c +SDnode *dndCreate(const SDnodeOpt *pOption); +void dndClose(SDnode *pDnode); +void dndHandleEvent(SDnode *pDnode, EDndEvent event); + +SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); +int32_t dndMarkWrapper(SMgmtWrapper *pWrapper); +void dndReleaseWrapper(SMgmtWrapper *pWrapper); + +// dndTransport.c +int32_t dndInitServer(SDnode *pDnode); +void dndCleanupServer(SDnode *pDnode); +int32_t dndInitClient(SDnode *pDnode); +void dndCleanupClient(SDnode *pDnode); +int32_t dndInitMsgHandle(SDnode *pDnode); +void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/container/src/dndExec.c b/source/dnode/mgmt/container/src/dndExec.c new file mode 100644 index 0000000000..d5c882398e --- /dev/null +++ b/source/dnode/mgmt/container/src/dndExec.c @@ -0,0 +1,243 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +static void dndResetLog(SMgmtWrapper *pMgmt) { + char logname[24] = {0}; + snprintf(logname, sizeof(logname), "%slog", pMgmt->name); + + dInfo("node:%s, reset log to %s", pMgmt->name, logname); + taosCloseLog(); + taosInitLog(logname, 1); +} + +static bool dndRequireNode(SMgmtWrapper *pWrapper) { + bool required = false; + int32_t code =(*pWrapper->fp.requiredFp)(pWrapper, &required); + if (!required) { + dDebug("node:%s, no need to start", pWrapper->name); + } else { + dDebug("node:%s, need to start", pWrapper->name); + } + return required; +} + +int32_t dndOpenNode(SMgmtWrapper *pWrapper) { + int32_t code = (*pWrapper->fp.openFp)(pWrapper); + if (code != 0) { + dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); + return -1; + } else { + dDebug("node:%s, has been opened", pWrapper->name); + } + + pWrapper->deployed = true; + return 0; +} + +void dndCloseNode(SMgmtWrapper *pWrapper) { + taosWLockLatch(&pWrapper->latch); + if (pWrapper->deployed) { + (*pWrapper->fp.closeFp)(pWrapper); + pWrapper->deployed = false; + } + if (pWrapper->pProc) { + taosProcCleanup(pWrapper->pProc); + pWrapper->pProc = NULL; + } + taosWUnLockLatch(&pWrapper->latch); +} + +static int32_t dndRunInSingleProcess(SDnode *pDnode) { + dInfo("dnode run in single process mode"); + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + pWrapper->required = dndRequireNode(pWrapper); + if (!pWrapper->required) continue; + + if (taosMkDir(pWrapper->path) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); + return -1; + } + + dInfo("node:%s, will start in single process", pWrapper->name); + pWrapper->procType = PROC_SINGLE; + if (dndOpenNode(pWrapper) != 0) { + dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); + return -1; + } + } + + dndSetStatus(pDnode, DND_STAT_RUNNING); + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + if (!pWrapper->required) continue; + if (pWrapper->fp.startFp == NULL) continue; + if ((*pWrapper->fp.startFp)(pWrapper) != 0) { + dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); + return -1; + } + } + + return 0; +} + +static void dndClearNodesExecpt(SDnode *pDnode, ENodeType except) { + dndCleanupServer(pDnode); + for (ENodeType n = 0; n < NODE_MAX; ++n) { + if (except == n) continue; + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + dndCloseNode(pWrapper); + } +} + +static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t msgLen, void *pCont, int32_t contLen) { + dTrace("msg:%p, get from child queue", pMsg); + SRpcMsg *pRpc = &pMsg->rpcMsg; + pRpc->pCont = pCont; + + NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; + int32_t code = (*msgFp)(pWrapper, pMsg); + + if (code != 0) { + if (pRpc->msgType & 1U) { + SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; + dndSendRsp(pWrapper, &rsp); + } + + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pCont); + } +} + +static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRsp, int32_t msgLen, void *pCont, int32_t contLen) { + dTrace("msg:%p, get from parent queue", pRsp); + pRsp->pCont = pCont; + dndSendRpcRsp(pWrapper, pRsp); + free(pRsp); +} + +static int32_t dndRunInMultiProcess(SDnode *pDnode) { + dInfo("dnode run in multi process mode"); + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + pWrapper->required = dndRequireNode(pWrapper); + if (!pWrapper->required) continue; + + if (taosMkDir(pWrapper->path) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); + return -1; + } + + if (n == DNODE) { + dInfo("node:%s, will start in parent process", pWrapper->name); + pWrapper->procType = PROC_SINGLE; + if (dndOpenNode(pWrapper) != 0) { + dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); + return -1; + } + continue; + } + + SProcCfg cfg = {.childQueueSize = 1024 * 1024 * 2, // size will be a configuration item + .childConsumeFp = (ProcConsumeFp)dndConsumeChildQueue, + .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, + .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, + .childMallocBodyFp = (ProcMallocFp)rpcMallocCont, + .childFreeBodyFp = (ProcFreeFp)rpcFreeCont, + .parentQueueSize = 1024 * 1024 * 2, // size will be a configuration item + .parentConsumeFp = (ProcConsumeFp)dndConsumeParentQueue, + .parentdMallocHeadFp = (ProcMallocFp)malloc, + .parentFreeHeadFp = (ProcFreeFp)free, + .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont, + .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont, + .testFlag = 0, + .pParent = pWrapper, + .name = pWrapper->name}; + SProcObj *pProc = taosProcInit(&cfg); + if (pProc == NULL) { + dError("node:%s, failed to fork since %s", pWrapper->name, terrstr()); + return -1; + } + + pWrapper->pProc = pProc; + + if (taosProcIsChild(pProc)) { + dInfo("node:%s, will start in child process", pWrapper->name); + pWrapper->procType = PROC_CHILD; + dndResetLog(pWrapper); + + dInfo("node:%s, clean up resources inherited from parent", pWrapper->name); + dndClearNodesExecpt(pDnode, n); + + dInfo("node:%s, will be initialized in child process", pWrapper->name); + dndOpenNode(pWrapper); + } else { + dInfo("node:%s, will not start in parent process", pWrapper->name); + pWrapper->procType = PROC_PARENT; + } + + if (taosProcRun(pProc) != 0) { + dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr()); + return -1; + } + } + +#if 0 + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); + if (pWrapper->procType == PROC_PARENT && dmStart(pWrapper->pMgmt) != 0) { + dndReleaseWrapper(pWrapper); + dError("failed to start dnode worker since %s", terrstr()); + return -1; + } + + dndReleaseWrapper(pWrapper); +#endif + return 0; +} + +int32_t dndRun(SDnode *pDnode) { + if (tsMultiProcess == 0) { + if (dndRunInSingleProcess(pDnode) != 0) { + dError("failed to run dnode in single process mode since %s", terrstr()); + return -1; + } + } else { + if (dndRunInMultiProcess(pDnode) != 0) { + dError("failed to run dnode in multi process mode since %s", terrstr()); + return -1; + } + } + + dndReportStartup(pDnode, "TDengine", "initialized successfully"); + + while (1) { + if (pDnode->event == DND_EVENT_STOP) { + dInfo("dnode is about to stop"); + break; + } + taosMsleep(100); + } + + return 0; +} diff --git a/source/dnode/mgmt/container/src/dndFile.c b/source/dnode/mgmt/container/src/dndFile.c new file mode 100644 index 0000000000..a5f8fb7169 --- /dev/null +++ b/source/dnode/mgmt/container/src/dndFile.c @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed) { + int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; + int32_t len = 0; + int32_t maxLen = 1024; + char *content = calloc(1, maxLen + 1); + cJSON *root = NULL; + char file[PATH_MAX]; + TdFilePtr pFile = NULL; + + snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name); + pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { + dDebug("file %s not exist", file); + code = 0; + goto _OVER; + } + + len = (int32_t)taosReadFile(pFile, content, maxLen); + if (len <= 0) { + dError("failed to read %s since content is null", file); + goto _OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s since invalid json format", file); + goto _OVER; + } + + cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); + if (!deployed || deployed->type != cJSON_Number) { + dError("failed to read %s since deployed not found", file); + goto _OVER; + } + *pDeployed = deployed->valueint != 0; + + code = 0; + dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); + +_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (pFile != NULL) taosCloseFile(&pFile); + + terrno = code; + return code; +} + +int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed) { + char file[PATH_MAX]; + snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name); + + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to write %s since %s", file, terrstr()); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 1024; + char *content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", deployed); + len += snprintf(content + len, maxLen - len, "}\n"); + + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); + free(content); + + char realfile[PATH_MAX]; + snprintf(realfile, sizeof(realfile), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name); + + if (taosRenameFile(file, realfile) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to rename %s since %s", file, terrstr()); + return -1; + } + + dInfo("successed to write %s, deployed:%d", realfile, deployed); + return 0; +} diff --git a/source/dnode/mgmt/container/src/dndInt.c b/source/dnode/mgmt/container/src/dndInt.c new file mode 100644 index 0000000000..4decc6ba87 --- /dev/null +++ b/source/dnode/mgmt/container/src/dndInt.c @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +static int8_t once = DND_ENV_INIT; + +int32_t dndInit() { + dDebug("start to init dnode env"); + if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { + terrno = TSDB_CODE_REPEAT_INIT; + dError("failed to init dnode env since %s", terrstr()); + return -1; + } + + taosIgnSIGPIPE(); + taosBlockSIGPIPE(); + taosResolveCRC(); + + if (rpcInit() != 0) { + dError("failed to init rpc since %s", terrstr()); + dndCleanup(); + return -1; + } + + SMonCfg monCfg = {0}; + monCfg.maxLogs = tsMonitorMaxLogs; + monCfg.port = tsMonitorPort; + monCfg.server = tsMonitorFqdn; + monCfg.comp = tsMonitorComp; + if (monInit(&monCfg) != 0) { + dError("failed to init monitor since %s", terrstr()); + dndCleanup(); + return -1; + } + + dInfo("dnode env is initialized"); + return 0; +} + +void dndCleanup() { + dDebug("start to cleanup dnode env"); + if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { + dError("dnode env is already cleaned up"); + return; + } + + monCleanup(); + rpcCleanup(); + walCleanUp(); + taosStopCacheRefreshWorker(); + dInfo("dnode env is cleaned up"); +} + +void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp) { + pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp; +} + +EDndStatus dndGetStatus(SDnode *pDnode) { return pDnode->status; } + +void dndSetStatus(SDnode *pDnode, EDndStatus status) { + if (pDnode->status != status) { + dDebug("dnode status set from %s to %s", dndStatStr(pDnode->status), dndStatStr(status)); + pDnode->status = status; + } +} + +const char *dndStatStr(EDndStatus status) { + switch (status) { + case DND_STAT_INIT: + return "init"; + case DND_STAT_RUNNING: + return "running"; + case DND_STAT_STOPPED: + return "stopped"; + default: + return "unknown"; + } +} + +void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc) { + SStartupReq *pStartup = &pDnode->startup; + tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); + tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); + pStartup->finished = 0; +} + +void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { + memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); + pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING); +} + +TdFilePtr dndCheckRunning(char *dataDir) { + char filepath[PATH_MAX] = {0}; + snprintf(filepath, sizeof(filepath), "%s/.running", dataDir); + + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to lock file:%s since %s, quit", filepath, terrstr()); + return NULL; + } + + int32_t ret = taosLockFile(pFile); + if (ret != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to lock file:%s since %s, quit", filepath, terrstr()); + taosCloseFile(&pFile); + return NULL; + } + + dDebug("file:%s is locked", filepath); + return pFile; +} + +void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { + dDebug("startup req is received"); + + SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); + dndGetStartup(pDnode, pStartup); + + dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); + + SRpcMsg rpcRsp = {.handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq)}; + rpcSendResponse(&rpcRsp); +} diff --git a/source/dnode/mgmt/container/src/dndMonitor.c b/source/dnode/mgmt/container/src/dndMonitor.c new file mode 100644 index 0000000000..ef3db387de --- /dev/null +++ b/source/dnode/mgmt/container/src/dndMonitor.c @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +static int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) { + tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name)); + pInfo->logdir.size = tsLogSpace.size; + tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); + pInfo->tempdir.size = tsTempSpace.size; + + return vmMonitorTfsInfo(dndAcquireWrapper(pDnode, VNODES), pInfo); +} + +static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { + pInfo->protocol = 1; + pInfo->dnode_id = pDnode->dnodeId; + pInfo->cluster_id = pDnode->clusterId; + tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); +} + +static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { + pInfo->uptime = (taosGetTimestampMs() - pDnode->rebootTime) / (86400000.0f); + taosGetCpuUsage(&pInfo->cpu_engine, &pInfo->cpu_system); + taosGetCpuCores(&pInfo->cpu_cores); + taosGetProcMemory(&pInfo->mem_engine); + taosGetSysMemory(&pInfo->mem_system); + pInfo->mem_total = tsTotalMemoryKB; + pInfo->disk_engine = 0; + pInfo->disk_used = tsDataSpace.size.used; + pInfo->disk_total = tsDataSpace.size.total; + taosGetCardInfo(&pInfo->net_in, &pInfo->net_out); + taosGetProcIO(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk); + + vmMonitorVnodeReqs(dndAcquireWrapper(pDnode, VNODES), pInfo); + pInfo->has_mnode = (dndAcquireWrapper(pDnode, MNODE)->required); +} + +void dndSendMonitorReport(SDnode *pDnode) { + if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; + dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort); + + SMonInfo *pMonitor = monCreateMonitorInfo(); + if (pMonitor == NULL) return; + + SMonBasicInfo basicInfo = {0}; + dndGetMonitorBasicInfo(pDnode, &basicInfo); + monSetBasicInfo(pMonitor, &basicInfo); + + SMonClusterInfo clusterInfo = {0}; + SMonVgroupInfo vgroupInfo = {0}; + SMonGrantInfo grantInfo = {0}; + if (mmMonitorMnodeInfo(dndAcquireWrapper(pDnode, MNODE), &clusterInfo, &vgroupInfo, &grantInfo) == 0) { + monSetClusterInfo(pMonitor, &clusterInfo); + monSetVgroupInfo(pMonitor, &vgroupInfo); + monSetGrantInfo(pMonitor, &grantInfo); + } + + SMonDnodeInfo dnodeInfo = {0}; + dndGetMonitorDnodeInfo(pDnode, &dnodeInfo); + monSetDnodeInfo(pMonitor, &dnodeInfo); + + SMonDiskInfo diskInfo = {0}; + if (dndGetMonitorDiskInfo(pDnode, &diskInfo) == 0) { + monSetDiskInfo(pMonitor, &diskInfo); + } + + taosArrayDestroy(clusterInfo.dnodes); + taosArrayDestroy(clusterInfo.mnodes); + taosArrayDestroy(vgroupInfo.vgroups); + taosArrayDestroy(diskInfo.datadirs); + + monSendReport(pMonitor); + monCleanupMonitorInfo(pMonitor); +} \ No newline at end of file diff --git a/source/dnode/mgmt/container/src/dndMsg.c b/source/dnode/mgmt/container/src/dndMsg.c new file mode 100644 index 0000000000..21e9cc71a5 --- /dev/null +++ b/source/dnode/mgmt/container/src/dndMsg.c @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); + if (pWrapper != NULL) { + dmUpdateMnodeEpSet(pWrapper->pMgmt, pEpSet); + } + dndReleaseWrapper(pWrapper); +} + +static inline NodeMsgFp dndGetMsgFp(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; + if (msgFp == NULL) { + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + } + + return msgFp; +} + +static inline int32_t dndBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) { + SRpcConnInfo connInfo = {0}; + if ((pRpc->msgType & 1U) && rpcGetConnInfo(pRpc->handle, &connInfo) != 0) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + dError("failed to build msg since %s, app:%p RPC:%p", terrstr(), pRpc->ahandle, pRpc->handle); + return -1; + } + + memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN); + memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg)); + + return 0; +} + +void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) { + if (pEpSet && pEpSet->numOfEps > 0 && pRpc->msgType == TDMT_MND_STATUS_RSP) { + dndUpdateMnodeEpSet(pWrapper->pDnode, pEpSet); + } + + int32_t code = -1; + SNodeMsg *pMsg = NULL; + NodeMsgFp msgFp = NULL; + + if (dndMarkWrapper(pWrapper) != 0) goto _OVER; + if ((msgFp = dndGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER; + if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER; + if (dndBuildMsg(pMsg, pRpc) != 0) goto _OVER; + + dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user); + if (pWrapper->procType == PROC_SINGLE) { + code = (*msgFp)(pWrapper->pMgmt, pMsg); + } else if (pWrapper->procType == PROC_PARENT) { + code = taosProcPutToChildQueue(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen); + } else { + } + +_OVER: + if (code == 0) { + if (pWrapper->procType == PROC_PARENT) { + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pRpc->pCont); + } + } else { + dError("msg:%p, failed to process since 0x%04x:%s", pMsg, code & 0XFFFF, terrstr()); + if (pRpc->msgType & 1U) { + SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; + dndSendRsp(pWrapper, &rsp); + } + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pRpc->pCont); + } + + dndReleaseWrapper(pWrapper); +} + +static int32_t dndProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); + if (pWrapper != NULL) { + dndReleaseWrapper(pWrapper); + terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED; + dError("failed to create node since %s", terrstr()); + return -1; + } + + pWrapper = &pDnode->wrappers[ntype]; + + if (taosMkDir(pWrapper->path) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); + return -1; + } + + int32_t code = (*pWrapper->fp.createMsgFp)(pWrapper, pMsg); + if (code != 0) { + dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); + } else { + dDebug("node:%s, has been opened", pWrapper->name); + pWrapper->deployed = true; + } + + return code; +} + +static int32_t dndProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); + if (pWrapper == NULL) { + terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + dError("failed to drop node since %s", terrstr()); + return -1; + } + + taosWLockLatch(&pWrapper->latch); + pWrapper->deployed = false; + + int32_t code = (*pWrapper->fp.dropMsgFp)(pWrapper, pMsg); + if (code != 0) { + pWrapper->deployed = true; + dError("node:%s, failed to drop since %s", pWrapper->name, terrstr()); + } else { + pWrapper->deployed = false; + dDebug("node:%s, has been dropped", pWrapper->name); + } + + taosWUnLockLatch(&pWrapper->latch); + dndReleaseWrapper(pWrapper); + return code; +} + +int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg) { + switch (pMsg->rpcMsg.msgType) { + case TDMT_DND_CREATE_MNODE: + return dndProcessCreateNodeMsg(pDnode, MNODE, pMsg); + case TDMT_DND_DROP_MNODE: + return dndProcessDropNodeMsg(pDnode, MNODE, pMsg); + case TDMT_DND_CREATE_QNODE: + return dndProcessCreateNodeMsg(pDnode, QNODE, pMsg); + case TDMT_DND_DROP_QNODE: + return dndProcessDropNodeMsg(pDnode, QNODE, pMsg); + case TDMT_DND_CREATE_SNODE: + return dndProcessCreateNodeMsg(pDnode, SNODE, pMsg); + case TDMT_DND_DROP_SNODE: + return dndProcessDropNodeMsg(pDnode, SNODE, pMsg); + case TDMT_DND_CREATE_BNODE: + return dndProcessCreateNodeMsg(pDnode, BNODE, pMsg); + case TDMT_DND_DROP_BNODE: + return dndProcessDropNodeMsg(pDnode, BNODE, pMsg); + default: + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + return -1; + } +} \ No newline at end of file diff --git a/source/dnode/mgmt/container/src/dndObj.c b/source/dnode/mgmt/container/src/dndObj.c new file mode 100644 index 0000000000..75a06d5859 --- /dev/null +++ b/source/dnode/mgmt/container/src/dndObj.c @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +static int32_t dndInitMemory(SDnode *pDnode, const SDnodeOpt *pOption) { + pDnode->numOfSupportVnodes = pOption->numOfSupportVnodes; + pDnode->serverPort = pOption->serverPort; + pDnode->dataDir = strdup(pOption->dataDir); + pDnode->localEp = strdup(pOption->localEp); + pDnode->localFqdn = strdup(pOption->localFqdn); + pDnode->firstEp = strdup(pOption->firstEp); + pDnode->secondEp = strdup(pOption->secondEp); + pDnode->pDisks = pOption->pDisks; + pDnode->numOfDisks = pOption->numOfDisks; + pDnode->rebootTime = taosGetTimestampMs(); + + if (pDnode->dataDir == NULL || pDnode->localEp == NULL || pDnode->localFqdn == NULL || pDnode->firstEp == NULL || + pDnode->secondEp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + return 0; +} + +static void dndClearMemory(SDnode *pDnode) { + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pMgmt = &pDnode->wrappers[n]; + tfree(pMgmt->path); + } + if (pDnode->pLockFile != NULL) { + taosUnLockFile(pDnode->pLockFile); + taosCloseFile(&pDnode->pLockFile); + pDnode->pLockFile = NULL; + } + tfree(pDnode->localEp); + tfree(pDnode->localFqdn); + tfree(pDnode->firstEp); + tfree(pDnode->secondEp); + tfree(pDnode->dataDir); + free(pDnode); + dDebug("dnode object memory is cleared, data:%p", pDnode); +} + +SDnode *dndCreate(const SDnodeOpt *pOption) { + dInfo("start to create dnode object"); + int32_t code = -1; + char path[PATH_MAX]; + SDnode *pDnode = NULL; + + pDnode = calloc(1, sizeof(SDnode)); + if (pDnode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + if (dndInitMemory(pDnode, pOption) != 0) { + goto _OVER; + } + + dndSetStatus(pDnode, DND_STAT_INIT); + pDnode->pLockFile = dndCheckRunning(pDnode->dataDir); + if (pDnode->pLockFile == NULL) { + goto _OVER; + } + + if (dndInitServer(pDnode) != 0) { + dError("failed to init trans server since %s", terrstr()); + goto _OVER; + } + + if (dndInitClient(pDnode) != 0) { + dError("failed to init trans client since %s", terrstr()); + goto _OVER; + } + + dmGetMgmtFp(&pDnode->wrappers[DNODE]); + mmGetMgmtFp(&pDnode->wrappers[MNODE]); + vmGetMgmtFp(&pDnode->wrappers[VNODES]); + qmGetMgmtFp(&pDnode->wrappers[QNODE]); + smGetMgmtFp(&pDnode->wrappers[SNODE]); + bmGetMgmtFp(&pDnode->wrappers[BNODE]); + + if (dndInitMsgHandle(pDnode) != 0) { + goto _OVER; + } + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + snprintf(path, sizeof(path), "%s%s%s", pDnode->dataDir, TD_DIRSEP, pWrapper->name); + pWrapper->path = strdup(path); + pWrapper->pDnode = pDnode; + if (pWrapper->path == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + pWrapper->procType = PROC_SINGLE; + taosInitRWLatch(&pWrapper->latch); + } + + code = 0; + +_OVER: + if (code != 0 && pDnode) { + dndClearMemory(pDnode); + dError("failed to create dnode object since %s", terrstr()); + } else { + dInfo("dnode object is created, data:%p", pDnode); + } + + return pDnode; +} + +void dndClose(SDnode *pDnode) { + if (pDnode == NULL) return; + + if (dndGetStatus(pDnode) == DND_STAT_STOPPED) { + dError("dnode is shutting down, data:%p", pDnode); + return; + } + + dInfo("start to close dnode, data:%p", pDnode); + dndSetStatus(pDnode, DND_STAT_STOPPED); + + dndCleanupServer(pDnode); + dndCleanupClient(pDnode); + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + dndCloseNode(pWrapper); + } + + dndClearMemory(pDnode); + dInfo("dnode object is closed, data:%p", pDnode); +} + +void dndHandleEvent(SDnode *pDnode, EDndEvent event) { + dInfo("dnode object receive event %d, data:%p", event, pDnode); + pDnode->event = event; +} + +SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType ntype) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; + SMgmtWrapper *pRetWrapper = pWrapper; + + taosRLockLatch(&pWrapper->latch); + if (pWrapper->deployed) { + int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1); + dTrace("node:%s, is acquired, refCount:%d", pWrapper->name, refCount); + } else { + terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + pRetWrapper = NULL; + } + taosRUnLockLatch(&pWrapper->latch); + + return pRetWrapper; +} + +int32_t dndMarkWrapper(SMgmtWrapper *pWrapper) { + int32_t code = 0; + + taosRLockLatch(&pWrapper->latch); + if (pWrapper->deployed) { + int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1); + dTrace("node:%s, is marked, refCount:%d", pWrapper->name, refCount); + } else { + terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + code = -1; + } + taosRUnLockLatch(&pWrapper->latch); + + return code; +} + +void dndReleaseWrapper(SMgmtWrapper *pWrapper) { + if (pWrapper == NULL) return; + + taosRLockLatch(&pWrapper->latch); + int32_t refCount = atomic_sub_fetch_32(&pWrapper->refCount, 1); + taosRUnLockLatch(&pWrapper->latch); + dTrace("node:%s, is released, refCount:%d", pWrapper->name, refCount); +} \ No newline at end of file diff --git a/source/dnode/mgmt/container/src/dndTransport.c b/source/dnode/mgmt/container/src/dndTransport.c new file mode 100644 index 0000000000..2593a762e8 --- /dev/null +++ b/source/dnode/mgmt/container/src/dndTransport.c @@ -0,0 +1,321 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +#define INTERNAL_USER "_dnd" +#define INTERNAL_CKEY "_key" +#define INTERNAL_SECRET "_pwd" + +static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { + SDnode *pDnode = parent; + STransMgmt *pMgmt = &pDnode->trans; + tmsg_t msgType = pRsp->msgType; + + if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { + // if (pRsp == NULL || pRsp->pCont == NULL) return; + dTrace("rsp:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle); + rpcFreeCont(pRsp->pCont); + return; + } + + SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; + if (pHandle->msgFp != NULL) { + dTrace("rsp:%s will be processed by %s, handle:%p app:%p code:0x%04x:%s", TMSG_INFO(msgType), + pHandle->pWrapper->name, pRsp->handle, pRsp->ahandle, pRsp->code & 0XFFFF, tstrerror(pRsp->code)); + dndProcessRpcMsg(pHandle->pWrapper, pRsp, pEpSet); + } else { + dError("rsp:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle); + rpcFreeCont(pRsp->pCont); + } +} + +int32_t dndInitClient(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.label = "DND"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = dndProcessResponse; + rpcInit.sessions = 1024; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.user = INTERNAL_USER; + rpcInit.ckey = INTERNAL_CKEY; + rpcInit.spi = 1; + rpcInit.parent = pDnode; + + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); + rpcInit.secret = pass; + + pMgmt->clientRpc = rpcOpen(&rpcInit); + if (pMgmt->clientRpc == NULL) { + dError("failed to init dnode rpc client"); + return -1; + } + + dDebug("dnode rpc client is initialized"); + return 0; +} + +void dndCleanupClient(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + if (pMgmt->clientRpc) { + rpcClose(pMgmt->clientRpc); + pMgmt->clientRpc = NULL; + dDebug("dnode rpc client is closed"); + } +} + +static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) { + SDnode *pDnode = param; + STransMgmt *pMgmt = &pDnode->trans; + tmsg_t msgType = pReq->msgType; + + if (msgType == TDMT_DND_NETWORK_TEST) { + dTrace("network test req will be processed, handle:%p, app:%p", pReq->handle, pReq->ahandle); + dndProcessStartupReq(pDnode, pReq); + return; + } + + if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { + dError("req:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); + SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pReq->ahandle}; + rpcSendResponse(&rspMsg); + rpcFreeCont(pReq->pCont); + return; + } + + if (pReq->pCont == NULL) { + dTrace("req:%s not processed since its empty, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); + SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN, .ahandle = pReq->ahandle}; + rpcSendResponse(&rspMsg); + return; + } + + SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; + if (pHandle->msgFp != NULL) { + dTrace("req:%s will be processed by %s, handle:%p app:%p", TMSG_INFO(msgType), pHandle->pWrapper->name, + pReq->handle, pReq->ahandle); + dndProcessRpcMsg(pHandle->pWrapper, pReq, pEpSet); + } else { + dError("req:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); + SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pReq->ahandle}; + rpcSendResponse(&rspMsg); + rpcFreeCont(pReq->pCont); + } +} + +static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRpcRsp) { + STransMgmt *pMgmt = &pDnode->trans; + + SEpSet epSet = {0}; + dmGetMnodeEpSet(dndAcquireWrapper(pDnode, DNODE)->pMgmt, &epSet); + rpcSendRecv(pMgmt->clientRpc, &epSet, pRpcMsg, pRpcRsp); +} + +static int32_t dndGetHideUserAuth(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { + int32_t code = 0; + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + + if (strcmp(user, INTERNAL_USER) == 0) { + taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); + } else if (strcmp(user, TSDB_NETTEST_USER) == 0) { + taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); + } else { + code = -1; + } + + if (code == 0) { + memcpy(secret, pass, TSDB_PASSWORD_LEN); + *spi = 1; + *encrypt = 0; + *ckey = 0; + } + + return code; +} + +static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey) { + SDnode *pDnode = parent; + + if (dndGetHideUserAuth(parent, user, spi, encrypt, secret, ckey) == 0) { + dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); + return 0; + } + + if (mmGetUserAuth(dndAcquireWrapper(pDnode, MNODE), user, spi, encrypt, secret, ckey) == 0) { + dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); + return 0; + } + + if (terrno != TSDB_CODE_APP_NOT_READY) { + dTrace("failed to get user auth from mnode since %s", terrstr()); + return -1; + } + + SAuthReq authReq = {0}; + tstrncpy(authReq.user, user, TSDB_USER_LEN); + int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); + void *pReq = rpcMallocCont(contLen); + tSerializeSAuthReq(pReq, contLen, &authReq); + + SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; + SRpcMsg rpcRsp = {0}; + dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, authReq.spi, authReq.encrypt); + dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp); + + if (rpcRsp.code != 0) { + terrno = rpcRsp.code; + dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr()); + } else { + SAuthRsp authRsp = {0}; + tDeserializeSAuthReq(rpcRsp.pCont, rpcRsp.contLen, &authRsp); + memcpy(secret, authRsp.secret, TSDB_PASSWORD_LEN); + memcpy(ckey, authRsp.ckey, TSDB_PASSWORD_LEN); + *spi = authRsp.spi; + *encrypt = authRsp.encrypt; + dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, authRsp.spi, + authRsp.encrypt); + } + + rpcFreeCont(rpcRsp.pCont); + return rpcRsp.code; +} + +int32_t dndInitServer(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + + int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); + if (numOfThreads < 1) { + numOfThreads = 1; + } + + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = pDnode->serverPort; + rpcInit.label = "DND"; + rpcInit.numOfThreads = numOfThreads; + rpcInit.cfp = dndProcessRequest; + rpcInit.sessions = tsMaxShellConns; + rpcInit.connType = TAOS_CONN_SERVER; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.afp = dndRetrieveUserAuthInfo; + rpcInit.parent = pDnode; + + pMgmt->serverRpc = rpcOpen(&rpcInit); + if (pMgmt->serverRpc == NULL) { + dError("failed to init dnode rpc server"); + return -1; + } + + dDebug("dnode rpc server is initialized"); + return 0; +} + +void dndCleanupServer(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + if (pMgmt->serverRpc) { + rpcClose(pMgmt->serverRpc); + pMgmt->serverRpc = NULL; + dDebug("dnode rpc server is closed"); + } +} + +int32_t dndInitMsgHandle(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + + for (int32_t msgIndex = 0; msgIndex < TDMT_MAX; ++msgIndex) { + NodeMsgFp msgFp = pWrapper->msgFps[msgIndex]; + if (msgFp == NULL) continue; + + SMsgHandle *pHandle = &pMgmt->msgHandles[msgIndex]; + if (pHandle->msgFp != NULL) { + dError("msg:%s has multiple process nodes, prev node:%s, curr node:%s", tMsgInfo[msgIndex], + pHandle->pWrapper->name, pWrapper->name); + return -1; + } else { + dTrace("msg:%s will be processed by %s", tMsgInfo[msgIndex], pWrapper->name); + pHandle->msgFp = msgFp; + pHandle->pWrapper = pWrapper; + } + } + } + + return 0; +} + +static int32_t dndSendRpcReq(STransMgmt *pMgmt, SEpSet *pEpSet, SRpcMsg *pReq) { + if (pMgmt->clientRpc == NULL) { + terrno = TSDB_CODE_DND_OFFLINE; + return -1; + } + + rpcSendRequest(pMgmt->clientRpc, pEpSet, pReq, NULL); + return 0; +} + +int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pReq) { + if (pWrapper->procType == PROC_CHILD) { + } else { + SDnode *pDnode = pWrapper->pDnode; + if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { + terrno = TSDB_CODE_DND_OFFLINE; + dError("failed to send rpc msg since %s, handle:%p", terrstr(), pReq->handle); + return -1; + } + return dndSendRpcReq(&pDnode->trans, pEpSet, pReq); + } +} + +int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pReq) { + if (pWrapper->procType == PROC_CHILD) { + } else { + SDnode *pDnode = pWrapper->pDnode; + STransMgmt *pTrans = &pDnode->trans; + SEpSet epSet = {0}; + dmGetMnodeEpSet(dndAcquireWrapper(pDnode, DNODE)->pMgmt, &epSet); + return dndSendRpcReq(pTrans, &epSet, pReq); + } +} + +void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) { + if (pRsp->code == TSDB_CODE_APP_NOT_READY) { + SMgmtWrapper *pDnodeWrapper = dndAcquireWrapper(pWrapper->pDnode, DNODE); + dmSendRedirectRsp(pDnodeWrapper->pMgmt, pRsp); + } else { + rpcSendResponse(pRsp); + } +} + +void dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) { + if (pWrapper->procType == PROC_CHILD) { + int32_t code = -1; + do { + code = taosProcPutToParentQueue(pWrapper->pProc, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen); + if (code != 0) { + taosMsleep(10); + } + } while (code != 0); + } else { + dndSendRpcRsp(pWrapper, pRsp); + } +} diff --git a/source/dnode/mgmt/impl/src/dndWorker.c b/source/dnode/mgmt/container/src/dndWorker.c similarity index 74% rename from source/dnode/mgmt/impl/src/dndWorker.c rename to source/dnode/mgmt/container/src/dndWorker.c index 38f8737b2b..5d99ec5f0d 100644 --- a/source/dnode/mgmt/impl/src/dndWorker.c +++ b/source/dnode/mgmt/container/src/dndWorker.c @@ -14,11 +14,11 @@ */ #define _DEFAULT_SOURCE -#include "dndWorker.h" +#include "dndInt.h" -int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum, +int32_t dndInitWorker(void *param, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum, int32_t maxNum, void *queueFp) { - if (pDnode == NULL || pWorker == NULL || name == NULL || minNum < 0 || maxNum <= 0 || queueFp == NULL) { + if (pWorker == NULL || name == NULL || minNum < 0 || maxNum <= 0 || queueFp == NULL) { terrno = TSDB_CODE_INVALID_PARA; return -1; } @@ -28,7 +28,7 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, c pWorker->minNum = minNum; pWorker->maxNum = maxNum; pWorker->queueFp = queueFp; - pWorker->pDnode = pDnode; + pWorker->param = param; if (pWorker->type == DND_WORKER_SINGLE) { SQWorkerPool *pPool = &pWorker->pool; @@ -39,7 +39,7 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, c terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pWorker->queue = tQWorkerAllocQueue(pPool, pDnode, (FItem)queueFp); + pWorker->queue = tQWorkerAllocQueue(pPool, param, (FItem)queueFp); if (pWorker->queue == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -52,7 +52,7 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, c terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pWorker->queue = tWWorkerAllocQueue(pPool, pDnode, (FItems)queueFp); + pWorker->queue = tWWorkerAllocQueue(pPool, param, (FItems)queueFp); if (pWorker->queue == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -65,6 +65,8 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, c } void dndCleanupWorker(SDnodeWorker *pWorker) { + if (pWorker->queue == NULL) return; + while (!taosQueueEmpty(pWorker->queue)) { taosMsleep(10); } @@ -79,31 +81,13 @@ void dndCleanupWorker(SDnodeWorker *pWorker) { } } -int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pCont, int32_t contLen) { +int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pMsg) { if (pWorker == NULL || pWorker->queue == NULL) { terrno = TSDB_CODE_INVALID_PARA; return -1; } - void *pMsg = NULL; - if (contLen != 0) { - pMsg = taosAllocateQitem(contLen); - if (pMsg != NULL) { - memcpy(pMsg, pCont, contLen); - } - } else { - pMsg = pCont; - } - - if (pMsg == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - if (taosWriteQitem(pWorker->queue, pMsg) != 0) { - if (contLen != 0) { - taosFreeQitem(pMsg); - } terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } diff --git a/source/dnode/mgmt/daemon/src/dmnCfg.c b/source/dnode/mgmt/daemon/src/dmnCfg.c deleted file mode 100644 index afb9ffbdd0..0000000000 --- a/source/dnode/mgmt/daemon/src/dmnCfg.c +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dmnInt.h" -#include "tconfig.h" - -SDnodeObjCfg dmnGetObjCfg() { - SConfig *pCfg = taosGetCfg(); - SDnodeObjCfg objCfg = {0}; - - objCfg.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32; - tstrncpy(objCfg.dataDir, tsDataDir, sizeof(objCfg.dataDir)); - tstrncpy(objCfg.firstEp, tsFirst, sizeof(objCfg.firstEp)); - tstrncpy(objCfg.secondEp, tsSecond, sizeof(objCfg.firstEp)); - objCfg.serverPort = tsServerPort; - tstrncpy(objCfg.localFqdn, tsLocalFqdn, sizeof(objCfg.localFqdn)); - snprintf(objCfg.localEp, sizeof(objCfg.localEp), "%s:%u", objCfg.localFqdn, objCfg.serverPort); - objCfg.pDisks = tsDiskCfg; - objCfg.numOfDisks = tsDiskCfgNum; - return objCfg; -} - -void dmnDumpCfg() { - SConfig *pCfg = taosGetCfg(); - cfgDumpCfg(pCfg, 0, 1); -} \ No newline at end of file diff --git a/source/dnode/mgmt/daemon/src/dmnMain.c b/source/dnode/mgmt/daemon/src/dmnMain.c deleted file mode 100644 index 3d4de18dcb..0000000000 --- a/source/dnode/mgmt/daemon/src/dmnMain.c +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dmnInt.h" - -static struct { - bool stop; - bool dumpConfig; - bool generateGrant; - bool printAuth; - bool printVersion; - char envFile[PATH_MAX]; - char apolloUrl[PATH_MAX]; -} dmn = {0}; - -static void dmnSigintHandle(int signum, void *info, void *ctx) { - uInfo("signal:%d is received", signum); - dmn.stop = true; -} - -static void dmnSetSignalHandle() { - taosSetSignal(SIGTERM, dmnSigintHandle); - taosSetSignal(SIGHUP, dmnSigintHandle); - taosSetSignal(SIGINT, dmnSigintHandle); - taosSetSignal(SIGABRT, dmnSigintHandle); - taosSetSignal(SIGBREAK, dmnSigintHandle); -} - -static void dmnWaitSignal() { - dmnSetSignalHandle(); - while (!dmn.stop) { - taosMsleep(100); - } -} - -static int32_t dmnParseOption(int32_t argc, char const *argv[]) { - for (int32_t i = 1; i < argc; ++i) { - if (strcmp(argv[i], "-c") == 0) { - if (i < argc - 1) { - if (strlen(argv[++i]) >= PATH_MAX) { - printf("config file path overflow"); - return -1; - } - tstrncpy(configDir, argv[i], PATH_MAX); - } else { - printf("'-c' requires a parameter, default is %s\n", configDir); - return -1; - } - } else if (strcmp(argv[i], "-C") == 0) { - dmn.dumpConfig = true; - } else if (strcmp(argv[i], "-k") == 0) { - dmn.generateGrant = true; - } else if (strcmp(argv[i], "-V") == 0) { - dmn.printVersion = true; - } else { - } - } - - return 0; -} - -int32_t dmnRunDnode() { - if (dndInit() != 0) { - uInfo("Failed to start TDengine, please check the log"); - return -1; - } - - SDnodeObjCfg objCfg = dmnGetObjCfg(); - SDnode *pDnode = dndCreate(&objCfg); - if (pDnode == NULL) { - uInfo("Failed to start TDengine, please check the log"); - return -1; - } - - uInfo("Started TDengine service successfully."); - dmnWaitSignal(); - uInfo("TDengine is shut down!"); - - dndClose(pDnode); - dndCleanup(); - taosCloseLog(); - taosCleanupCfg(); - return 0; -} - -int main(int argc, char const *argv[]) { - if (!taosCheckSystemIsSmallEnd()) { - uError("TDengine does not run on non-small-end machines."); - return -1; - } - - if (dmnParseOption(argc, argv) != 0) { - return -1; - } - - if (dmn.generateGrant) { - dmnGenerateGrant(); - return 0; - } - - if (dmn.printVersion) { - dmnPrintVersion(); - return 0; - } - - if (taosCreateLog("taosdlog", 1, configDir, dmn.envFile, dmn.apolloUrl, NULL, 0) != 0) { - uInfo("Failed to start TDengine since read config error"); - return -1; - } - - if (taosInitCfg(configDir, dmn.envFile, dmn.apolloUrl, NULL, 0) != 0) { - uInfo("Failed to start TDengine since read config error"); - return -1; - } - - if (dmn.dumpConfig) { - dmnDumpCfg(); - taosCleanupCfg(); - return 0; - } - - return dmnRunDnode(); -} diff --git a/source/dnode/mgmt/dnode/inc/dm.h b/source/dnode/mgmt/dnode/inc/dm.h new file mode 100644 index 0000000000..6c18d7969c --- /dev/null +++ b/source/dnode/mgmt/dnode/inc/dm.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_DNODE_H_ +#define _TD_DND_DNODE_H_ + +#include "dnd.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SDnodeMgmt SDnodeMgmt; + +void dmGetMgmtFp(SMgmtWrapper *pWrapper); +void dmInitMsgHandles(SMgmtWrapper *pWrapper); + +void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); +void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); +void dmSendRedirectRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_DNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/dnode/inc/dmInt.h b/source/dnode/mgmt/dnode/inc/dmInt.h new file mode 100644 index 0000000000..46e70727af --- /dev/null +++ b/source/dnode/mgmt/dnode/inc/dmInt.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_DNODE_INT_H_ +#define _TD_DND_DNODE_INT_H_ + +#include "dm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SDnodeMgmt { + int64_t dver; + int64_t updateTime; + int8_t statusSent; + SEpSet mnodeEpSet; + SHashObj *dnodeHash; + SArray *dnodeEps; + TdThread *threadId; + SRWLatch latch; + SDnodeWorker mgmtWorker; + SDnodeWorker statusWorker; + const char *path; + SDnode *pDnode; + SMgmtWrapper *pWrapper; +} SDnodeMgmt; + +// dmFile.c +int32_t dmReadFile(SDnodeMgmt *pMgmt); +int32_t dmWriteFile(SDnodeMgmt *pMgmt); +void dmUpdateDnodeEps(SDnodeMgmt *pMgmt, SArray *pDnodeEps); + +// dmMsg.c +void dmSendStatusReq(SDnodeMgmt *pMgmt); +int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); + +// dmWorker.c +int32_t dmStartWorker(SDnodeMgmt *pMgmt); +void dmStopWorker(SDnodeMgmt *pMgmt); +int32_t dmStartThread(SDnodeMgmt *pMgmt); +int32_t dmProcessMgmtMsg(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_DNODE_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/dnode/src/dmFile.c b/source/dnode/mgmt/dnode/src/dmFile.c new file mode 100644 index 0000000000..aa2247bcd5 --- /dev/null +++ b/source/dnode/mgmt/dnode/src/dmFile.c @@ -0,0 +1,290 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmInt.h" + +static void dmPrintDnodes(SDnodeMgmt *pMgmt); +static bool dmIsEpChanged(SDnodeMgmt *pMgmt, int32_t dnodeId, const char *ep); +static void dmResetDnodes(SDnodeMgmt *pMgmt, SArray *dnodeEps); + +int32_t dmReadFile(SDnodeMgmt *pMgmt) { + int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; + int32_t len = 0; + int32_t maxLen = 256 * 1024; + char *content = calloc(1, maxLen + 1); + cJSON *root = NULL; + char file[PATH_MAX]; + TdFilePtr pFile = NULL; + SDnode *pDnode = pMgmt->pDnode; + + pMgmt->dnodeEps = taosArrayInit(1, sizeof(SDnodeEp)); + if (pMgmt->dnodeEps == NULL) { + dError("failed to calloc dnodeEp array since %s", strerror(errno)); + goto PRASE_DNODE_OVER; + } + + snprintf(file, sizeof(file), "%s%sdnode.json", pMgmt->path, TD_DIRSEP); + pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { + dDebug("file %s not exist", file); + code = 0; + goto PRASE_DNODE_OVER; + } + + len = (int32_t)taosReadFile(pFile, content, maxLen); + if (len <= 0) { + dError("failed to read %s since content is null", file); + goto PRASE_DNODE_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s since invalid json format", file); + goto PRASE_DNODE_OVER; + } + + cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId"); + if (!dnodeId || dnodeId->type != cJSON_Number) { + dError("failed to read %s since dnodeId not found", file); + goto PRASE_DNODE_OVER; + } + pDnode->dnodeId = dnodeId->valueint; + + cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId"); + if (!clusterId || clusterId->type != cJSON_String) { + dError("failed to read %s since clusterId not found", file); + goto PRASE_DNODE_OVER; + } + pDnode->clusterId = atoll(clusterId->valuestring); + + cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); + if (!dropped || dropped->type != cJSON_Number) { + dError("failed to read %s since dropped not found", file); + goto PRASE_DNODE_OVER; + } + pDnode->dropped = dropped->valueint; + + cJSON *dnodes = cJSON_GetObjectItem(root, "dnodes"); + if (!dnodes || dnodes->type != cJSON_Array) { + dError("failed to read %s since dnodes not found", file); + goto PRASE_DNODE_OVER; + } + + int32_t numOfDnodes = cJSON_GetArraySize(dnodes); + if (numOfDnodes <= 0) { + dError("failed to read %s since numOfDnodes:%d invalid", file, numOfDnodes); + goto PRASE_DNODE_OVER; + } + + for (int32_t i = 0; i < numOfDnodes; ++i) { + cJSON *node = cJSON_GetArrayItem(dnodes, i); + if (node == NULL) break; + + SDnodeEp dnodeEp = {0}; + + cJSON *did = cJSON_GetObjectItem(node, "id"); + if (!did || did->type != cJSON_Number) { + dError("failed to read %s since dnodeId not found", file); + goto PRASE_DNODE_OVER; + } + + dnodeEp.id = dnodeId->valueint; + + cJSON *dnodeFqdn = cJSON_GetObjectItem(node, "fqdn"); + if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) { + dError("failed to read %s since dnodeFqdn not found", file); + goto PRASE_DNODE_OVER; + } + tstrncpy(dnodeEp.ep.fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN); + + cJSON *dnodePort = cJSON_GetObjectItem(node, "port"); + if (!dnodePort || dnodePort->type != cJSON_Number) { + dError("failed to read %s since dnodePort not found", file); + goto PRASE_DNODE_OVER; + } + + dnodeEp.ep.port = dnodePort->valueint; + + cJSON *isMnode = cJSON_GetObjectItem(node, "isMnode"); + if (!isMnode || isMnode->type != cJSON_Number) { + dError("failed to read %s since isMnode not found", file); + goto PRASE_DNODE_OVER; + } + dnodeEp.isMnode = isMnode->valueint; + + taosArrayPush(pMgmt->dnodeEps, &dnodeEp); + } + + code = 0; + dInfo("succcessed to read file %s", file); + dmPrintDnodes(pMgmt); + +PRASE_DNODE_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (pFile != NULL) taosCloseFile(&pFile); + + if (dmIsEpChanged(pMgmt, pDnode->dnodeId, pDnode->localEp)) { + dError("localEp %s different with %s and need reconfigured", pDnode->localEp, file); + return -1; + } + + if (taosArrayGetSize(pMgmt->dnodeEps) == 0) { + SDnodeEp dnodeEp = {0}; + dnodeEp.isMnode = 1; + taosGetFqdnPortFromEp(pDnode->firstEp, &dnodeEp.ep); + taosArrayPush(pMgmt->dnodeEps, &dnodeEp); + } + + dmResetDnodes(pMgmt, pMgmt->dnodeEps); + + terrno = code; + return code; +} + +int32_t dmWriteFile(SDnodeMgmt *pMgmt) { + SDnode *pDnode = pMgmt->pDnode; + + char file[PATH_MAX]; + snprintf(file, sizeof(file), "%s%sdnode.json.bak", pMgmt->path, TD_DIRSEP); + + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + dError("failed to write %s since %s", file, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 256 * 1024; + char *content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", pDnode->dnodeId); + len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%" PRId64 "\",\n", pDnode->clusterId); + len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pDnode->dropped); + len += snprintf(content + len, maxLen - len, " \"dnodes\": [{\n"); + + int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->dnodeEps); + for (int32_t i = 0; i < numOfEps; ++i) { + SDnodeEp *pDnodeEp = taosArrayGet(pMgmt->dnodeEps, i); + len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pDnodeEp->id); + len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pDnodeEp->ep.fqdn); + len += snprintf(content + len, maxLen - len, " \"port\": %u,\n", pDnodeEp->ep.port); + len += snprintf(content + len, maxLen - len, " \"isMnode\": %d\n", pDnodeEp->isMnode); + if (i < numOfEps - 1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }]\n"); + } + } + len += snprintf(content + len, maxLen - len, "}\n"); + + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); + free(content); + + char realfile[PATH_MAX]; + snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP); + + if (taosRenameFile(file, realfile) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to rename %s since %s", file, terrstr()); + return -1; + } + + pMgmt->updateTime = taosGetTimestampMs(); + dDebug("successed to write %s", file); + return 0; +} + +void dmUpdateDnodeEps(SDnodeMgmt *pMgmt, SArray *dnodeEps) { + int32_t numOfEps = taosArrayGetSize(dnodeEps); + if (numOfEps <= 0) return; + + taosWLockLatch(&pMgmt->latch); + + int32_t numOfEpsOld = (int32_t)taosArrayGetSize(pMgmt->dnodeEps); + if (numOfEps != numOfEpsOld) { + dmResetDnodes(pMgmt, dnodeEps); + dmWriteFile(pMgmt); + } else { + int32_t size = numOfEps * sizeof(SDnodeEp); + if (memcmp(pMgmt->dnodeEps->pData, dnodeEps->pData, size) != 0) { + dmResetDnodes(pMgmt, dnodeEps); + dmWriteFile(pMgmt); + } + } + + taosWUnLockLatch(&pMgmt->latch); +} + +static void dmResetDnodes(SDnodeMgmt *pMgmt, SArray *dnodeEps) { + if (pMgmt->dnodeEps != dnodeEps) { + SArray *tmp = pMgmt->dnodeEps; + pMgmt->dnodeEps = taosArrayDup(dnodeEps); + taosArrayDestroy(tmp); + } + + pMgmt->mnodeEpSet.inUse = 0; + pMgmt->mnodeEpSet.numOfEps = 0; + + int32_t mIndex = 0; + int32_t numOfEps = (int32_t)taosArrayGetSize(dnodeEps); + + for (int32_t i = 0; i < numOfEps; i++) { + SDnodeEp *pDnodeEp = taosArrayGet(dnodeEps, i); + if (!pDnodeEp->isMnode) continue; + if (mIndex >= TSDB_MAX_REPLICA) continue; + pMgmt->mnodeEpSet.numOfEps++; + + pMgmt->mnodeEpSet.eps[mIndex] = pDnodeEp->ep; + mIndex++; + } + + for (int32_t i = 0; i < numOfEps; i++) { + SDnodeEp *pDnodeEp = taosArrayGet(dnodeEps, i); + taosHashPut(pMgmt->dnodeHash, &pDnodeEp->id, sizeof(int32_t), pDnodeEp, sizeof(SDnodeEp)); + } + + dmPrintDnodes(pMgmt); +} + +static void dmPrintDnodes(SDnodeMgmt *pMgmt) { + int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->dnodeEps); + dDebug("print dnode ep list, num:%d", numOfEps); + for (int32_t i = 0; i < numOfEps; i++) { + SDnodeEp *pEp = taosArrayGet(pMgmt->dnodeEps, i); + dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", pEp->id, pEp->ep.fqdn, pEp->ep.port, pEp->isMnode); + } +} + +static bool dmIsEpChanged(SDnodeMgmt *pMgmt, int32_t dnodeId, const char *ep) { + bool changed = false; + taosRLockLatch(&pMgmt->latch); + + SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t)); + if (pDnodeEp != NULL) { + char epstr[TSDB_EP_LEN + 1]; + snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port); + changed = strcmp(ep, epstr) != 0; + } + + taosRUnLockLatch(&pMgmt->latch); + return changed; +} diff --git a/source/dnode/mgmt/dnode/src/dmInt.c b/source/dnode/mgmt/dnode/src/dmInt.c new file mode 100644 index 0000000000..1746cbe6e1 --- /dev/null +++ b/source/dnode/mgmt/dnode/src/dmInt.c @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmInt.h" + +void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet) { + taosRLockLatch(&pMgmt->latch); + *pEpSet = pMgmt->mnodeEpSet; + taosRUnLockLatch(&pMgmt->latch); +} + +void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet) { + dInfo("mnode is changed, num:%d use:%d", pEpSet->numOfEps, pEpSet->inUse); + + taosWLockLatch(&pMgmt->latch); + pMgmt->mnodeEpSet = *pEpSet; + for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { + dInfo("mnode index:%d %s:%u", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + } + + taosWUnLockLatch(&pMgmt->latch); +} + +void dmGetDnodeEp(SMgmtWrapper *pWrapper, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort) { + SDnodeMgmt *pMgmt = pWrapper->pMgmt; + taosRLockLatch(&pMgmt->latch); + + SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t)); + if (pDnodeEp != NULL) { + if (pPort != NULL) { + *pPort = pDnodeEp->ep.port; + } + if (pFqdn != NULL) { + tstrncpy(pFqdn, pDnodeEp->ep.fqdn, TSDB_FQDN_LEN); + } + if (pEp != NULL) { + snprintf(pEp, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port); + } + } + + taosRUnLockLatch(&pMgmt->latch); +} + +void dmSendRedirectRsp(SDnodeMgmt *pMgmt, SRpcMsg *pReq) { + SDnode *pDnode = pMgmt->pDnode; + + SEpSet epSet = {0}; + dmGetMnodeEpSet(pMgmt, &epSet); + + dDebug("RPC %p, req is redirected, num:%d use:%d", pReq->handle, epSet.numOfEps, epSet.inUse); + for (int32_t i = 0; i < epSet.numOfEps; ++i) { + dDebug("mnode index:%d %s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); + if (strcmp(epSet.eps[i].fqdn, pDnode->localFqdn) == 0 && epSet.eps[i].port == pDnode->serverPort) { + epSet.inUse = (i + 1) % epSet.numOfEps; + } + + epSet.eps[i].port = htons(epSet.eps[i].port); + } + + rpcSendRedirectRsp(pReq->handle, &epSet); +} + +static int32_t dmStart(SMgmtWrapper *pWrapper) { + dDebug("dnode mgmt start to run"); + return dmStartThread(pWrapper->pMgmt); +} + +int32_t dmInit(SMgmtWrapper *pWrapper) { + SDnode *pDnode = pWrapper->pDnode; + SDnodeMgmt *pMgmt = calloc(1, sizeof(SDnodeMgmt)); + dInfo("dnode-mgmt is initialized"); + + pDnode->dnodeId = 0; + pDnode->dropped = 0; + pDnode->clusterId = 0; + pMgmt->path = pWrapper->path; + pMgmt->pDnode = pDnode; + pMgmt->pWrapper = pWrapper; + taosInitRWLatch(&pMgmt->latch); + + pMgmt->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + if (pMgmt->dnodeHash == NULL) { + dError("failed to init dnode hash"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (dmReadFile(pMgmt) != 0) { + dError("failed to read file since %s", terrstr()); + return -1; + } + + if (pDnode->dropped) { + dError("dnode will not start since its already dropped"); + return -1; + } + + if (dmStartWorker(pMgmt) != 0) { + return -1; + } + + pWrapper->pMgmt = pMgmt; + dInfo("dnode-mgmt is initialized"); + return 0; +} + +void dmCleanup(SMgmtWrapper *pWrapper) { + SDnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return; + + dInfo("dnode-mgmt start to clean up"); + dmStopWorker(pMgmt); + + taosWLockLatch(&pMgmt->latch); + + if (pMgmt->dnodeEps != NULL) { + taosArrayDestroy(pMgmt->dnodeEps); + pMgmt->dnodeEps = NULL; + } + + if (pMgmt->dnodeHash != NULL) { + taosHashCleanup(pMgmt->dnodeHash); + pMgmt->dnodeHash = NULL; + } + + taosWUnLockLatch(&pMgmt->latch); + + free(pMgmt); + pWrapper->pMgmt = NULL; + dInfo("dnode-mgmt is cleaned up"); +} + +int32_t dmRequire(SMgmtWrapper *pWrapper, bool *required) { + *required = true; + return 0; +} + +void dmGetMgmtFp(SMgmtWrapper *pWrapper) { + SMgmtFp mgmtFp = {0}; + mgmtFp.openFp = dmInit; + mgmtFp.closeFp = dmCleanup; + mgmtFp.startFp = dmStart; + mgmtFp.requiredFp = dmRequire; + + dmInitMsgHandles(pWrapper); + pWrapper->name = "dnode"; + pWrapper->fp = mgmtFp; +} diff --git a/source/dnode/mgmt/dnode/src/dmMsg.c b/source/dnode/mgmt/dnode/src/dmMsg.c new file mode 100644 index 0000000000..f451c2ab05 --- /dev/null +++ b/source/dnode/mgmt/dnode/src/dmMsg.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmInt.h" +#include "vm.h" + +void dmSendStatusReq(SDnodeMgmt *pMgmt) { + SDnode *pDnode = pMgmt->pDnode; + SStatusReq req = {0}; + + taosRLockLatch(&pMgmt->latch); + req.sver = tsVersion; + req.dver = pMgmt->dver; + req.dnodeId = pDnode->dnodeId; + req.clusterId = pDnode->clusterId; + req.rebootTime = pDnode->rebootTime; + req.updateTime = pMgmt->updateTime; + req.numOfCores = tsNumOfCores; + req.numOfSupportVnodes = pDnode->numOfSupportVnodes; + tstrncpy(req.dnodeEp, pDnode->localEp, TSDB_EP_LEN); + + req.clusterCfg.statusInterval = tsStatusInterval; + req.clusterCfg.checkTime = 0; + char timestr[32] = "1970-01-01 00:00:00.00"; + (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + memcpy(req.clusterCfg.timezone, tsTimezone, TD_TIMEZONE_LEN); + memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN); + memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN); + taosRUnLockLatch(&pMgmt->latch); + + req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad)); + vmMonitorVnodeLoads(dndAcquireWrapper(pDnode, VNODES), req.pVloads); + + int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); + void *pHead = rpcMallocCont(contLen); + tSerializeSStatusReq(pHead, contLen, &req); + taosArrayDestroy(req.pVloads); + + SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)9527}; + pMgmt->statusSent = 1; + + dTrace("send req:%s to mnode, app:%p", TMSG_INFO(rpcMsg.msgType), rpcMsg.ahandle); + dndSendReqToMnode(pMgmt->pWrapper, &rpcMsg); +} + +static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) { + SDnode *pDnode = pMgmt->pDnode; + + if (pDnode->dnodeId == 0) { + dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId); + taosWLockLatch(&pMgmt->latch); + pDnode->dnodeId = pCfg->dnodeId; + pDnode->clusterId = pCfg->clusterId; + dmWriteFile(pMgmt); + taosWUnLockLatch(&pMgmt->latch); + } +} + +int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SDnode *pDnode = pMgmt->pDnode; + SRpcMsg *pRsp = &pMsg->rpcMsg; + + if (pRsp->code != TSDB_CODE_SUCCESS) { + if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pDnode->dropped && pDnode->dnodeId > 0) { + dInfo("dnode:%d, set to dropped since not exist in mnode", pDnode->dnodeId); + pDnode->dropped = 1; + dmWriteFile(pMgmt); + } + } else { + SStatusRsp statusRsp = {0}; + if (pRsp->pCont != NULL && pRsp->contLen != 0 && + tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) { + pMgmt->dver = statusRsp.dver; + dmUpdateDnodeCfg(pMgmt, &statusRsp.dnodeCfg); + dmUpdateDnodeEps(pMgmt, statusRsp.pDnodeEps); + } + tFreeSStatusRsp(&statusRsp); + } + + pMgmt->statusSent = 0; +} + +int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pRsp = &pMsg->rpcMsg; + dError("auth rsp is received, but not supported yet"); + return 0; +} + +int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pRsp = &pMsg->rpcMsg; + dError("grant rsp is received, but not supported yet"); + return 0; +} + +int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pReq = &pMsg->rpcMsg; + SDCfgDnodeReq *pCfg = pReq->pCont; + dError("config req is received, but not supported yet"); + return TSDB_CODE_OPS_NOT_SUPPORT; +} + +void dmInitMsgHandles(SMgmtWrapper *pWrapper) { + // Requests handled by DNODE + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, (NodeMsgFp)dmProcessMgmtMsg); + + // Requests handled by MNODE + dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, (NodeMsgFp)dmProcessMgmtMsg); +} diff --git a/source/dnode/mgmt/dnode/src/dmWorker.c b/source/dnode/mgmt/dnode/src/dmWorker.c new file mode 100644 index 0000000000..e06cbf4351 --- /dev/null +++ b/source/dnode/mgmt/dnode/src/dmWorker.c @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "bm.h" +#include "dmInt.h" +#include "mm.h" +#include "qm.h" +#include "sm.h" +#include "vm.h" + +static void *dmThreadRoutine(void *param) { + SDnodeMgmt *pMgmt = param; + SDnode *pDnode = pMgmt->pDnode; + int64_t lastStatusTime = taosGetTimestampMs(); + int64_t lastMonitorTime = lastStatusTime; + + setThreadName("dnode-hb"); + + while (true) { + taosThreadTestCancel(); + taosMsleep(200); + if (dndGetStatus(pDnode) != DND_STAT_RUNNING || pDnode->dropped) { + continue; + } + + int64_t curTime = taosGetTimestampMs(); + + float statusInterval = (curTime - lastStatusTime) / 1000.0f; + if (statusInterval >= tsStatusInterval && !pMgmt->statusSent) { + dmSendStatusReq(pMgmt); + lastStatusTime = curTime; + } + + float monitorInterval = (curTime - lastMonitorTime) / 1000.0f; + if (monitorInterval >= tsMonitorInterval) { + dndSendMonitorReport(pDnode); + lastMonitorTime = curTime; + } + } +} + +static void dmProcessQueue(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SDnode *pDnode = pMgmt->pDnode; + SRpcMsg *pRpc = &pMsg->rpcMsg; + int32_t code = -1; + dTrace("msg:%p, will be processed in dnode queue", pMsg); + + switch (pRpc->msgType) { + case TDMT_DND_CREATE_MNODE: + case TDMT_DND_CREATE_QNODE: + case TDMT_DND_CREATE_SNODE: + case TDMT_DND_CREATE_BNODE: + case TDMT_DND_DROP_MNODE: + case TDMT_DND_DROP_QNODE: + case TDMT_DND_DROP_SNODE: + case TDMT_DND_DROP_BNODE: + code = dndProcessNodeMsg(pMgmt->pDnode, pMsg); + break; + case TDMT_DND_CONFIG_DNODE: + code = dmProcessConfigReq(pMgmt, pMsg); + break; + case TDMT_MND_STATUS_RSP: + code = dmProcessStatusRsp(pMgmt, pMsg); + break; + case TDMT_MND_AUTH_RSP: + code = dmProcessAuthRsp(pMgmt, pMsg); + break; + case TDMT_MND_GRANT_RSP: + code = dmProcessGrantRsp(pMgmt, pMsg); + break; + default: + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + dError("msg:%p, type:%s not processed in dnode queue", pRpc->handle, TMSG_INFO(pRpc->msgType)); + } + + if (pRpc->msgType & 1u) { + if (code != 0) code = terrno; + SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code}; + rpcSendResponse(&rsp); + } + + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); +} + +int32_t dmStartWorker(SDnodeMgmt *pMgmt) { + if (dndInitWorker(pMgmt, &pMgmt->mgmtWorker, DND_WORKER_SINGLE, "dnode-mgmt", 1, 1, dmProcessQueue) != 0) { + dError("failed to start dnode mgmt worker since %s", terrstr()); + return -1; + } + + if (dndInitWorker(pMgmt, &pMgmt->statusWorker, DND_WORKER_SINGLE, "dnode-status", 1, 1, dmProcessQueue) != 0) { + dError("failed to start dnode mgmt worker since %s", terrstr()); + return -1; + } + + return 0; +} + +int32_t dmStartThread(SDnodeMgmt *pMgmt) { + pMgmt->threadId = taosCreateThread(dmThreadRoutine, pMgmt); + if (pMgmt->threadId == NULL) { + dError("failed to init dnode thread"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +void dmStopWorker(SDnodeMgmt *pMgmt) { + dndCleanupWorker(&pMgmt->mgmtWorker); + dndCleanupWorker(&pMgmt->statusWorker); + + if (pMgmt->threadId != NULL) { + taosDestoryThread(pMgmt->threadId); + pMgmt->threadId = NULL; + } +} + +int32_t dmProcessMgmtMsg(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SDnodeWorker *pWorker = &pMgmt->mgmtWorker; + if (pMsg->rpcMsg.msgType == TDMT_MND_STATUS_RSP) { + pWorker = &pMgmt->statusWorker; + } + + dTrace("msg:%p, will be written to worker %s", pMsg, pWorker->name); + return dndWriteMsgToWorker(pWorker, pMsg); +} diff --git a/source/dnode/mgmt/impl/CMakeLists.txt b/source/dnode/mgmt/impl/CMakeLists.txt deleted file mode 100644 index c99a405527..0000000000 --- a/source/dnode/mgmt/impl/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -aux_source_directory(src DNODE_SRC) -add_library(dnode STATIC ${DNODE_SRC}) -target_link_libraries( - dnode cjson mnode vnode qnode snode bnode wal sync taos tfs monitor -) -target_include_directories( - dnode - PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) - -if(${BUILD_TEST}) - add_subdirectory(test) -endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/dnode/mgmt/impl/inc/dndEnv.h b/source/dnode/mgmt/impl/inc/dndEnv.h deleted file mode 100644 index aeea5386b4..0000000000 --- a/source/dnode/mgmt/impl/inc/dndEnv.h +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_ENV_H_ -#define _TD_DND_ENV_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "dndInt.h" - -typedef struct { - EWorkerType type; - const char *name; - int32_t minNum; - int32_t maxNum; - void *queueFp; - SDnode *pDnode; - STaosQueue *queue; - union { - SQWorkerPool pool; - SWWorkerPool mpool; - }; -} SDnodeWorker; - -typedef struct { - char *dnode; - char *mnode; - char *snode; - char *bnode; - char *vnodes; -} SDnodeDir; - -typedef struct { - int32_t dnodeId; - int32_t dropped; - int64_t clusterId; - int64_t dver; - int64_t rebootTime; - int64_t updateTime; - int8_t statusSent; - SEpSet mnodeEpSet; - char *file; - SHashObj *dnodeHash; - SArray *pDnodeEps; - pthread_t *threadId; - SRWLatch latch; - SDnodeWorker mgmtWorker; - SDnodeWorker statusWorker; -} SDnodeMgmt; - -typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - SMnode *pMnode; - SRWLatch latch; - SDnodeWorker readWorker; - SDnodeWorker writeWorker; - SDnodeWorker syncWorker; - int8_t replica; - int8_t selfIndex; - SReplica replicas[TSDB_MAX_REPLICA]; -} SMnodeMgmt; - -typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - SQnode *pQnode; - SRWLatch latch; - SDnodeWorker queryWorker; - SDnodeWorker fetchWorker; -} SQnodeMgmt; - -typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - int8_t uniqueWorkerInUse; - SSnode *pSnode; - SRWLatch latch; - SArray *uniqueWorkers; // SArray - SDnodeWorker sharedWorker; -} SSnodeMgmt; - -typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - SBnode *pBnode; - SRWLatch latch; - SDnodeWorker writeWorker; -} SBnodeMgmt; - -typedef struct { - int32_t openVnodes; - int32_t totalVnodes; - int32_t masterNum; - int64_t numOfSelectReqs; - int64_t numOfInsertReqs; - int64_t numOfInsertSuccessReqs; - int64_t numOfBatchInsertReqs; - int64_t numOfBatchInsertSuccessReqs; -} SVnodesStat; - -typedef struct { - SVnodesStat stat; - SHashObj *hash; - SRWLatch latch; - SQWorkerPool queryPool; - SFWorkerPool fetchPool; - SWWorkerPool syncPool; - SWWorkerPool writePool; -} SVnodesMgmt; - -typedef struct { - void *serverRpc; - void *clientRpc; - DndMsgFp msgFp[TDMT_MAX]; -} STransMgmt; - -typedef struct SDnode { - EStat stat; - SDnodeObjCfg cfg; - SDnodeDir dir; - TdFilePtr pLockFile; - SDnodeMgmt dmgmt; - SMnodeMgmt mmgmt; - SQnodeMgmt qmgmt; - SSnodeMgmt smgmt; - SBnodeMgmt bmgmt; - SVnodesMgmt vmgmt; - STransMgmt tmgmt; - STfs *pTfs; - SStartupReq startup; -} SDnode; - -int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_ENV_H_*/ diff --git a/source/dnode/mgmt/impl/inc/dndMgmt.h b/source/dnode/mgmt/impl/inc/dndMgmt.h deleted file mode 100644 index 9cc0c4ae66..0000000000 --- a/source/dnode/mgmt/impl/inc/dndMgmt.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_DNODE_H_ -#define _TD_DND_DNODE_H_ - -#ifdef __cplusplus -extern "C" { -#endif -#include "dndEnv.h" - -int32_t dndInitMgmt(SDnode *pDnode); -void dndStopMgmt(SDnode *pDnode); -void dndCleanupMgmt(SDnode *pDnode); - -int32_t dndGetDnodeId(SDnode *pDnode); -int64_t dndGetClusterId(SDnode *pDnode); -void dndGetDnodeEp(SDnode *pDnode, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort); -void dndGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet); - -void dndSendRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg); -void dndSendStatusReq(SDnode *pDnode); -void dndProcessMgmtMsg(SDnode *pDnode, SRpcMsg *pRpcMsg, SEpSet *pEpSet); -void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_DNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/inc/dndMnode.h b/source/dnode/mgmt/impl/inc/dndMnode.h deleted file mode 100644 index 0f03bb3832..0000000000 --- a/source/dnode/mgmt/impl/inc/dndMnode.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_MNODE_H_ -#define _TD_DND_MNODE_H_ - -#ifdef __cplusplus -extern "C" { -#endif -#include "dndEnv.h" - -int32_t dndInitMnode(SDnode *pDnode); -void dndCleanupMnode(SDnode *pDnode); - -int32_t dndGetUserAuthFromMnode(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey); -void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); -int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); -int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); - -int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, - SMonGrantInfo *pGrantInfo); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_MNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/inc/dndSnode.h b/source/dnode/mgmt/impl/inc/dndSnode.h deleted file mode 100644 index f72d2a137a..0000000000 --- a/source/dnode/mgmt/impl/inc/dndSnode.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_SNODE_H_ -#define _TD_DND_SNODE_H_ - -#ifdef __cplusplus -extern "C" { -#endif -#include "dndEnv.h" - -int32_t dndInitSnode(SDnode *pDnode); -void dndCleanupSnode(SDnode *pDnode); - -// void dndProcessSnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); -int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); - -void dndProcessSnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessSnodeUniqueMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessSnodeSharedMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessSnodeExecMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_SNODE_H_*/ diff --git a/source/dnode/mgmt/impl/inc/dndVnodes.h b/source/dnode/mgmt/impl/inc/dndVnodes.h deleted file mode 100644 index 895e94060f..0000000000 --- a/source/dnode/mgmt/impl/inc/dndVnodes.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_VNODES_H_ -#define _TD_DND_VNODES_H_ - -#ifdef __cplusplus -extern "C" { -#endif -#include "dndEnv.h" - -int32_t dndInitVnodes(SDnode *pDnode); -void dndCleanupVnodes(SDnode *pDnode); -void dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads); -void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); - -int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq); -int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *pReq); -int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *pReq); -int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *pReq); -int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *pReq); -int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *pReq); - -int32_t dndPutReqToVQueryQ(SDnode *pDnode, SRpcMsg *pReq); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_VNODES_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndBnode.c b/source/dnode/mgmt/impl/src/dndBnode.c deleted file mode 100644 index 81b020c152..0000000000 --- a/source/dnode/mgmt/impl/src/dndBnode.c +++ /dev/null @@ -1,392 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndBnode.h" -#include "dndMgmt.h" -#include "dndTransport.h" -#include "dndWorker.h" - -static void dndProcessBnodeQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs); - -static SBnode *dndAcquireBnode(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - SBnode *pBnode = NULL; - int32_t refCount = 0; - - taosRLockLatch(&pMgmt->latch); - if (pMgmt->deployed && !pMgmt->dropped && pMgmt->pBnode != NULL) { - refCount = atomic_add_fetch_32(&pMgmt->refCount, 1); - pBnode = pMgmt->pBnode; - } else { - terrno = TSDB_CODE_DND_BNODE_NOT_DEPLOYED; - } - taosRUnLockLatch(&pMgmt->latch); - - if (pBnode != NULL) { - dTrace("acquire bnode, refCount:%d", refCount); - } - return pBnode; -} - -static void dndReleaseBnode(SDnode *pDnode, SBnode *pBnode) { - if (pBnode == NULL) return; - - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - taosRLockLatch(&pMgmt->latch); - int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); - taosRUnLockLatch(&pMgmt->latch); - dTrace("release bnode, refCount:%d", refCount); -} - -static int32_t dndReadBnodeFile(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - int32_t code = TSDB_CODE_DND_BNODE_READ_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "r"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); - if (pFile == NULL) { - dDebug("file %s not exist", file); - code = 0; - goto PRASE_BNODE_OVER; - } - - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", file); - goto PRASE_BNODE_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); - goto PRASE_BNODE_OVER; - } - - cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); - if (!deployed || deployed->type != cJSON_Number) { - dError("failed to read %s since deployed not found", file); - goto PRASE_BNODE_OVER; - } - pMgmt->deployed = deployed->valueint; - - cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", file); - goto PRASE_BNODE_OVER; - } - pMgmt->dropped = dropped->valueint; - - code = 0; - dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); - -PRASE_BNODE_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (pFile != NULL) taosCloseFile(&pFile); - - terrno = code; - return code; -} - -static int32_t dndWriteBnodeFile(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "w"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR; - dError("failed to write %s since %s", file, terrstr()); - return -1; - } - - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", pMgmt->deployed); - len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); - len += snprintf(content + len, maxLen - len, "}\n"); - - taosWriteFile(pFile, content, len); - taosFsyncFile(pFile); - taosCloseFile(&pFile); - free(content); - - char realfile[PATH_MAX + 20]; - snprintf(realfile, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); - - if (taosRenameFile(file, realfile) != 0) { - terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR; - dError("failed to rename %s since %s", file, terrstr()); - return -1; - } - - dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped); - return 0; -} - -static int32_t dndStartBnodeWorker(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_MULTI, "bnode-write", 0, 1, dndProcessBnodeQueue) != 0) { - dError("failed to start bnode write worker since %s", terrstr()); - return -1; - } - - return 0; -} - -static void dndStopBnodeWorker(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - - taosWLockLatch(&pMgmt->latch); - pMgmt->deployed = 0; - taosWUnLockLatch(&pMgmt->latch); - - while (pMgmt->refCount > 0) { - taosMsleep(10); - } - - dndCleanupWorker(&pMgmt->writeWorker); -} - -static void dndBuildBnodeOption(SDnode *pDnode, SBnodeOpt *pOption) { - pOption->pDnode = pDnode; - pOption->sendReqToDnodeFp = dndSendReqToDnode; - pOption->sendReqToMnodeFp = dndSendReqToMnode; - pOption->sendRedirectRspFp = dndSendRedirectRsp; - pOption->dnodeId = dndGetDnodeId(pDnode); - pOption->clusterId = dndGetClusterId(pDnode); - pOption->sver = tsVersion; -} - -static int32_t dndOpenBnode(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - SBnode *pBnode = dndAcquireBnode(pDnode); - if (pBnode != NULL) { - dndReleaseBnode(pDnode, pBnode); - terrno = TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED; - dError("failed to create bnode since %s", terrstr()); - return -1; - } - - SBnodeOpt option = {0}; - dndBuildBnodeOption(pDnode, &option); - - pBnode = bndOpen(pDnode->dir.bnode, &option); - if (pBnode == NULL) { - dError("failed to open bnode since %s", terrstr()); - return -1; - } - - if (dndStartBnodeWorker(pDnode) != 0) { - dError("failed to start bnode worker since %s", terrstr()); - bndClose(pBnode); - return -1; - } - - pMgmt->deployed = 1; - if (dndWriteBnodeFile(pDnode) != 0) { - pMgmt->deployed = 0; - dError("failed to write bnode file since %s", terrstr()); - dndStopBnodeWorker(pDnode); - bndClose(pBnode); - return -1; - } - - taosWLockLatch(&pMgmt->latch); - pMgmt->pBnode = pBnode; - taosWUnLockLatch(&pMgmt->latch); - - dInfo("bnode open successfully"); - return 0; -} - -static int32_t dndDropBnode(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - - SBnode *pBnode = dndAcquireBnode(pDnode); - if (pBnode == NULL) { - dError("failed to drop bnode since %s", terrstr()); - return -1; - } - - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 1; - taosRUnLockLatch(&pMgmt->latch); - - if (dndWriteBnodeFile(pDnode) != 0) { - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 0; - taosRUnLockLatch(&pMgmt->latch); - - dndReleaseBnode(pDnode, pBnode); - dError("failed to drop bnode since %s", terrstr()); - return -1; - } - - dndReleaseBnode(pDnode, pBnode); - dndStopBnodeWorker(pDnode); - pMgmt->deployed = 0; - dndWriteBnodeFile(pDnode); - bndClose(pBnode); - pMgmt->pBnode = NULL; - bndDestroy(pDnode->dir.bnode); - - return 0; -} - -int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDCreateBnodeReq createReq = {0}; - if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (createReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_BNODE_INVALID_OPTION; - dError("failed to create bnode since %s", terrstr()); - return -1; - } else { - return dndOpenBnode(pDnode); - } -} - -int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDDropBnodeReq dropReq = {0}; - if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (dropReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_BNODE_INVALID_OPTION; - dError("failed to drop bnode since %s", terrstr()); - return -1; - } else { - return dndDropBnode(pDnode); - } -} - -static void dndSendBnodeErrorRsp(SRpcMsg *pMsg, int32_t code) { - SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rpcRsp); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); -} - -static void dndSendBnodeErrorRsps(STaosQall *qall, int32_t numOfMsgs, int32_t code) { - for (int32_t i = 0; i < numOfMsgs; ++i) { - SRpcMsg *pMsg = NULL; - taosGetQitem(qall, (void **)&pMsg); - dndSendBnodeErrorRsp(pMsg, code); - } -} - -static void dndProcessBnodeQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs) { - SBnode *pBnode = dndAcquireBnode(pDnode); - if (pBnode == NULL) { - dndSendBnodeErrorRsps(qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY); - return; - } - - SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); - if (pArray == NULL) { - dndReleaseBnode(pDnode, pBnode); - dndSendBnodeErrorRsps(qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY); - return; - } - - for (int32_t i = 0; i < numOfMsgs; ++i) { - SRpcMsg *pMsg = NULL; - taosGetQitem(qall, (void **)&pMsg); - void *ptr = taosArrayPush(pArray, &pMsg); - if (ptr == NULL) { - dndSendBnodeErrorRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY); - } - } - - bndProcessWMsgs(pBnode, pArray); - - for (size_t i = 0; i < numOfMsgs; i++) { - SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); - } - taosArrayDestroy(pArray); - dndReleaseBnode(pDnode, pBnode); -} - -static void dndWriteBnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) { - int32_t code = TSDB_CODE_DND_BNODE_NOT_DEPLOYED; - - SBnode *pBnode = dndAcquireBnode(pDnode); - if (pBnode != NULL) { - code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)); - } - dndReleaseBnode(pDnode, pBnode); - - if (code != 0) { - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - } -} - -void dndProcessBnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteBnodeMsgToWorker(pDnode, &pDnode->bmgmt.writeWorker, pMsg); -} - -int32_t dndInitBnode(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - taosInitRWLatch(&pMgmt->latch); - - if (dndReadBnodeFile(pDnode) != 0) { - return -1; - } - - if (pMgmt->dropped) { - dInfo("bnode has been deployed and needs to be deleted"); - bndDestroy(pDnode->dir.bnode); - return 0; - } - - if (!pMgmt->deployed) return 0; - - return dndOpenBnode(pDnode); -} - -void dndCleanupBnode(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - if (pMgmt->pBnode) { - dndStopBnodeWorker(pDnode); - bndClose(pMgmt->pBnode); - pMgmt->pBnode = NULL; - } -} diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c deleted file mode 100644 index 84b2dca326..0000000000 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ /dev/null @@ -1,334 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndBnode.h" -#include "dndMgmt.h" -#include "dndMnode.h" -#include "dndQnode.h" -#include "dndSnode.h" -#include "dndTransport.h" -#include "dndVnodes.h" -#include "monitor.h" -#include "sync.h" -#include "tfs.h" -#include "wal.h" - -static int8_t once = DND_ENV_INIT; - -EStat dndGetStat(SDnode *pDnode) { return pDnode->stat; } - -void dndSetStat(SDnode *pDnode, EStat stat) { - dDebug("dnode status set from %s to %s", dndStatStr(pDnode->stat), dndStatStr(stat)); - pDnode->stat = stat; -} - -const char *dndStatStr(EStat stat) { - switch (stat) { - case DND_STAT_INIT: - return "init"; - case DND_STAT_RUNNING: - return "running"; - case DND_STAT_STOPPED: - return "stopped"; - default: - return "unknown"; - } -} - -void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc) { - SStartupReq *pStartup = &pDnode->startup; - tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); - tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); - pStartup->finished = 0; -} - -void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { - memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); - pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING); -} - -static TdFilePtr dndCheckRunning(char *dataDir) { - char filepath[PATH_MAX] = {0}; - snprintf(filepath, sizeof(filepath), "%s/.running", dataDir); - - TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to lock file:%s since %s, quit", filepath, terrstr()); - return NULL; - } - - int32_t ret = taosLockFile(pFile); - if (ret != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to lock file:%s since %s, quit", filepath, terrstr()); - taosCloseFile(&pFile); - return NULL; - } - - return pFile; -} - -static int32_t dndInitDir(SDnode *pDnode, SDnodeObjCfg *pCfg) { - pDnode->pLockFile = dndCheckRunning(pCfg->dataDir); - if (pDnode->pLockFile == NULL) { - return -1; - } - - char path[PATH_MAX + 100]; - snprintf(path, sizeof(path), "%s%smnode", pCfg->dataDir, TD_DIRSEP); - pDnode->dir.mnode = tstrdup(path); - snprintf(path, sizeof(path), "%s%svnode", pCfg->dataDir, TD_DIRSEP); - pDnode->dir.vnodes = tstrdup(path); - snprintf(path, sizeof(path), "%s%sdnode", pCfg->dataDir, TD_DIRSEP); - pDnode->dir.dnode = tstrdup(path); - snprintf(path, sizeof(path), "%s%ssnode", pCfg->dataDir, TD_DIRSEP); - pDnode->dir.snode = tstrdup(path); - snprintf(path, sizeof(path), "%s%sbnode", pCfg->dataDir, TD_DIRSEP); - pDnode->dir.bnode = tstrdup(path); - - if (pDnode->dir.mnode == NULL || pDnode->dir.vnodes == NULL || pDnode->dir.dnode == NULL || - pDnode->dir.snode == NULL || pDnode->dir.bnode == NULL) { - dError("failed to malloc dir object"); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - if (taosMkDir(pDnode->dir.dnode) != 0) { - dError("failed to create dir:%s since %s", pDnode->dir.dnode, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosMkDir(pDnode->dir.mnode) != 0) { - dError("failed to create dir:%s since %s", pDnode->dir.mnode, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosMkDir(pDnode->dir.vnodes) != 0) { - dError("failed to create dir:%s since %s", pDnode->dir.vnodes, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosMkDir(pDnode->dir.snode) != 0) { - dError("failed to create dir:%s since %s", pDnode->dir.snode, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosMkDir(pDnode->dir.bnode) != 0) { - dError("failed to create dir:%s since %s", pDnode->dir.bnode, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - memcpy(&pDnode->cfg, pCfg, sizeof(SDnodeObjCfg)); - return 0; -} - -static void dndCloseDir(SDnode *pDnode) { - tfree(pDnode->dir.mnode); - tfree(pDnode->dir.vnodes); - tfree(pDnode->dir.dnode); - tfree(pDnode->dir.snode); - tfree(pDnode->dir.bnode); - - if (pDnode->pLockFile != NULL) { - taosUnLockFile(pDnode->pLockFile); - taosCloseFile(&pDnode->pLockFile); - pDnode->pLockFile = NULL; - } -} - -SDnode *dndCreate(SDnodeObjCfg *pCfg) { - dInfo("start to create dnode object"); - - SDnode *pDnode = calloc(1, sizeof(SDnode)); - if (pDnode == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - dError("failed to create dnode object since %s", terrstr()); - return NULL; - } - - dndSetStat(pDnode, DND_STAT_INIT); - - if (dndInitDir(pDnode, pCfg) != 0) { - dError("failed to init dnode dir since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - SDiskCfg dCfg = {0}; - tstrncpy(dCfg.dir, pDnode->cfg.dataDir, TSDB_FILENAME_LEN); - dCfg.level = 0; - dCfg.primary = 1; - SDiskCfg *pDisks = pDnode->cfg.pDisks; - int32_t numOfDisks = pDnode->cfg.numOfDisks; - if (numOfDisks <= 0 || pDisks == NULL) { - pDisks = &dCfg; - numOfDisks = 1; - } - - pDnode->pTfs = tfsOpen(pDisks, numOfDisks); - if (pDnode->pTfs == NULL) { - dError("failed to init tfs since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (dndInitMgmt(pDnode) != 0) { - dError("failed to init mgmt since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (dndInitVnodes(pDnode) != 0) { - dError("failed to init vnodes since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (dndInitQnode(pDnode) != 0) { - dError("failed to init qnode since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (dndInitSnode(pDnode) != 0) { - dError("failed to init snode since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (dndInitBnode(pDnode) != 0) { - dError("failed to init bnode since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (dndInitMnode(pDnode) != 0) { - dError("failed to init mnode since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (dndInitTrans(pDnode) != 0) { - dError("failed to init transport since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - dndSetStat(pDnode, DND_STAT_RUNNING); - dndSendStatusReq(pDnode); - dndReportStartup(pDnode, "TDengine", "initialized successfully"); - dInfo("dnode object is created, data:%p", pDnode); - - return pDnode; -} - -void dndClose(SDnode *pDnode) { - if (pDnode == NULL) return; - - if (dndGetStat(pDnode) == DND_STAT_STOPPED) { - dError("dnode is shutting down, data:%p", pDnode); - return; - } - - dInfo("start to close dnode, data:%p", pDnode); - dndSetStat(pDnode, DND_STAT_STOPPED); - dndCleanupTrans(pDnode); - dndStopMgmt(pDnode); - dndCleanupMnode(pDnode); - dndCleanupBnode(pDnode); - dndCleanupSnode(pDnode); - dndCleanupQnode(pDnode); - dndCleanupVnodes(pDnode); - dndCleanupMgmt(pDnode); - tfsClose(pDnode->pTfs); - - dndCloseDir(pDnode); - free(pDnode); - dInfo("dnode object is closed, data:%p", pDnode); -} - -int32_t dndInit() { - if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { - terrno = TSDB_CODE_REPEAT_INIT; - dError("failed to init dnode env since %s", terrstr()); - return -1; - } - - taosIgnSIGPIPE(); - taosBlockSIGPIPE(); - taosResolveCRC(); - - if (rpcInit() != 0) { - dError("failed to init rpc since %s", terrstr()); - dndCleanup(); - return -1; - } - - if (walInit() != 0) { - dError("failed to init wal since %s", terrstr()); - dndCleanup(); - return -1; - } - - SVnodeOpt vnodeOpt = { - .nthreads = tsNumOfCommitThreads, .putReqToVQueryQFp = dndPutReqToVQueryQ, .sendReqToDnodeFp = dndSendReqToDnode}; - - if (vnodeInit(&vnodeOpt) != 0) { - dError("failed to init vnode since %s", terrstr()); - dndCleanup(); - return -1; - } - - SMonCfg monCfg = {.maxLogs = tsMonitorMaxLogs, .port = tsMonitorPort, .server = tsMonitorFqdn, .comp = tsMonitorComp}; - if (monInit(&monCfg) != 0) { - dError("failed to init monitor since %s", terrstr()); - dndCleanup(); - return -1; - } - - dInfo("dnode env is initialized"); - return 0; -} - -void dndCleanup() { - if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { - dError("dnode env is already cleaned up"); - return; - } - - walCleanUp(); - vnodeCleanup(); - rpcCleanup(); - monCleanup(); - - taosStopCacheRefreshWorker(); - dInfo("dnode env is cleaned up"); -} - -int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) { - tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name)); - pInfo->logdir.size = tsLogSpace.size; - tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); - pInfo->tempdir.size = tsTempSpace.size; - - return tfsGetMonitorInfo(pDnode->pTfs, pInfo); -} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c deleted file mode 100644 index 866076a3d0..0000000000 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ /dev/null @@ -1,763 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndMgmt.h" -#include "dndBnode.h" -#include "dndMnode.h" -#include "dndQnode.h" -#include "dndSnode.h" -#include "dndTransport.h" -#include "dndVnodes.h" -#include "dndWorker.h" -#include "monitor.h" - -static void dndProcessMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg); - -static int32_t dndReadDnodes(SDnode *pDnode); -static int32_t dndWriteDnodes(SDnode *pDnode); -static void *dnodeThreadRoutine(void *param); - -static int32_t dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pReq); -static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pRsp); -static void dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pRsp); -static void dndProcessGrantRsp(SDnode *pDnode, SRpcMsg *pRsp); - -int32_t dndGetDnodeId(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosRLockLatch(&pMgmt->latch); - int32_t dnodeId = pMgmt->dnodeId; - taosRUnLockLatch(&pMgmt->latch); - return dnodeId; -} - -int64_t dndGetClusterId(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosRLockLatch(&pMgmt->latch); - int64_t clusterId = pMgmt->clusterId; - taosRUnLockLatch(&pMgmt->latch); - return clusterId; -} - -void dndGetDnodeEp(SDnode *pDnode, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosRLockLatch(&pMgmt->latch); - - SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t)); - if (pDnodeEp != NULL) { - if (pPort != NULL) { - *pPort = pDnodeEp->ep.port; - } - if (pFqdn != NULL) { - tstrncpy(pFqdn, pDnodeEp->ep.fqdn, TSDB_FQDN_LEN); - } - if (pEp != NULL) { - snprintf(pEp, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port); - } - } - - taosRUnLockLatch(&pMgmt->latch); -} - -void dndGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosRLockLatch(&pMgmt->latch); - *pEpSet = pMgmt->mnodeEpSet; - taosRUnLockLatch(&pMgmt->latch); -} - -void dndSendRedirectRsp(SDnode *pDnode, SRpcMsg *pReq) { - tmsg_t msgType = pReq->msgType; - - SEpSet epSet = {0}; - dndGetMnodeEpSet(pDnode, &epSet); - - dDebug("RPC %p, req:%s is redirected, num:%d use:%d", pReq->handle, TMSG_INFO(msgType), epSet.numOfEps, epSet.inUse); - for (int32_t i = 0; i < epSet.numOfEps; ++i) { - dDebug("mnode index:%d %s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); - if (strcmp(epSet.eps[i].fqdn, pDnode->cfg.localFqdn) == 0 && epSet.eps[i].port == pDnode->cfg.serverPort) { - epSet.inUse = (i + 1) % epSet.numOfEps; - } - - epSet.eps[i].port = htons(epSet.eps[i].port); - } - - rpcSendRedirectRsp(pReq->handle, &epSet); -} - -static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { - dInfo("mnode is changed, num:%d use:%d", pEpSet->numOfEps, pEpSet->inUse); - - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosWLockLatch(&pMgmt->latch); - - pMgmt->mnodeEpSet = *pEpSet; - for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { - dInfo("mnode index:%d %s:%u", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); - } - - taosWUnLockLatch(&pMgmt->latch); -} - -static void dndPrintDnodes(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - - int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->pDnodeEps); - dDebug("print dnode ep list, num:%d", numOfEps); - for (int32_t i = 0; i < numOfEps; i++) { - SDnodeEp *pEp = taosArrayGet(pMgmt->pDnodeEps, i); - dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", pEp->id, pEp->ep.fqdn, pEp->ep.port, pEp->isMnode); - } -} - -static void dndResetDnodes(SDnode *pDnode, SArray *pDnodeEps) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - - if (pMgmt->pDnodeEps != pDnodeEps) { - SArray *tmp = pMgmt->pDnodeEps; - pMgmt->pDnodeEps = taosArrayDup(pDnodeEps); - taosArrayDestroy(tmp); - } - - pMgmt->mnodeEpSet.inUse = 0; - pMgmt->mnodeEpSet.numOfEps = 0; - - int32_t mIndex = 0; - int32_t numOfEps = (int32_t)taosArrayGetSize(pDnodeEps); - - for (int32_t i = 0; i < numOfEps; i++) { - SDnodeEp *pDnodeEp = taosArrayGet(pDnodeEps, i); - if (!pDnodeEp->isMnode) continue; - if (mIndex >= TSDB_MAX_REPLICA) continue; - pMgmt->mnodeEpSet.numOfEps++; - - pMgmt->mnodeEpSet.eps[mIndex] = pDnodeEp->ep; - mIndex++; - } - - for (int32_t i = 0; i < numOfEps; i++) { - SDnodeEp *pDnodeEp = taosArrayGet(pDnodeEps, i); - taosHashPut(pMgmt->dnodeHash, &pDnodeEp->id, sizeof(int32_t), pDnodeEp, sizeof(SDnodeEp)); - } - - dndPrintDnodes(pDnode); -} - -static bool dndIsEpChanged(SDnode *pDnode, int32_t dnodeId, char *pEp) { - bool changed = false; - - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosRLockLatch(&pMgmt->latch); - - SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t)); - if (pDnodeEp != NULL) { - char epstr[TSDB_EP_LEN + 1]; - snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port); - changed = strcmp(pEp, epstr) != 0; - } - - taosRUnLockLatch(&pMgmt->latch); - return changed; -} - -static int32_t dndReadDnodes(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - - pMgmt->pDnodeEps = taosArrayInit(1, sizeof(SDnodeEp)); - if (pMgmt->pDnodeEps == NULL) { - dError("failed to calloc dnodeEp array since %s", strerror(errno)); - goto PRASE_DNODE_OVER; - } - - int32_t code = TSDB_CODE_DND_DNODE_READ_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 256 * 1024; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - - // fp = fopen(pMgmt->file, "r"); - TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_READ); - if (pFile == NULL) { - dDebug("file %s not exist", pMgmt->file); - code = 0; - goto PRASE_DNODE_OVER; - } - - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", pMgmt->file); - goto PRASE_DNODE_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", pMgmt->file); - goto PRASE_DNODE_OVER; - } - - cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId"); - if (!dnodeId || dnodeId->type != cJSON_Number) { - dError("failed to read %s since dnodeId not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - pMgmt->dnodeId = dnodeId->valueint; - - cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId"); - if (!clusterId || clusterId->type != cJSON_String) { - dError("failed to read %s since clusterId not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - pMgmt->clusterId = atoll(clusterId->valuestring); - - cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - pMgmt->dropped = dropped->valueint; - - cJSON *dnodes = cJSON_GetObjectItem(root, "dnodes"); - if (!dnodes || dnodes->type != cJSON_Array) { - dError("failed to read %s since dnodes not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - - int32_t numOfDnodes = cJSON_GetArraySize(dnodes); - if (numOfDnodes <= 0) { - dError("failed to read %s since numOfDnodes:%d invalid", pMgmt->file, numOfDnodes); - goto PRASE_DNODE_OVER; - } - - for (int32_t i = 0; i < numOfDnodes; ++i) { - cJSON *node = cJSON_GetArrayItem(dnodes, i); - if (node == NULL) break; - - SDnodeEp dnodeEp = {0}; - - cJSON *did = cJSON_GetObjectItem(node, "id"); - if (!did || did->type != cJSON_Number) { - dError("failed to read %s since dnodeId not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - - dnodeEp.id = dnodeId->valueint; - - cJSON *dnodeFqdn = cJSON_GetObjectItem(node, "fqdn"); - if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) { - dError("failed to read %s since dnodeFqdn not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - tstrncpy(dnodeEp.ep.fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN); - - cJSON *dnodePort = cJSON_GetObjectItem(node, "port"); - if (!dnodePort || dnodePort->type != cJSON_Number) { - dError("failed to read %s since dnodePort not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - - dnodeEp.ep.port = dnodePort->valueint; - - cJSON *isMnode = cJSON_GetObjectItem(node, "isMnode"); - if (!isMnode || isMnode->type != cJSON_Number) { - dError("failed to read %s since isMnode not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - dnodeEp.isMnode = isMnode->valueint; - - taosArrayPush(pMgmt->pDnodeEps, &dnodeEp); - } - - code = 0; - dInfo("succcessed to read file %s", pMgmt->file); - dndPrintDnodes(pDnode); - -PRASE_DNODE_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (pFile != NULL) taosCloseFile(&pFile); - - if (dndIsEpChanged(pDnode, pMgmt->dnodeId, pDnode->cfg.localEp)) { - dError("localEp %s different with %s and need reconfigured", pDnode->cfg.localEp, pMgmt->file); - return -1; - } - - if (taosArrayGetSize(pMgmt->pDnodeEps) == 0) { - SDnodeEp dnodeEp = {0}; - dnodeEp.isMnode = 1; - taosGetFqdnPortFromEp(pDnode->cfg.firstEp, &dnodeEp.ep); - taosArrayPush(pMgmt->pDnodeEps, &dnodeEp); - } - - dndResetDnodes(pDnode, pMgmt->pDnodeEps); - - terrno = 0; - return 0; -} - -static int32_t dndWriteDnodes(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - - // FILE *fp = fopen(pMgmt->file, "w"); - TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - dError("failed to write %s since %s", pMgmt->file, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - int32_t len = 0; - int32_t maxLen = 256 * 1024; - char *content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", pMgmt->dnodeId); - len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%" PRId64 "\",\n", pMgmt->clusterId); - len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pMgmt->dropped); - len += snprintf(content + len, maxLen - len, " \"dnodes\": [{\n"); - - int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->pDnodeEps); - for (int32_t i = 0; i < numOfEps; ++i) { - SDnodeEp *pDnodeEp = taosArrayGet(pMgmt->pDnodeEps, i); - len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pDnodeEp->id); - len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pDnodeEp->ep.fqdn); - len += snprintf(content + len, maxLen - len, " \"port\": %u,\n", pDnodeEp->ep.port); - len += snprintf(content + len, maxLen - len, " \"isMnode\": %d\n", pDnodeEp->isMnode); - if (i < numOfEps - 1) { - len += snprintf(content + len, maxLen - len, " },{\n"); - } else { - len += snprintf(content + len, maxLen - len, " }]\n"); - } - } - len += snprintf(content + len, maxLen - len, "}\n"); - - taosWriteFile(pFile, content, len); - taosFsyncFile(pFile); - taosCloseFile(&pFile); - free(content); - terrno = 0; - - pMgmt->updateTime = taosGetTimestampMs(); - dDebug("successed to write %s", pMgmt->file); - return 0; -} - -void dndSendStatusReq(SDnode *pDnode) { - SStatusReq req = {0}; - - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosRLockLatch(&pMgmt->latch); - req.sver = tsVersion; - req.dver = pMgmt->dver; - req.dnodeId = pMgmt->dnodeId; - req.clusterId = pMgmt->clusterId; - req.rebootTime = pMgmt->rebootTime; - req.updateTime = pMgmt->updateTime; - req.numOfCores = tsNumOfCores; - req.numOfSupportVnodes = pDnode->cfg.numOfSupportVnodes; - memcpy(req.dnodeEp, pDnode->cfg.localEp, TSDB_EP_LEN); - - req.clusterCfg.statusInterval = tsStatusInterval; - req.clusterCfg.checkTime = 0; - char timestr[32] = "1970-01-01 00:00:00.00"; - (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); - memcpy(req.clusterCfg.timezone, tsTimezone, TD_TIMEZONE_LEN); - memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN); - memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN); - taosRUnLockLatch(&pMgmt->latch); - - req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad)); - dndGetVnodeLoads(pDnode, req.pVloads); - - int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); - void *pHead = rpcMallocCont(contLen); - tSerializeSStatusReq(pHead, contLen, &req); - taosArrayDestroy(req.pVloads); - - SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)9527}; - pMgmt->statusSent = 1; - - dTrace("pDnode:%p, send status req to mnode", pDnode); - dndSendReqToMnode(pDnode, &rpcMsg); -} - -static void dndUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - if (pMgmt->dnodeId == 0) { - dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId); - taosWLockLatch(&pMgmt->latch); - pMgmt->dnodeId = pCfg->dnodeId; - pMgmt->clusterId = pCfg->clusterId; - dndWriteDnodes(pDnode); - taosWUnLockLatch(&pMgmt->latch); - } -} - -static void dndUpdateDnodeEps(SDnode *pDnode, SArray *pDnodeEps) { - int32_t numOfEps = taosArrayGetSize(pDnodeEps); - if (numOfEps <= 0) return; - - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosWLockLatch(&pMgmt->latch); - - int32_t numOfEpsOld = (int32_t)taosArrayGetSize(pMgmt->pDnodeEps); - if (numOfEps != numOfEpsOld) { - dndResetDnodes(pDnode, pDnodeEps); - dndWriteDnodes(pDnode); - } else { - int32_t size = numOfEps * sizeof(SDnodeEp); - if (memcmp(pMgmt->pDnodeEps->pData, pDnodeEps->pData, size) != 0) { - dndResetDnodes(pDnode, pDnodeEps); - dndWriteDnodes(pDnode); - } - } - - taosWUnLockLatch(&pMgmt->latch); -} - -static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pRsp) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - - if (pRsp->code != TSDB_CODE_SUCCESS) { - if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pMgmt->dropped && pMgmt->dnodeId > 0) { - dInfo("dnode:%d, set to dropped since not exist in mnode", pMgmt->dnodeId); - pMgmt->dropped = 1; - dndWriteDnodes(pDnode); - } - } else { - SStatusRsp statusRsp = {0}; - if (pRsp->pCont != NULL && pRsp->contLen != 0 && - tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) { - pMgmt->dver = statusRsp.dver; - dndUpdateDnodeCfg(pDnode, &statusRsp.dnodeCfg); - dndUpdateDnodeEps(pDnode, statusRsp.pDnodeEps); - } - taosArrayDestroy(statusRsp.pDnodeEps); - } - - pMgmt->statusSent = 0; -} - -static void dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pReq) { dError("auth rsp is received, but not supported yet"); } - -static void dndProcessGrantRsp(SDnode *pDnode, SRpcMsg *pReq) { - dError("grant rsp is received, but not supported yet"); -} - -static int32_t dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - dError("config req is received, but not supported yet"); - SDCfgDnodeReq *pCfg = pReq->pCont; - return TSDB_CODE_OPS_NOT_SUPPORT; -} - -void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { - dDebug("startup req is received"); - - SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); - dndGetStartup(pDnode, pStartup); - - dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); - - SRpcMsg rpcRsp = {.handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq)}; - rpcSendResponse(&rpcRsp); -} - -static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { - pInfo->dnode_id = dndGetDnodeId(pDnode); - tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); - pInfo->cluster_id = dndGetClusterId(pDnode); - pInfo->protocol = 1; -} - -static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { - pInfo->uptime = (taosGetTimestampMs() - pDnode->dmgmt.rebootTime) / (86400000.0f); - taosGetCpuUsage(&pInfo->cpu_engine, &pInfo->cpu_system); - pInfo->cpu_cores = tsNumOfCores; - taosGetProcMemory(&pInfo->mem_engine); - taosGetSysMemory(&pInfo->mem_system); - pInfo->mem_total = tsTotalMemoryKB; - pInfo->disk_engine = 0; - pInfo->disk_used = tsDataSpace.size.used; - pInfo->disk_total = tsDataSpace.size.total; - taosGetCardInfo(&pInfo->net_in, &pInfo->net_out); - taosGetProcIO(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk); - - SVnodesStat *pStat = &pDnode->vmgmt.stat; - pInfo->req_select = pStat->numOfSelectReqs; - pInfo->req_insert = pStat->numOfInsertReqs; - pInfo->req_insert_success = pStat->numOfInsertSuccessReqs; - pInfo->req_insert_batch = pStat->numOfBatchInsertReqs; - pInfo->req_insert_batch_success = pStat->numOfBatchInsertSuccessReqs; - pInfo->errors = tsNumOfErrorLogs; - pInfo->vnodes_num = pStat->totalVnodes; - pInfo->masters = pStat->masterNum; - pInfo->has_mnode = pDnode->mmgmt.deployed; -} - -static void dndSendMonitorReport(SDnode *pDnode) { - if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; - dTrace("pDnode:%p, send monitor report to %s:%u", pDnode, tsMonitorFqdn, tsMonitorPort); - - SMonInfo *pMonitor = monCreateMonitorInfo(); - if (pMonitor == NULL) return; - - SMonBasicInfo basicInfo = {0}; - dndGetMonitorBasicInfo(pDnode, &basicInfo); - monSetBasicInfo(pMonitor, &basicInfo); - - SMonClusterInfo clusterInfo = {0}; - SMonVgroupInfo vgroupInfo = {0}; - SMonGrantInfo grantInfo = {0}; - if (dndGetMnodeMonitorInfo(pDnode, &clusterInfo, &vgroupInfo, &grantInfo) == 0) { - monSetClusterInfo(pMonitor, &clusterInfo); - monSetVgroupInfo(pMonitor, &vgroupInfo); - monSetGrantInfo(pMonitor, &grantInfo); - } - - SMonDnodeInfo dnodeInfo = {0}; - dndGetMonitorDnodeInfo(pDnode, &dnodeInfo); - monSetDnodeInfo(pMonitor, &dnodeInfo); - - SMonDiskInfo diskInfo = {0}; - if (dndGetMonitorDiskInfo(pDnode, &diskInfo) == 0) { - monSetDiskInfo(pMonitor, &diskInfo); - } - - taosArrayDestroy(clusterInfo.dnodes); - taosArrayDestroy(clusterInfo.mnodes); - taosArrayDestroy(vgroupInfo.vgroups); - taosArrayDestroy(diskInfo.datadirs); - - monSendReport(pMonitor); - monCleanupMonitorInfo(pMonitor); -} - -static void *dnodeThreadRoutine(void *param) { - SDnode *pDnode = param; - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - int64_t lastStatusTime = taosGetTimestampMs(); - int64_t lastMonitorTime = lastStatusTime; - - setThreadName("dnode-hb"); - - while (true) { - pthread_testcancel(); - taosMsleep(200); - if (dndGetStat(pDnode) != DND_STAT_RUNNING || pMgmt->dropped) { - continue; - } - - int64_t curTime = taosGetTimestampMs(); - - float statusInterval = (curTime - lastStatusTime) / 1000.0f; - if (statusInterval >= tsStatusInterval && !pMgmt->statusSent) { - dndSendStatusReq(pDnode); - lastStatusTime = curTime; - } - - float monitorInterval = (curTime - lastMonitorTime) / 1000.0f; - if (monitorInterval >= tsMonitorInterval) { - dndSendMonitorReport(pDnode); - lastMonitorTime = curTime; - } - } -} - -int32_t dndInitMgmt(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - - pMgmt->dnodeId = 0; - pMgmt->rebootTime = taosGetTimestampMs(); - pMgmt->dropped = 0; - pMgmt->clusterId = 0; - taosInitRWLatch(&pMgmt->latch); - - char path[PATH_MAX]; - snprintf(path, PATH_MAX, "%s/dnode.json", pDnode->dir.dnode); - pMgmt->file = strdup(path); - if (pMgmt->file == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - pMgmt->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - if (pMgmt->dnodeHash == NULL) { - dError("failed to init dnode hash"); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - if (dndReadDnodes(pDnode) != 0) { - dError("failed to read file:%s since %s", pMgmt->file, terrstr()); - return -1; - } - - if (pMgmt->dropped) { - dError("dnode not start since its already dropped"); - return -1; - } - - if (dndInitWorker(pDnode, &pMgmt->mgmtWorker, DND_WORKER_SINGLE, "dnode-mgmt", 1, 1, dndProcessMgmtQueue) != 0) { - dError("failed to start dnode mgmt worker since %s", terrstr()); - return -1; - } - - if (dndInitWorker(pDnode, &pMgmt->statusWorker, DND_WORKER_SINGLE, "dnode-status", 1, 1, dndProcessMgmtQueue) != 0) { - dError("failed to start dnode mgmt worker since %s", terrstr()); - return -1; - } - - pMgmt->threadId = taosCreateThread(dnodeThreadRoutine, pDnode); - if (pMgmt->threadId == NULL) { - dError("failed to init dnode thread"); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - dInfo("dnode-mgmt is initialized"); - return 0; -} - -void dndStopMgmt(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - dndCleanupWorker(&pMgmt->mgmtWorker); - dndCleanupWorker(&pMgmt->statusWorker); - - if (pMgmt->threadId != NULL) { - taosDestoryThread(pMgmt->threadId); - pMgmt->threadId = NULL; - } -} - -void dndCleanupMgmt(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosWLockLatch(&pMgmt->latch); - - if (pMgmt->pDnodeEps != NULL) { - taosArrayDestroy(pMgmt->pDnodeEps); - pMgmt->pDnodeEps = NULL; - } - - if (pMgmt->dnodeHash != NULL) { - taosHashCleanup(pMgmt->dnodeHash); - pMgmt->dnodeHash = NULL; - } - - if (pMgmt->file != NULL) { - free(pMgmt->file); - pMgmt->file = NULL; - } - - taosWUnLockLatch(&pMgmt->latch); - dInfo("dnode-mgmt is cleaned up"); -} - -void dndProcessMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - - if (pEpSet && pEpSet->numOfEps > 0 && pMsg->msgType == TDMT_MND_STATUS_RSP) { - dndUpdateMnodeEpSet(pDnode, pEpSet); - } - - SDnodeWorker *pWorker = &pMgmt->mgmtWorker; - if (pMsg->msgType == TDMT_MND_STATUS_RSP) { - pWorker = &pMgmt->statusWorker; - } - - if (dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)) != 0) { - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = TSDB_CODE_OUT_OF_MEMORY}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); - } -} - -static void dndProcessMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) { - int32_t code = 0; - - switch (pMsg->msgType) { - case TDMT_DND_CREATE_MNODE: - code = dndProcessCreateMnodeReq(pDnode, pMsg); - break; - case TDMT_DND_ALTER_MNODE: - code = dndProcessAlterMnodeReq(pDnode, pMsg); - break; - case TDMT_DND_DROP_MNODE: - code = dndProcessDropMnodeReq(pDnode, pMsg); - break; - case TDMT_DND_CREATE_QNODE: - code = dndProcessCreateQnodeReq(pDnode, pMsg); - break; - case TDMT_DND_DROP_QNODE: - code = dndProcessDropQnodeReq(pDnode, pMsg); - break; - case TDMT_DND_CREATE_SNODE: - code = dndProcessCreateSnodeReq(pDnode, pMsg); - break; - case TDMT_DND_DROP_SNODE: - code = dndProcessDropSnodeReq(pDnode, pMsg); - break; - case TDMT_DND_CREATE_BNODE: - code = dndProcessCreateBnodeReq(pDnode, pMsg); - break; - case TDMT_DND_DROP_BNODE: - code = dndProcessDropBnodeReq(pDnode, pMsg); - break; - case TDMT_DND_CONFIG_DNODE: - code = dndProcessConfigDnodeReq(pDnode, pMsg); - break; - case TDMT_MND_STATUS_RSP: - dndProcessStatusRsp(pDnode, pMsg); - break; - case TDMT_MND_AUTH_RSP: - dndProcessAuthRsp(pDnode, pMsg); - break; - case TDMT_MND_GRANT_RSP: - dndProcessGrantRsp(pDnode, pMsg); - break; - case TDMT_DND_CREATE_VNODE: - code = dndProcessCreateVnodeReq(pDnode, pMsg); - break; - case TDMT_DND_ALTER_VNODE: - code = dndProcessAlterVnodeReq(pDnode, pMsg); - break; - case TDMT_DND_DROP_VNODE: - code = dndProcessDropVnodeReq(pDnode, pMsg); - break; - case TDMT_DND_SYNC_VNODE: - code = dndProcessSyncVnodeReq(pDnode, pMsg); - break; - case TDMT_DND_COMPACT_VNODE: - code = dndProcessCompactVnodeReq(pDnode, pMsg); - break; - default: - terrno = TSDB_CODE_MSG_NOT_PROCESSED; - code = -1; - dError("RPC %p, dnode msg:%s not processed", pMsg->handle, TMSG_INFO(pMsg->msgType)); - break; - } - - if (pMsg->msgType & 1u) { - if (code != 0) code = terrno; - SRpcMsg rsp = {.code = code, .handle = pMsg->handle, .ahandle = pMsg->ahandle}; - rpcSendResponse(&rsp); - } - - rpcFreeCont(pMsg->pCont); - pMsg->pCont = NULL; - taosFreeQitem(pMsg); -} diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c deleted file mode 100644 index 0ea47c89d8..0000000000 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ /dev/null @@ -1,642 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndMnode.h" -#include "dndMgmt.h" -#include "dndTransport.h" -#include "dndWorker.h" - -static void dndWriteMnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pRpcMsg); -static void dndProcessMnodeQueue(SDnode *pDnode, SMnodeMsg *pMsg); - -static SMnode *dndAcquireMnode(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - SMnode *pMnode = NULL; - int32_t refCount = 0; - - taosRLockLatch(&pMgmt->latch); - if (pMgmt->deployed && !pMgmt->dropped) { - refCount = atomic_add_fetch_32(&pMgmt->refCount, 1); - pMnode = pMgmt->pMnode; - } else { - terrno = TSDB_CODE_DND_MNODE_NOT_DEPLOYED; - } - taosRUnLockLatch(&pMgmt->latch); - - if (pMnode != NULL) { - dTrace("acquire mnode, refCount:%d", refCount); - } - return pMnode; -} - -static void dndReleaseMnode(SDnode *pDnode, SMnode *pMnode) { - if (pMnode == NULL) return; - - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - taosRLockLatch(&pMgmt->latch); - int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); - taosRUnLockLatch(&pMgmt->latch); - dTrace("release mnode, refCount:%d", refCount); -} - -static int32_t dndReadMnodeFile(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - int32_t code = TSDB_CODE_DND_MNODE_READ_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 4096; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "r"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); - if (pFile == NULL) { - dDebug("file %s not exist", file); - code = 0; - goto PRASE_MNODE_OVER; - } - - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", file); - goto PRASE_MNODE_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); - goto PRASE_MNODE_OVER; - } - - cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); - if (!deployed || deployed->type != cJSON_Number) { - dError("failed to read %s since deployed not found", file); - goto PRASE_MNODE_OVER; - } - pMgmt->deployed = deployed->valueint; - - cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", file); - goto PRASE_MNODE_OVER; - } - pMgmt->dropped = dropped->valueint; - - cJSON *mnodes = cJSON_GetObjectItem(root, "mnodes"); - if (!mnodes || mnodes->type != cJSON_Array) { - dError("failed to read %s since nodes not found", file); - goto PRASE_MNODE_OVER; - } - - pMgmt->replica = cJSON_GetArraySize(mnodes); - if (pMgmt->replica <= 0 || pMgmt->replica > TSDB_MAX_REPLICA) { - dError("failed to read %s since mnodes size %d invalid", file, pMgmt->replica); - goto PRASE_MNODE_OVER; - } - - for (int32_t i = 0; i < pMgmt->replica; ++i) { - cJSON *node = cJSON_GetArrayItem(mnodes, i); - if (node == NULL) break; - - SReplica *pReplica = &pMgmt->replicas[i]; - - cJSON *id = cJSON_GetObjectItem(node, "id"); - if (!id || id->type != cJSON_Number) { - dError("failed to read %s since id not found", file); - goto PRASE_MNODE_OVER; - } - pReplica->id = id->valueint; - - cJSON *fqdn = cJSON_GetObjectItem(node, "fqdn"); - if (!fqdn || fqdn->type != cJSON_String || fqdn->valuestring == NULL) { - dError("failed to read %s since fqdn not found", file); - goto PRASE_MNODE_OVER; - } - tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN); - - cJSON *port = cJSON_GetObjectItem(node, "port"); - if (!port || port->type != cJSON_Number) { - dError("failed to read %s since port not found", file); - goto PRASE_MNODE_OVER; - } - pReplica->port = port->valueint; - } - - code = 0; - dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); - -PRASE_MNODE_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (pFile != NULL) taosCloseFile(&pFile); - - terrno = code; - return code; -} - -static int32_t dndWriteMnodeFile(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/mnode.json.bak", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "w"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR; - dError("failed to write %s since %s", file, terrstr()); - return -1; - } - - int32_t len = 0; - int32_t maxLen = 4096; - char *content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", pMgmt->deployed); - - len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pMgmt->dropped); - len += snprintf(content + len, maxLen - len, " \"mnodes\": [{\n"); - for (int32_t i = 0; i < pMgmt->replica; ++i) { - SReplica *pReplica = &pMgmt->replicas[i]; - len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); - len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); - len += snprintf(content + len, maxLen - len, " \"port\": %u\n", pReplica->port); - if (i < pMgmt->replica - 1) { - len += snprintf(content + len, maxLen - len, " },{\n"); - } else { - len += snprintf(content + len, maxLen - len, " }]\n"); - } - } - len += snprintf(content + len, maxLen - len, "}\n"); - - taosWriteFile(pFile, content, len); - taosFsyncFile(pFile); - taosCloseFile(&pFile); - free(content); - - char realfile[PATH_MAX + 20]; - snprintf(realfile, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode); - - if (taosRenameFile(file, realfile) != 0) { - terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR; - dError("failed to rename %s since %s", file, terrstr()); - return -1; - } - - dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped); - return 0; -} - -static int32_t dndStartMnodeWorker(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - if (dndInitWorker(pDnode, &pMgmt->readWorker, DND_WORKER_SINGLE, "mnode-read", 0, 1, dndProcessMnodeQueue) != 0) { - dError("failed to start mnode read worker since %s", terrstr()); - return -1; - } - - if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_SINGLE, "mnode-write", 0, 1, dndProcessMnodeQueue) != 0) { - dError("failed to start mnode write worker since %s", terrstr()); - return -1; - } - - if (dndInitWorker(pDnode, &pMgmt->syncWorker, DND_WORKER_SINGLE, "mnode-sync", 0, 1, dndProcessMnodeQueue) != 0) { - dError("failed to start mnode sync worker since %s", terrstr()); - return -1; - } - - return 0; -} - -static void dndStopMnodeWorker(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - taosWLockLatch(&pMgmt->latch); - pMgmt->deployed = 0; - taosWUnLockLatch(&pMgmt->latch); - - while (pMgmt->refCount > 1) { - taosMsleep(10); - } - - dndCleanupWorker(&pMgmt->readWorker); - dndCleanupWorker(&pMgmt->writeWorker); - dndCleanupWorker(&pMgmt->syncWorker); -} - -static bool dndNeedDeployMnode(SDnode *pDnode) { - if (dndGetDnodeId(pDnode) > 0) { - return false; - } - - if (dndGetClusterId(pDnode) > 0) { - return false; - } - - if (strcmp(pDnode->cfg.localEp, pDnode->cfg.firstEp) != 0) { - return false; - } - - return true; -} - -static int32_t dndPutMsgToMWriteQ(SDnode *pDnode, SRpcMsg *pRpcMsg) { - dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.writeWorker, pRpcMsg); - return 0; -} - -static int32_t dndPutMsgToMReadQ(SDnode *pDnode, SRpcMsg* pRpcMsg) { - dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.readWorker, pRpcMsg); - return 0; -} - -static void dndInitMnodeOption(SDnode *pDnode, SMnodeOpt *pOption) { - pOption->pDnode = pDnode; - pOption->sendReqToDnodeFp = dndSendReqToDnode; - pOption->sendReqToMnodeFp = dndSendReqToMnode; - pOption->sendRedirectRspFp = dndSendRedirectRsp; - pOption->putReqToMWriteQFp = dndPutMsgToMWriteQ; - pOption->putReqToMReadQFp = dndPutMsgToMReadQ; - pOption->dnodeId = dndGetDnodeId(pDnode); - pOption->clusterId = dndGetClusterId(pDnode); -} - -static void dndBuildMnodeDeployOption(SDnode *pDnode, SMnodeOpt *pOption) { - dndInitMnodeOption(pDnode, pOption); - pOption->replica = 1; - pOption->selfIndex = 0; - SReplica *pReplica = &pOption->replicas[0]; - pReplica->id = 1; - pReplica->port = pDnode->cfg.serverPort; - memcpy(pReplica->fqdn, pDnode->cfg.localFqdn, TSDB_FQDN_LEN); - - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - pMgmt->selfIndex = pOption->selfIndex; - pMgmt->replica = pOption->replica; - memcpy(&pMgmt->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); -} - -static void dndBuildMnodeOpenOption(SDnode *pDnode, SMnodeOpt *pOption) { - dndInitMnodeOption(pDnode, pOption); - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - pOption->selfIndex = pMgmt->selfIndex; - pOption->replica = pMgmt->replica; - memcpy(&pOption->replicas, pMgmt->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); -} - -static int32_t dndBuildMnodeOptionFromReq(SDnode *pDnode, SMnodeOpt *pOption, SDCreateMnodeReq *pCreate) { - dndInitMnodeOption(pDnode, pOption); - pOption->dnodeId = dndGetDnodeId(pDnode); - pOption->clusterId = dndGetClusterId(pDnode); - - pOption->replica = pCreate->replica; - pOption->selfIndex = -1; - for (int32_t i = 0; i < pCreate->replica; ++i) { - SReplica *pReplica = &pOption->replicas[i]; - pReplica->id = pCreate->replicas[i].id; - pReplica->port = pCreate->replicas[i].port; - memcpy(pReplica->fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN); - if (pReplica->id == pOption->dnodeId) { - pOption->selfIndex = i; - } - } - - if (pOption->selfIndex == -1) { - dError("failed to build mnode options since %s", terrstr()); - return -1; - } - - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - pMgmt->selfIndex = pOption->selfIndex; - pMgmt->replica = pOption->replica; - memcpy(&pMgmt->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); - return 0; -} - -static int32_t dndOpenMnode(SDnode *pDnode, SMnodeOpt *pOption) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - SMnode *pMnode = mndOpen(pDnode->dir.mnode, pOption); - if (pMnode == NULL) { - dError("failed to open mnode since %s", terrstr()); - return -1; - } - - if (dndStartMnodeWorker(pDnode) != 0) { - dError("failed to start mnode worker since %s", terrstr()); - mndClose(pMnode); - mndDestroy(pDnode->dir.mnode); - return -1; - } - - pMgmt->deployed = 1; - if (dndWriteMnodeFile(pDnode) != 0) { - dError("failed to write mnode file since %s", terrstr()); - pMgmt->deployed = 0; - dndStopMnodeWorker(pDnode); - mndClose(pMnode); - mndDestroy(pDnode->dir.mnode); - return -1; - } - - taosWLockLatch(&pMgmt->latch); - pMgmt->pMnode = pMnode; - pMgmt->deployed = 1; - taosWUnLockLatch(&pMgmt->latch); - - dInfo("mnode open successfully"); - return 0; -} - -static int32_t dndAlterMnode(SDnode *pDnode, SMnodeOpt *pOption) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL) { - dError("failed to alter mnode since %s", terrstr()); - return -1; - } - - if (mndAlter(pMnode, pOption) != 0) { - dError("failed to alter mnode since %s", terrstr()); - dndReleaseMnode(pDnode, pMnode); - return -1; - } - - dndReleaseMnode(pDnode, pMnode); - return 0; -} - -static int32_t dndDropMnode(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL) { - dError("failed to drop mnode since %s", terrstr()); - return -1; - } - - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 1; - taosRUnLockLatch(&pMgmt->latch); - - if (dndWriteMnodeFile(pDnode) != 0) { - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 0; - taosRUnLockLatch(&pMgmt->latch); - - dndReleaseMnode(pDnode, pMnode); - dError("failed to drop mnode since %s", terrstr()); - return -1; - } - - dndReleaseMnode(pDnode, pMnode); - dndStopMnodeWorker(pDnode); - pMgmt->deployed = 0; - dndWriteMnodeFile(pDnode); - mndClose(pMnode); - pMgmt->pMnode = NULL; - mndDestroy(pDnode->dir.mnode); - - return 0; -} - - -int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDCreateMnodeReq createReq = {0}; - if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (createReq.replica <= 1 || createReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION; - dError("failed to create mnode since %s", terrstr()); - return -1; - } - - SMnodeOpt option = {0}; - if (dndBuildMnodeOptionFromReq(pDnode, &option, &createReq) != 0) { - terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION; - dError("failed to create mnode since %s", terrstr()); - return -1; - } - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode != NULL) { - dndReleaseMnode(pDnode, pMnode); - terrno = TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED; - dError("failed to create mnode since %s", terrstr()); - return -1; - } - - dDebug("start to create mnode"); - return dndOpenMnode(pDnode, &option); -} - -int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDAlterMnodeReq alterReq = {0}; - if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (alterReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION; - dError("failed to alter mnode since %s", terrstr()); - return -1; - } - - SMnodeOpt option = {0}; - if (dndBuildMnodeOptionFromReq(pDnode, &option, &alterReq) != 0) { - terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION; - dError("failed to alter mnode since %s", terrstr()); - return -1; - } - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL) { - terrno = TSDB_CODE_DND_MNODE_NOT_DEPLOYED; - dError("failed to alter mnode since %s", terrstr()); - return -1; - } - - dDebug("start to alter mnode"); - int32_t code = dndAlterMnode(pDnode, &option); - dndReleaseMnode(pDnode, pMnode); - - return code; -} - -int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDDropMnodeReq dropReq = {0}; - if (tDeserializeSMCreateDropMnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (dropReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION; - dError("failed to drop mnode since %s", terrstr()); - return -1; - } - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL) { - terrno = TSDB_CODE_DND_MNODE_NOT_DEPLOYED; - dError("failed to drop mnode since %s", terrstr()); - return -1; - } - - dDebug("start to drop mnode"); - int32_t code = dndDropMnode(pDnode); - dndReleaseMnode(pDnode, pMnode); - - return code; -} - -static void dndProcessMnodeQueue(SDnode *pDnode, SMnodeMsg *pMsg) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode != NULL) { - mndProcessMsg(pMsg); - dndReleaseMnode(pDnode, pMnode); - } else { - mndSendRsp(pMsg, terrno); - } - - mndCleanupMsg(pMsg); -} - -static void dndWriteMnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pRpcMsg) { - int32_t code = TSDB_CODE_DND_MNODE_NOT_DEPLOYED; - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode != NULL) { - SMnodeMsg *pMsg = mndInitMsg(pMnode, pRpcMsg); - if (pMsg == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - } else { - code = dndWriteMsgToWorker(pWorker, pMsg, 0); - if (code != 0) code = terrno; - } - - if (code != 0) { - mndCleanupMsg(pMsg); - } - } - dndReleaseMnode(pDnode, pMnode); - - if (code != 0) { - if (pRpcMsg->msgType & 1u) { - if (code == TSDB_CODE_DND_MNODE_NOT_DEPLOYED || code == TSDB_CODE_APP_NOT_READY) { - dndSendRedirectRsp(pDnode, pRpcMsg); - } else { - SRpcMsg rsp = {.handle = pRpcMsg->handle, .ahandle = pRpcMsg->ahandle, .code = code}; - rpcSendResponse(&rsp); - } - } - rpcFreeCont(pRpcMsg->pCont); - } -} - -void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.writeWorker, pMsg); -} - -void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.syncWorker, pMsg); -} - -void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.readWorker, pMsg); -} - -int32_t dndInitMnode(SDnode *pDnode) { - dInfo("dnode-mnode start to init"); - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - taosInitRWLatch(&pMgmt->latch); - - if (dndReadMnodeFile(pDnode) != 0) { - return -1; - } - - if (pMgmt->dropped) { - dInfo("mnode has been deployed and needs to be deleted"); - mndDestroy(pDnode->dir.mnode); - return 0; - } - - if (!pMgmt->deployed) { - bool needDeploy = dndNeedDeployMnode(pDnode); - if (!needDeploy) { - dDebug("mnode does not need to be deployed"); - return 0; - } - - dInfo("start to deploy mnode"); - SMnodeOpt option = {0}; - dndBuildMnodeDeployOption(pDnode, &option); - return dndOpenMnode(pDnode, &option); - } else { - dInfo("start to open mnode"); - SMnodeOpt option = {0}; - dndBuildMnodeOpenOption(pDnode, &option); - return dndOpenMnode(pDnode, &option); - } -} - -void dndCleanupMnode(SDnode *pDnode) { - dInfo("dnode-mnode start to clean up"); - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - if (pMgmt->pMnode) { - dndStopMnodeWorker(pDnode); - mndClose(pMgmt->pMnode); - pMgmt->pMnode = NULL; - } - dInfo("dnode-mnode is cleaned up"); -} - -int32_t dndGetUserAuthFromMnode(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL) { - terrno = TSDB_CODE_APP_NOT_READY; - dTrace("failed to get user auth since %s", terrstr()); - return -1; - } - - int32_t code = mndRetriveAuth(pMnode, user, spi, encrypt, secret, ckey); - dndReleaseMnode(pDnode, pMnode); - - dTrace("user:%s, retrieve auth spi:%d encrypt:%d", user, *spi, *encrypt); - return code; -} - -int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, - SMonGrantInfo *pGrantInfo) { - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL) return -1; - - int32_t code = mndGetMonitorInfo(pMnode, pClusterInfo, pVgroupInfo, pGrantInfo); - dndReleaseMnode(pDnode, pMnode); - return code; -} diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c deleted file mode 100644 index 93e2209610..0000000000 --- a/source/dnode/mgmt/impl/src/dndQnode.c +++ /dev/null @@ -1,375 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndQnode.h" -#include "dndMgmt.h" -#include "dndTransport.h" -#include "dndWorker.h" - -static void dndProcessQnodeQueue(SDnode *pDnode, SRpcMsg *pMsg); - -static SQnode *dndAcquireQnode(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - SQnode *pQnode = NULL; - int32_t refCount = 0; - - taosRLockLatch(&pMgmt->latch); - if (pMgmt->deployed && !pMgmt->dropped && pMgmt->pQnode != NULL) { - refCount = atomic_add_fetch_32(&pMgmt->refCount, 1); - pQnode = pMgmt->pQnode; - } else { - terrno = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; - } - taosRUnLockLatch(&pMgmt->latch); - - if (pQnode != NULL) { - dTrace("acquire qnode, refCount:%d", refCount); - } - return pQnode; -} - -static void dndReleaseQnode(SDnode *pDnode, SQnode *pQnode) { - if (pQnode == NULL) return; - - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - taosRLockLatch(&pMgmt->latch); - int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); - taosRUnLockLatch(&pMgmt->latch); - dTrace("release qnode, refCount:%d", refCount); -} - -static int32_t dndReadQnodeFile(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - int32_t code = TSDB_CODE_DND_QNODE_READ_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "r"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); - if (pFile == NULL) { - dDebug("file %s not exist", file); - code = 0; - goto PRASE_QNODE_OVER; - } - - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", file); - goto PRASE_QNODE_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); - goto PRASE_QNODE_OVER; - } - - cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); - if (!deployed || deployed->type != cJSON_Number) { - dError("failed to read %s since deployed not found", file); - goto PRASE_QNODE_OVER; - } - pMgmt->deployed = deployed->valueint; - - cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", file); - goto PRASE_QNODE_OVER; - } - pMgmt->dropped = dropped->valueint; - - code = 0; - dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); - -PRASE_QNODE_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (pFile != NULL) taosCloseFile(&pFile); - - terrno = code; - return code; -} - -static int32_t dndWriteQnodeFile(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "w"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR; - dError("failed to write %s since %s", file, terrstr()); - return -1; - } - - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", pMgmt->deployed); - len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); - len += snprintf(content + len, maxLen - len, "}\n"); - - taosWriteFile(pFile, content, len); - taosFsyncFile(pFile); - taosCloseFile(&pFile); - free(content); - - char realfile[PATH_MAX + 20]; - snprintf(realfile, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); - - if (taosRenameFile(file, realfile) != 0) { - terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR; - dError("failed to rename %s since %s", file, terrstr()); - return -1; - } - - dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped); - return 0; -} - -static int32_t dndStartQnodeWorker(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - if (dndInitWorker(pDnode, &pMgmt->queryWorker, DND_WORKER_SINGLE, "qnode-query", 0, 1, dndProcessQnodeQueue) != 0) { - dError("failed to start qnode query worker since %s", terrstr()); - return -1; - } - - if (dndInitWorker(pDnode, &pMgmt->fetchWorker, DND_WORKER_SINGLE, "qnode-fetch", 0, 1, dndProcessQnodeQueue) != 0) { - dError("failed to start qnode fetch worker since %s", terrstr()); - return -1; - } - - return 0; -} - -static void dndStopQnodeWorker(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - - taosWLockLatch(&pMgmt->latch); - pMgmt->deployed = 0; - taosWUnLockLatch(&pMgmt->latch); - - while (pMgmt->refCount > 0) { - taosMsleep(10); - } - - dndCleanupWorker(&pMgmt->queryWorker); - dndCleanupWorker(&pMgmt->fetchWorker); -} - -static void dndBuildQnodeOption(SDnode *pDnode, SQnodeOpt *pOption) { - pOption->pDnode = pDnode; - pOption->sendReqToDnodeFp = dndSendReqToDnode; - pOption->sendReqToMnodeFp = dndSendReqToMnode; - pOption->sendRedirectRspFp = dndSendRedirectRsp; - pOption->dnodeId = dndGetDnodeId(pDnode); - pOption->clusterId = dndGetClusterId(pDnode); - pOption->sver = tsVersion; -} - -static int32_t dndOpenQnode(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - - SQnode *pQnode = dndAcquireQnode(pDnode); - if (pQnode != NULL) { - dndReleaseQnode(pDnode, pQnode); - terrno = TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED; - dError("failed to create qnode since %s", terrstr()); - return -1; - } - - SQnodeOpt option = {0}; - dndBuildQnodeOption(pDnode, &option); - - pQnode = qndOpen(&option); - if (pQnode == NULL) { - dError("failed to open qnode since %s", terrstr()); - return -1; - } - - if (dndStartQnodeWorker(pDnode) != 0) { - dError("failed to start qnode worker since %s", terrstr()); - qndClose(pQnode); - return -1; - } - - pMgmt->deployed = 1; - if (dndWriteQnodeFile(pDnode) != 0) { - pMgmt->deployed = 0; - dError("failed to write qnode file since %s", terrstr()); - dndStopQnodeWorker(pDnode); - qndClose(pQnode); - return -1; - } - - taosWLockLatch(&pMgmt->latch); - pMgmt->pQnode = pQnode; - taosWUnLockLatch(&pMgmt->latch); - - dInfo("qnode open successfully"); - return 0; -} - -static int32_t dndDropQnode(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - - SQnode *pQnode = dndAcquireQnode(pDnode); - if (pQnode == NULL) { - dError("failed to drop qnode since %s", terrstr()); - return -1; - } - - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 1; - taosRUnLockLatch(&pMgmt->latch); - - if (dndWriteQnodeFile(pDnode) != 0) { - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 0; - taosRUnLockLatch(&pMgmt->latch); - - dndReleaseQnode(pDnode, pQnode); - dError("failed to drop qnode since %s", terrstr()); - return -1; - } - - dndReleaseQnode(pDnode, pQnode); - dndStopQnodeWorker(pDnode); - pMgmt->deployed = 0; - dndWriteQnodeFile(pDnode); - qndClose(pQnode); - pMgmt->pQnode = NULL; - - return 0; -} - -int32_t dndProcessCreateQnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDCreateQnodeReq createReq = {0}; - if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (createReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_QNODE_INVALID_OPTION; - dError("failed to create qnode since %s", terrstr()); - return -1; - } else { - return dndOpenQnode(pDnode); - } -} - -int32_t dndProcessDropQnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDDropQnodeReq dropReq = {0}; - if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (dropReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_QNODE_INVALID_OPTION; - dError("failed to drop qnode since %s", terrstr()); - return -1; - } else { - return dndDropQnode(pDnode); - } -} - -static void dndProcessQnodeQueue(SDnode *pDnode, SRpcMsg *pMsg) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - SRpcMsg *pRsp = NULL; - int32_t code = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; - - SQnode *pQnode = dndAcquireQnode(pDnode); - if (pQnode != NULL) { - code = qndProcessMsg(pQnode, pMsg, &pRsp); - } - dndReleaseQnode(pDnode, pQnode); - - if (pMsg->msgType & 1u) { - if (pRsp != NULL) { - pRsp->ahandle = pMsg->ahandle; - rpcSendResponse(pRsp); - free(pRsp); - } else { - if (code != 0) code = terrno; - SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rpcRsp); - } - } - - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); -} - -static void dndWriteQnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) { - int32_t code = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; - - SQnode *pQnode = dndAcquireQnode(pDnode); - if (pQnode != NULL) { - code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)); - } - dndReleaseQnode(pDnode, pQnode); - - if (code != 0) { - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - } -} - -void dndProcessQnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteQnodeMsgToWorker(pDnode, &pDnode->qmgmt.queryWorker, pMsg); -} - -void dndProcessQnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteQnodeMsgToWorker(pDnode, &pDnode->qmgmt.queryWorker, pMsg); -} - -int32_t dndInitQnode(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - taosInitRWLatch(&pMgmt->latch); - - if (dndReadQnodeFile(pDnode) != 0) { - return -1; - } - - if (pMgmt->dropped) return 0; - if (!pMgmt->deployed) return 0; - - return dndOpenQnode(pDnode); -} - -void dndCleanupQnode(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - if (pMgmt->pQnode) { - dndStopQnodeWorker(pDnode); - qndClose(pMgmt->pQnode); - pMgmt->pQnode = NULL; - } -} diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c deleted file mode 100644 index ea06c8c751..0000000000 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ /dev/null @@ -1,505 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndSnode.h" -#include "dndMgmt.h" -#include "dndTransport.h" -#include "dndWorker.h" - -typedef struct { - int32_t vgId; - int32_t refCount; - int32_t snVersion; - int8_t dropped; - char *path; - SSnode *pImpl; - STaosQueue *pSharedQ; - STaosQueue *pUniqueQ; -} SSnodeObj; - -static void dndProcessSnodeSharedQueue(SDnode *pDnode, SRpcMsg *pMsg); - -static void dndProcessSnodeUniqueQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs); - -static SSnode *dndAcquireSnode(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - SSnode *pSnode = NULL; - int32_t refCount = 0; - - taosRLockLatch(&pMgmt->latch); - if (pMgmt->deployed && !pMgmt->dropped && pMgmt->pSnode != NULL) { - refCount = atomic_add_fetch_32(&pMgmt->refCount, 1); - pSnode = pMgmt->pSnode; - } else { - terrno = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; - } - taosRUnLockLatch(&pMgmt->latch); - - if (pSnode != NULL) { - dTrace("acquire snode, refCount:%d", refCount); - } - return pSnode; -} - -static void dndReleaseSnode(SDnode *pDnode, SSnode *pSnode) { - if (pSnode == NULL) return; - - SSnodeMgmt *pMgmt = &pDnode->smgmt; - taosRLockLatch(&pMgmt->latch); - int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); - taosRUnLockLatch(&pMgmt->latch); - dTrace("release snode, refCount:%d", refCount); -} - -static int32_t dndReadSnodeFile(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - int32_t code = TSDB_CODE_DND_SNODE_READ_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "r"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); - if (pFile == NULL) { - dDebug("file %s not exist", file); - code = 0; - goto PRASE_SNODE_OVER; - } - - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", file); - goto PRASE_SNODE_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); - goto PRASE_SNODE_OVER; - } - - cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); - if (!deployed || deployed->type != cJSON_Number) { - dError("failed to read %s since deployed not found", file); - goto PRASE_SNODE_OVER; - } - pMgmt->deployed = deployed->valueint; - - cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", file); - goto PRASE_SNODE_OVER; - } - pMgmt->dropped = dropped->valueint; - - code = 0; - dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); - -PRASE_SNODE_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (pFile != NULL) taosCloseFile(&pFile); - - terrno = code; - return code; -} - -static int32_t dndWriteSnodeFile(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "w"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR; - dError("failed to write %s since %s", file, terrstr()); - return -1; - } - - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", pMgmt->deployed); - len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); - len += snprintf(content + len, maxLen - len, "}\n"); - - taosWriteFile(pFile, content, len); - taosFsyncFile(pFile); - taosCloseFile(&pFile); - free(content); - - char realfile[PATH_MAX + 20]; - snprintf(realfile, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); - - if (taosRenameFile(file, realfile) != 0) { - terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR; - dError("failed to rename %s since %s", file, terrstr()); - return -1; - } - - dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped); - return 0; -} - -static int32_t dndStartSnodeWorker(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - pMgmt->uniqueWorkers = taosArrayInit(0, sizeof(void *)); - for (int32_t i = 0; i < SND_UNIQUE_THREAD_NUM; i++) { - SDnodeWorker *pUniqueWorker = malloc(sizeof(SDnodeWorker)); - if (pUniqueWorker == NULL) { - return -1; - } - if (dndInitWorker(pDnode, pUniqueWorker, DND_WORKER_MULTI, "snode-unique", 1, 1, dndProcessSnodeSharedQueue) != 0) { - dError("failed to start snode unique worker since %s", terrstr()); - return -1; - } - taosArrayPush(pMgmt->uniqueWorkers, &pUniqueWorker); - } - if (dndInitWorker(pDnode, &pMgmt->sharedWorker, DND_WORKER_SINGLE, "snode-shared", SND_SHARED_THREAD_NUM, - SND_SHARED_THREAD_NUM, dndProcessSnodeSharedQueue)) { - dError("failed to start snode shared worker since %s", terrstr()); - return -1; - } - - return 0; -} - -static void dndStopSnodeWorker(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - - taosWLockLatch(&pMgmt->latch); - pMgmt->deployed = 0; - taosWUnLockLatch(&pMgmt->latch); - - while (pMgmt->refCount > 0) { - taosMsleep(10); - } - - for (int32_t i = 0; i < taosArrayGetSize(pMgmt->uniqueWorkers); i++) { - SDnodeWorker *worker = taosArrayGetP(pMgmt->uniqueWorkers, i); - dndCleanupWorker(worker); - } - taosArrayDestroy(pMgmt->uniqueWorkers); -} - -static void dndBuildSnodeOption(SDnode *pDnode, SSnodeOpt *pOption) { - pOption->pDnode = pDnode; - pOption->sendReqToDnodeFp = dndSendReqToDnode; - pOption->sendReqToMnodeFp = dndSendReqToMnode; - pOption->sendRedirectRspFp = dndSendRedirectRsp; - pOption->dnodeId = dndGetDnodeId(pDnode); - pOption->clusterId = dndGetClusterId(pDnode); - pOption->sver = tsVersion; -} - -static int32_t dndOpenSnode(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode != NULL) { - dndReleaseSnode(pDnode, pSnode); - terrno = TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED; - dError("failed to create snode since %s", terrstr()); - return -1; - } - - SSnodeOpt option = {0}; - dndBuildSnodeOption(pDnode, &option); - - pSnode = sndOpen(pDnode->dir.snode, &option); - if (pSnode == NULL) { - dError("failed to open snode since %s", terrstr()); - return -1; - } - - if (dndStartSnodeWorker(pDnode) != 0) { - dError("failed to start snode worker since %s", terrstr()); - sndClose(pSnode); - return -1; - } - - pMgmt->deployed = 1; - if (dndWriteSnodeFile(pDnode) != 0) { - pMgmt->deployed = 0; - dError("failed to write snode file since %s", terrstr()); - dndStopSnodeWorker(pDnode); - sndClose(pSnode); - return -1; - } - - taosWLockLatch(&pMgmt->latch); - pMgmt->pSnode = pSnode; - taosWUnLockLatch(&pMgmt->latch); - - dInfo("snode open successfully"); - return 0; -} - -static int32_t dndDropSnode(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode == NULL) { - dError("failed to drop snode since %s", terrstr()); - return -1; - } - - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 1; - taosRUnLockLatch(&pMgmt->latch); - - if (dndWriteSnodeFile(pDnode) != 0) { - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 0; - taosRUnLockLatch(&pMgmt->latch); - - dndReleaseSnode(pDnode, pSnode); - dError("failed to drop snode since %s", terrstr()); - return -1; - } - - dndReleaseSnode(pDnode, pSnode); - dndStopSnodeWorker(pDnode); - pMgmt->deployed = 0; - dndWriteSnodeFile(pDnode); - sndClose(pSnode); - pMgmt->pSnode = NULL; - sndDestroy(pDnode->dir.snode); - - return 0; -} - -int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDCreateSnodeReq createReq = {0}; - if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (createReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_SNODE_INVALID_OPTION; - dError("failed to create snode since %s", terrstr()); - return -1; - } else { - return dndOpenSnode(pDnode); - } -} - -int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDDropSnodeReq dropReq = {0}; - if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (dropReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_SNODE_INVALID_OPTION; - dError("failed to drop snode since %s", terrstr()); - return -1; - } else { - return dndDropSnode(pDnode); - } -} - -static void dndProcessSnodeUniqueQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs) { - /*SSnodeMgmt *pMgmt = &pDnode->smgmt;*/ - int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; - - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode != NULL) { - for (int32_t i = 0; i < numOfMsgs; i++) { - SRpcMsg *pMsg = NULL; - taosGetQitem(qall, (void **)&pMsg); - - sndProcessUMsg(pSnode, pMsg); - - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); - } - dndReleaseSnode(pDnode, pSnode); - } else { - for (int32_t i = 0; i < numOfMsgs; i++) { - SRpcMsg *pMsg = NULL; - taosGetQitem(qall, (void **)&pMsg); - SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rpcRsp); - - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); - } - } -} - -static void dndProcessSnodeSharedQueue(SDnode *pDnode, SRpcMsg *pMsg) { - /*SSnodeMgmt *pMgmt = &pDnode->smgmt;*/ - int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; - - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode != NULL) { - sndProcessSMsg(pSnode, pMsg); - dndReleaseSnode(pDnode, pSnode); - } else { - SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rpcRsp); - } - -#if 0 - if (pMsg->msgType & 1u) { - if (pRsp != NULL) { - pRsp->ahandle = pMsg->ahandle; - rpcSendResponse(pRsp); - free(pRsp); - } else { - if (code != 0) code = terrno; - SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rpcRsp); - } - } -#endif - - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); -} - -static FORCE_INLINE int32_t dndGetSWTypeFromMsg(SRpcMsg *pMsg) { - SStreamExecMsgHead *pHead = pMsg->pCont; - pHead->workerType = htonl(pHead->workerType); - return pHead->workerType; -} - -static FORCE_INLINE int32_t dndGetSWIdFromMsg(SRpcMsg *pMsg) { - SMsgHead *pHead = pMsg->pCont; - pHead->streamTaskId = htonl(pHead->streamTaskId); - return pHead->streamTaskId % SND_UNIQUE_THREAD_NUM; -} - -static void dndWriteSnodeMsgToWorkerByMsg(SDnode *pDnode, SRpcMsg *pMsg) { - int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; - - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode != NULL) { - int32_t index = dndGetSWIdFromMsg(pMsg); - SDnodeWorker *pWorker = taosArrayGetP(pDnode->smgmt.uniqueWorkers, index); - code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)); - } - - dndReleaseSnode(pDnode, pSnode); - - if (code != 0) { - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - } -} - -static void dndWriteSnodeMsgToMgmtWorker(SDnode *pDnode, SRpcMsg *pMsg) { - int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; - - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode != NULL) { - SDnodeWorker *pWorker = taosArrayGet(pDnode->smgmt.uniqueWorkers, 0); - code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)); - } - dndReleaseSnode(pDnode, pSnode); - - if (code != 0) { - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - } -} - -static void dndWriteSnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) { - int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; - - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode != NULL) { - code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)); - } - dndReleaseSnode(pDnode, pSnode); - - if (code != 0) { - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - } -} - -void dndProcessSnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteSnodeMsgToMgmtWorker(pDnode, pMsg); -} - -void dndProcessSnodeExecMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode != NULL) { - int32_t workerType = dndGetSWTypeFromMsg(pMsg); - if (workerType == SND_WORKER_TYPE__SHARED) { - dndWriteSnodeMsgToWorker(pDnode, &pDnode->smgmt.sharedWorker, pMsg); - } else { - dndWriteSnodeMsgToWorkerByMsg(pDnode, pMsg); - } - } -} - -void dndProcessSnodeUniqueMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteSnodeMsgToWorkerByMsg(pDnode, pMsg); -} - -void dndProcessSnodeSharedMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteSnodeMsgToWorker(pDnode, &pDnode->smgmt.sharedWorker, pMsg); -} - -int32_t dndInitSnode(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - taosInitRWLatch(&pMgmt->latch); - - if (dndReadSnodeFile(pDnode) != 0) { - return -1; - } - - if (pMgmt->dropped) { - dInfo("snode has been deployed and needs to be deleted"); - sndDestroy(pDnode->dir.snode); - return 0; - } - - if (!pMgmt->deployed) return 0; - - return dndOpenSnode(pDnode); -} - -void dndCleanupSnode(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - if (pMgmt->pSnode) { - dndStopSnodeWorker(pDnode); - sndClose(pMgmt->pSnode); - pMgmt->pSnode = NULL; - } -} diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c deleted file mode 100644 index 617b6c0fc3..0000000000 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ /dev/null @@ -1,422 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -/* this file is mainly responsible for the communication between DNODEs. Each - * dnode works as both server and client. Dnode may send status, grant, config - * messages to mnode, mnode may send create/alter/drop table/vnode messages - * to dnode. All theses messages are handled from here - */ - -#define _DEFAULT_SOURCE -#include "dndTransport.h" -#include "dndMgmt.h" -#include "dndMnode.h" -#include "dndSnode.h" -#include "dndVnodes.h" - -#define INTERNAL_USER "_dnd" -#define INTERNAL_CKEY "_key" -#define INTERNAL_SECRET "_pwd" - -static void dndInitMsgFp(STransMgmt *pMgmt) { - // Requests handled by DNODE - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_MNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_MNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_MNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_MNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_MNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_MNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_QNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_QNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_QNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_QNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_SNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_SNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_SNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_SNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_BNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_BNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_BNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_BNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_VNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_VNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_VNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_VNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_VNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_VNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_SYNC_VNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_SYNC_VNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_COMPACT_VNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_COMPACT_VNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CONFIG_DNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CONFIG_DNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_NETWORK_TEST)] = dndProcessMgmtMsg; - - // Requests handled by MNODE - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CONNECT)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_ACCT)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_ACCT)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_ACCT)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_USER)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_USER)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_USER)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_DNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CONFIG_DNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_DNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_MNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_MNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_QNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_QNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_SNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_SNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_BNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_BNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_DB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_USE_DB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_DB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SYNC_DB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_COMPACT_DB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_FUNC)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_FUNC)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_STB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_STB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_TABLE_META)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_VGROUP_LIST)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_QUERY)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_CONN)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_HEARTBEAT)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW_RETRIEVE)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS_RSP)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_TRANS)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GRANT)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GRANT_RSP)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_AUTH)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_AUTH_RSP)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_TOPIC)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_TOPIC)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_TOPIC)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SUBSCRIBE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_MQ_COMMIT_OFFSET)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GET_SUB_EP)] = dndProcessMnodeReadMsg; - - // Requests handled by VNODE - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBMIT)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY)] = dndProcessVnodeQueryMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_CONTINUE)] = dndProcessVnodeQueryMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH_RSP)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_UPDATE_TAG_VAL)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLE_META)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLES_META)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_CONSUME)] = dndProcessVnodeQueryMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_QUERY)] = dndProcessVnodeQueryMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_CONNECT)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_DISCONNECT)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_RES_READY)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TASKS_STATUS)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CANCEL_TASK)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_TASK)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CREATE_STB)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CREATE_STB_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_STB)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_STB_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_STB)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_STB_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CREATE_TABLE)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_TABLE)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES_FETCH)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CONSUME)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_HEARTBEAT)] = dndProcessVnodeFetchMsg; - - // Requests handled by SNODE - pMgmt->msgFp[TMSG_INDEX(TDMT_SND_TASK_DEPLOY)] = dndProcessSnodeMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_SND_TASK_EXEC)] = dndProcessSnodeExecMsg; -} - -static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { - SDnode *pDnode = parent; - STransMgmt *pMgmt = &pDnode->tmgmt; - - tmsg_t msgType = pRsp->msgType; - - if (dndGetStat(pDnode) == DND_STAT_STOPPED) { - if (pRsp == NULL || pRsp->pCont == NULL) return; - dTrace("RPC %p, rsp:%s ignored since dnode exiting, app:%p", pRsp->handle, TMSG_INFO(msgType), pRsp->ahandle); - rpcFreeCont(pRsp->pCont); - return; - } - - DndMsgFp fp = pMgmt->msgFp[TMSG_INDEX(msgType)]; - if (fp != NULL) { - dTrace("RPC %p, rsp:%s will be processed, code:0x%x app:%p", pRsp->handle, TMSG_INFO(msgType), pRsp->code & 0XFFFF, - pRsp->ahandle); - (*fp)(pDnode, pRsp, pEpSet); - } else { - dError("RPC %p, rsp:%s not processed, app:%p", pRsp->handle, TMSG_INFO(msgType), pRsp->ahandle); - rpcFreeCont(pRsp->pCont); - } -} - -static int32_t dndInitClient(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->tmgmt; - - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.label = "D-C"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = dndProcessResponse; - rpcInit.sessions = 1024; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.user = INTERNAL_USER; - rpcInit.ckey = INTERNAL_CKEY; - rpcInit.spi = 1; - rpcInit.parent = pDnode; - - char pass[TSDB_PASSWORD_LEN + 1] = {0}; - taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); - rpcInit.secret = pass; - - pMgmt->clientRpc = rpcOpen(&rpcInit); - if (pMgmt->clientRpc == NULL) { - dError("failed to init dnode rpc client"); - return -1; - } - - dDebug("dnode rpc client is initialized"); - return 0; -} - -static void dndCleanupClient(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->tmgmt; - if (pMgmt->clientRpc) { - rpcClose(pMgmt->clientRpc); - pMgmt->clientRpc = NULL; - dDebug("dnode rpc client is closed"); - } -} - -static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) { - SDnode *pDnode = param; - STransMgmt *pMgmt = &pDnode->tmgmt; - - tmsg_t msgType = pReq->msgType; - if (msgType == TDMT_DND_NETWORK_TEST) { - dTrace("RPC %p, network test req will be processed, app:%p", pReq->handle, pReq->ahandle); - dndProcessStartupReq(pDnode, pReq); - return; - } - - if (dndGetStat(pDnode) == DND_STAT_STOPPED) { - dError("RPC %p, req:%s ignored since dnode exiting, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_OFFLINE, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - rpcFreeCont(pReq->pCont); - return; - } else if (dndGetStat(pDnode) != DND_STAT_RUNNING) { - dError("RPC %p, req:%s ignored since dnode not running, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - rpcFreeCont(pReq->pCont); - return; - } - - if (pReq->pCont == NULL) { - dTrace("RPC %p, req:%s not processed since its empty, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - return; - } - - DndMsgFp fp = pMgmt->msgFp[TMSG_INDEX(msgType)]; - if (fp != NULL) { - dTrace("RPC %p, req:%s will be processed, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); - (*fp)(pDnode, pReq, pEpSet); - } else { - dError("RPC %p, req:%s not processed since no handle, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - rpcFreeCont(pReq->pCont); - } -} - -static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRpcRsp) { - STransMgmt *pMgmt = &pDnode->tmgmt; - - SEpSet epSet = {0}; - dndGetMnodeEpSet(pDnode, &epSet); - rpcSendRecv(pMgmt->clientRpc, &epSet, pRpcMsg, pRpcRsp); -} - -static int32_t dndAuthInternalReq(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { - if (strcmp(user, INTERNAL_USER) == 0) { - char pass[TSDB_PASSWORD_LEN + 1] = {0}; - taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); - memcpy(secret, pass, TSDB_PASSWORD_LEN); - *spi = 1; - *encrypt = 0; - *ckey = 0; - return 0; - } else if (strcmp(user, TSDB_NETTEST_USER) == 0) { - char pass[TSDB_PASSWORD_LEN + 1] = {0}; - taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); - memcpy(secret, pass, TSDB_PASSWORD_LEN); - *spi = 1; - *encrypt = 0; - *ckey = 0; - return 0; - } else { - return -1; - } -} - -static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey) { - SDnode *pDnode = parent; - - if (dndAuthInternalReq(parent, user, spi, encrypt, secret, ckey) == 0) { - dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); - return 0; - } - - if (dndGetUserAuthFromMnode(pDnode, user, spi, encrypt, secret, ckey) == 0) { - dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); - return 0; - } - - if (terrno != TSDB_CODE_APP_NOT_READY) { - dTrace("failed to get user auth from mnode since %s", terrstr()); - return -1; - } - - SAuthReq authReq = {0}; - tstrncpy(authReq.user, user, TSDB_USER_LEN); - int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); - void *pReq = rpcMallocCont(contLen); - tSerializeSAuthReq(pReq, contLen, &authReq); - - SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; - SRpcMsg rpcRsp = {0}; - dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, authReq.spi, authReq.encrypt); - dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp); - - if (rpcRsp.code != 0) { - terrno = rpcRsp.code; - dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr()); - } else { - SAuthRsp authRsp = {0}; - tDeserializeSAuthReq(rpcRsp.pCont, rpcRsp.contLen, &authRsp); - memcpy(secret, authRsp.secret, TSDB_PASSWORD_LEN); - memcpy(ckey, authRsp.ckey, TSDB_PASSWORD_LEN); - *spi = authRsp.spi; - *encrypt = authRsp.encrypt; - dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, authRsp.spi, - authRsp.encrypt); - } - - rpcFreeCont(rpcRsp.pCont); - return rpcRsp.code; -} - -static int32_t dndInitServer(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->tmgmt; - dndInitMsgFp(pMgmt); - - int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); - if (numOfThreads < 1) { - numOfThreads = 1; - } - - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = pDnode->cfg.serverPort; - rpcInit.label = "D-S"; - rpcInit.numOfThreads = numOfThreads; - rpcInit.cfp = dndProcessRequest; - rpcInit.sessions = tsMaxShellConns; - rpcInit.connType = TAOS_CONN_SERVER; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.afp = dndRetrieveUserAuthInfo; - rpcInit.parent = pDnode; - - pMgmt->serverRpc = rpcOpen(&rpcInit); - if (pMgmt->serverRpc == NULL) { - dError("failed to init dnode rpc server"); - return -1; - } - - dDebug("dnode rpc server is initialized"); - return 0; -} - -static void dndCleanupServer(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->tmgmt; - if (pMgmt->serverRpc) { - rpcClose(pMgmt->serverRpc); - pMgmt->serverRpc = NULL; - dDebug("dnode rpc server is closed"); - } -} - -int32_t dndInitTrans(SDnode *pDnode) { - if (dndInitClient(pDnode) != 0) { - return -1; - } - - if (dndInitServer(pDnode) != 0) { - return -1; - } - - dInfo("dnode-transport is initialized"); - return 0; -} - -void dndCleanupTrans(SDnode *pDnode) { - dInfo("dnode-transport start to clean up"); - dndCleanupServer(pDnode); - dndCleanupClient(pDnode); - dInfo("dnode-transport is cleaned up"); -} - -int32_t dndSendReqToDnode(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pReq) { - STransMgmt *pMgmt = &pDnode->tmgmt; - if (pMgmt->clientRpc == NULL) { - terrno = TSDB_CODE_DND_OFFLINE; - return -1; - } - - rpcSendRequest(pMgmt->clientRpc, pEpSet, pReq, NULL); - return 0; -} - -int32_t dndSendReqToMnode(SDnode *pDnode, SRpcMsg *pReq) { - SEpSet epSet = {0}; - dndGetMnodeEpSet(pDnode, &epSet); - return dndSendReqToDnode(pDnode, &epSet, pReq); -} diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c deleted file mode 100644 index d311e1e417..0000000000 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ /dev/null @@ -1,1024 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndVnodes.h" -#include "dndMgmt.h" -#include "dndTransport.h" -#include "sync.h" - -typedef struct { - int32_t vgId; - int32_t vgVersion; - int8_t dropped; - uint64_t dbUid; - char db[TSDB_DB_FNAME_LEN]; - char path[PATH_MAX + 20]; -} SWrapperCfg; - -typedef struct { - int32_t vgId; - int32_t refCount; - int32_t vgVersion; - int8_t dropped; - int8_t accessState; - uint64_t dbUid; - char *db; - char *path; - SVnode *pImpl; - STaosQueue *pWriteQ; - STaosQueue *pSyncQ; - STaosQueue *pApplyQ; - STaosQueue *pQueryQ; - STaosQueue *pFetchQ; -} SVnodeObj; - -typedef struct { - int32_t vnodeNum; - int32_t opened; - int32_t failed; - int32_t threadIndex; - pthread_t thread; - SDnode *pDnode; - SWrapperCfg *pCfgs; -} SVnodeThread; - -static int32_t dndAllocVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode); -static void dndFreeVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode); - -static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SRpcMsg *pMsg); -static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg); -static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs); -static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs); -static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs); -void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SRpcMsg *pMsg); - -static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId); -static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode); -static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl); -static void dndCloseVnode(SDnode *pDnode, SVnodeObj *pVnode); -static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes); -static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_t *numOfVnodes); -static int32_t dndWriteVnodesToFile(SDnode *pDnode); - -static int32_t dndOpenVnodes(SDnode *pDnode); -static void dndCloseVnodes(SDnode *pDnode); - -static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - SVnodeObj *pVnode = NULL; - int32_t refCount = 0; - - taosRLockLatch(&pMgmt->latch); - taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); - if (pVnode == NULL) { - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - } else { - refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - } - taosRUnLockLatch(&pMgmt->latch); - - if (pVnode != NULL) { - dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); - } - - return pVnode; -} - -static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode) { - if (pVnode == NULL) return; - - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - taosRLockLatch(&pMgmt->latch); - int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); - taosRUnLockLatch(&pMgmt->latch); - dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount); -} - -static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - SVnodeObj *pVnode = calloc(1, sizeof(SVnodeObj)); - if (pVnode == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - pVnode->vgId = pCfg->vgId; - pVnode->refCount = 0; - pVnode->dropped = 0; - pVnode->accessState = TSDB_VN_ALL_ACCCESS; - pVnode->pImpl = pImpl; - pVnode->vgVersion = pCfg->vgVersion; - pVnode->dbUid = pCfg->dbUid; - pVnode->db = tstrdup(pCfg->db); - pVnode->path = tstrdup(pCfg->path); - - if (pVnode->path == NULL || pVnode->db == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - if (dndAllocVnodeQueue(pDnode, pVnode) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - taosWLockLatch(&pMgmt->latch); - int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); - taosWUnLockLatch(&pMgmt->latch); - - if (code != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - } - return code; -} - -static void dndCloseVnode(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - taosWLockLatch(&pMgmt->latch); - taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); - taosWUnLockLatch(&pMgmt->latch); - - dndReleaseVnode(pDnode, pVnode); - while (pVnode->refCount > 0) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10); - - dndFreeVnodeQueue(pDnode, pVnode); - vnodeClose(pVnode->pImpl); - pVnode->pImpl = NULL; - - dDebug("vgId:%d, vnode is closed", pVnode->vgId); - - if (pVnode->dropped) { - dDebug("vgId:%d, vnode is destroyed for dropped:%d", pVnode->vgId, pVnode->dropped); - vnodeDestroy(pVnode->path); - } - - free(pVnode->path); - free(pVnode->db); - free(pVnode); -} - -static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - taosRLockLatch(&pMgmt->latch); - - int32_t num = 0; - int32_t size = taosHashGetSize(pMgmt->hash); - SVnodeObj **pVnodes = calloc(size, sizeof(SVnodeObj *)); - - void *pIter = taosHashIterate(pMgmt->hash, NULL); - while (pIter) { - SVnodeObj **ppVnode = pIter; - SVnodeObj *pVnode = *ppVnode; - if (pVnode && num < size) { - int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); - pVnodes[num] = (*ppVnode); - num++; - pIter = taosHashIterate(pMgmt->hash, pIter); - } else { - taosHashCancelIterate(pMgmt->hash, pIter); - } - } - - taosRUnLockLatch(&pMgmt->latch); - *numOfVnodes = num; - - return pVnodes; -} - -static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) { - int32_t code = TSDB_CODE_DND_VNODE_READ_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 30000; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - FILE *fp = NULL; - char file[PATH_MAX + 20] = {0}; - SWrapperCfg *pCfgs = NULL; - - snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); - - // fp = fopen(file, "r"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); - if (pFile == NULL) { - dDebug("file %s not exist", file); - code = 0; - goto PRASE_VNODE_OVER; - } - - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", file); - goto PRASE_VNODE_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); - goto PRASE_VNODE_OVER; - } - - cJSON *vnodes = cJSON_GetObjectItem(root, "vnodes"); - if (!vnodes || vnodes->type != cJSON_Array) { - dError("failed to read %s since vnodes not found", file); - goto PRASE_VNODE_OVER; - } - - int32_t vnodesNum = cJSON_GetArraySize(vnodes); - if (vnodesNum > 0) { - pCfgs = calloc(vnodesNum, sizeof(SWrapperCfg)); - if (pCfgs == NULL) { - dError("failed to read %s since out of memory", file); - goto PRASE_VNODE_OVER; - } - - for (int32_t i = 0; i < vnodesNum; ++i) { - cJSON *vnode = cJSON_GetArrayItem(vnodes, i); - SWrapperCfg *pCfg = &pCfgs[i]; - - cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId"); - if (!vgId || vgId->type != cJSON_Number) { - dError("failed to read %s since vgId not found", file); - goto PRASE_VNODE_OVER; - } - pCfg->vgId = vgId->valueint; - snprintf(pCfg->path, sizeof(pCfg->path), "%s/vnode%d", pDnode->dir.vnodes, pCfg->vgId); - - cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", file); - goto PRASE_VNODE_OVER; - } - pCfg->dropped = dropped->valueint; - - cJSON *vgVersion = cJSON_GetObjectItem(vnode, "vgVersion"); - if (!vgVersion || vgVersion->type != cJSON_Number) { - dError("failed to read %s since vgVersion not found", file); - goto PRASE_VNODE_OVER; - } - pCfg->vgVersion = vgVersion->valueint; - - cJSON *dbUid = cJSON_GetObjectItem(vnode, "dbUid"); - if (!dbUid || dbUid->type != cJSON_String) { - dError("failed to read %s since dbUid not found", file); - goto PRASE_VNODE_OVER; - } - pCfg->dbUid = atoll(dbUid->valuestring); - - cJSON *db = cJSON_GetObjectItem(vnode, "db"); - if (!db || db->type != cJSON_String) { - dError("failed to read %s since db not found", file); - goto PRASE_VNODE_OVER; - } - tstrncpy(pCfg->db, db->valuestring, TSDB_DB_FNAME_LEN); - } - - *ppCfgs = pCfgs; - } - - *numOfVnodes = vnodesNum; - code = 0; - dInfo("succcessed to read file %s", file); - -PRASE_VNODE_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (pFile != NULL) taosCloseFile(&pFile); - - return code; -} - -static int32_t dndWriteVnodesToFile(SDnode *pDnode) { - char file[PATH_MAX + 20] = {0}; - char realfile[PATH_MAX + 20] = {0}; - snprintf(file, PATH_MAX + 20, "%s/vnodes.json.bak", pDnode->dir.vnodes); - snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); - - // FILE *fp = fopen(file, "w"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to write %s since %s", file, terrstr()); - return -1; - } - int32_t numOfVnodes = 0; - SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes); - - int32_t len = 0; - int32_t maxLen = 65536; - char *content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"vnodes\": [\n"); - for (int32_t i = 0; i < numOfVnodes; ++i) { - SVnodeObj *pVnode = pVnodes[i]; - len += snprintf(content + len, maxLen - len, " {\n"); - len += snprintf(content + len, maxLen - len, " \"vgId\": %d,\n", pVnode->vgId); - len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pVnode->dropped); - len += snprintf(content + len, maxLen - len, " \"vgVersion\": %d,\n", pVnode->vgVersion); - len += snprintf(content + len, maxLen - len, " \"dbUid\": \"%" PRIu64 "\",\n", pVnode->dbUid); - len += snprintf(content + len, maxLen - len, " \"db\": \"%s\"\n", pVnode->db); - if (i < numOfVnodes - 1) { - len += snprintf(content + len, maxLen - len, " },\n"); - } else { - len += snprintf(content + len, maxLen - len, " }\n"); - } - } - len += snprintf(content + len, maxLen - len, " ]\n"); - len += snprintf(content + len, maxLen - len, "}\n"); - - taosWriteFile(pFile, content, len); - taosFsyncFile(pFile); - taosCloseFile(&pFile); - free(content); - terrno = 0; - - for (int32_t i = 0; i < numOfVnodes; ++i) { - SVnodeObj *pVnode = pVnodes[i]; - dndReleaseVnode(pDnode, pVnode); - } - - if (pVnodes != NULL) { - free(pVnodes); - } - - dDebug("successed to write %s", realfile); - return taosRenameFile(file, realfile); -} - -static void *dnodeOpenVnodeFunc(void *param) { - SVnodeThread *pThread = param; - SDnode *pDnode = pThread->pDnode; - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - - dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); - setThreadName("open-vnodes"); - - for (int32_t v = 0; v < pThread->vnodeNum; ++v) { - SWrapperCfg *pCfg = &pThread->pCfgs[v]; - - char stepDesc[TSDB_STEP_DESC_LEN] = {0}; - snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pCfg->vgId, - pMgmt->stat.openVnodes, pMgmt->stat.totalVnodes); - dndReportStartup(pDnode, "open-vnodes", stepDesc); - - SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid}; - SVnode *pImpl = vnodeOpen(pCfg->path, &cfg); - if (pImpl == NULL) { - dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); - pThread->failed++; - } else { - dndOpenVnode(pDnode, pCfg, pImpl); - dDebug("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex); - pThread->opened++; - } - - atomic_add_fetch_32(&pMgmt->stat.openVnodes, 1); - } - - dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, - pThread->failed); - return NULL; -} - -static int32_t dndOpenVnodes(SDnode *pDnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - taosInitRWLatch(&pMgmt->latch); - - pMgmt->hash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - if (pMgmt->hash == NULL) { - dError("failed to init vnode hash"); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - SWrapperCfg *pCfgs = NULL; - int32_t numOfVnodes = 0; - if (dndGetVnodesFromFile(pDnode, &pCfgs, &numOfVnodes) != 0) { - dInfo("failed to get vnode list from disk since %s", terrstr()); - return -1; - } - - pMgmt->stat.totalVnodes = numOfVnodes; - - int32_t threadNum = tsNumOfCores; -#if 1 - threadNum = 1; -#endif - - int32_t vnodesPerThread = numOfVnodes / threadNum + 1; - - SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread)); - for (int32_t t = 0; t < threadNum; ++t) { - threads[t].threadIndex = t; - threads[t].pDnode = pDnode; - threads[t].pCfgs = calloc(vnodesPerThread, sizeof(SWrapperCfg)); - } - - for (int32_t v = 0; v < numOfVnodes; ++v) { - int32_t t = v % threadNum; - SVnodeThread *pThread = &threads[t]; - pThread->pCfgs[pThread->vnodeNum++] = pCfgs[v]; - } - - dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes); - - for (int32_t t = 0; t < threadNum; ++t) { - SVnodeThread *pThread = &threads[t]; - if (pThread->vnodeNum == 0) continue; - - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&pThread->thread, &thAttr, dnodeOpenVnodeFunc, pThread) != 0) { - dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno)); - } - - pthread_attr_destroy(&thAttr); - } - - for (int32_t t = 0; t < threadNum; ++t) { - SVnodeThread *pThread = &threads[t]; - if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) { - pthread_join(pThread->thread, NULL); - } - free(pThread->pCfgs); - } - free(threads); - free(pCfgs); - - if (pMgmt->stat.openVnodes != pMgmt->stat.totalVnodes) { - dError("there are total vnodes:%d, opened:%d", pMgmt->stat.totalVnodes, pMgmt->stat.openVnodes); - return -1; - } else { - dInfo("total vnodes:%d open successfully", pMgmt->stat.totalVnodes); - return 0; - } -} - -static void dndCloseVnodes(SDnode *pDnode) { - dInfo("start to close all vnodes"); - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - - int32_t numOfVnodes = 0; - SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes); - - for (int32_t i = 0; i < numOfVnodes; ++i) { - dndCloseVnode(pDnode, pVnodes[i]); - } - - if (pVnodes != NULL) { - free(pVnodes); - } - - if (pMgmt->hash != NULL) { - taosHashCleanup(pMgmt->hash); - pMgmt->hash = NULL; - } - - dInfo("total vnodes:%d are all closed", numOfVnodes); -} - -static void dndGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { - pCfg->vgId = pCreate->vgId; - pCfg->wsize = pCreate->cacheBlockSize; - pCfg->ssize = pCreate->cacheBlockSize; - pCfg->lsize = pCreate->cacheBlockSize; - pCfg->isHeapAllocator = true; - pCfg->ttl = 4; - pCfg->keep = pCreate->daysToKeep0; - pCfg->streamMode = pCreate->streamMode; - pCfg->isWeak = true; - pCfg->tsdbCfg.keep = pCreate->daysToKeep0; - pCfg->tsdbCfg.keep1 = pCreate->daysToKeep2; - pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0; - pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize; - pCfg->metaCfg.lruSize = pCreate->cacheBlockSize; - pCfg->walCfg.fsyncPeriod = pCreate->fsyncPeriod; - pCfg->walCfg.level = pCreate->walLevel; - pCfg->walCfg.retentionPeriod = 10; - pCfg->walCfg.retentionSize = 128; - pCfg->walCfg.rollPeriod = 128; - pCfg->walCfg.segSize = 128; - pCfg->walCfg.vgId = pCreate->vgId; - pCfg->hashBegin = pCreate->hashBegin; - pCfg->hashEnd = pCreate->hashEnd; - pCfg->hashMethod = pCreate->hashMethod; -} - -static void dndGenerateWrapperCfg(SDnode *pDnode, SCreateVnodeReq *pCreate, SWrapperCfg *pCfg) { - memcpy(pCfg->db, pCreate->db, TSDB_DB_FNAME_LEN); - pCfg->dbUid = pCreate->dbUid; - pCfg->dropped = 0; - snprintf(pCfg->path, sizeof(pCfg->path), "%s/vnode%d", pDnode->dir.vnodes, pCreate->vgId); - pCfg->vgId = pCreate->vgId; - pCfg->vgVersion = pCreate->vgVersion; -} - -int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SCreateVnodeReq createReq = {0}; - if (tDeserializeSCreateVnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - dDebug("vgId:%d, create vnode req is received", createReq.vgId); - - SVnodeCfg vnodeCfg = {0}; - dndGenerateVnodeCfg(&createReq, &vnodeCfg); - - SWrapperCfg wrapperCfg = {0}; - dndGenerateWrapperCfg(pDnode, &createReq, &wrapperCfg); - - if (createReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_VNODE_INVALID_OPTION; - dDebug("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); - return -1; - } - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, createReq.vgId); - if (pVnode != NULL) { - dDebug("vgId:%d, already exist", createReq.vgId); - dndReleaseVnode(pDnode, pVnode); - terrno = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED; - return -1; - } - - vnodeCfg.pDnode = pDnode; - vnodeCfg.pTfs = pDnode->pTfs; - vnodeCfg.dbId = wrapperCfg.dbUid; - SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg); - if (pImpl == NULL) { - dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); - return -1; - } - - int32_t code = dndOpenVnode(pDnode, &wrapperCfg, pImpl); - if (code != 0) { - dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr()); - vnodeClose(pImpl); - vnodeDestroy(wrapperCfg.path); - terrno = code; - return code; - } - - code = dndWriteVnodesToFile(pDnode); - if (code != 0) { - vnodeClose(pImpl); - vnodeDestroy(wrapperCfg.path); - terrno = code; - return code; - } - - return 0; -} - -int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SAlterVnodeReq alterReq = {0}; - if (tDeserializeSCreateVnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - dDebug("vgId:%d, alter vnode req is received", alterReq.vgId); - - SVnodeCfg vnodeCfg = {0}; - dndGenerateVnodeCfg(&alterReq, &vnodeCfg); - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, alterReq.vgId); - if (pVnode == NULL) { - dDebug("vgId:%d, failed to alter vnode since %s", alterReq.vgId, terrstr()); - return -1; - } - - if (alterReq.vgVersion == pVnode->vgVersion) { - dndReleaseVnode(pDnode, pVnode); - dDebug("vgId:%d, no need to alter vnode cfg for version unchanged ", alterReq.vgId); - return 0; - } - - if (vnodeAlter(pVnode->pImpl, &vnodeCfg) != 0) { - dError("vgId:%d, failed to alter vnode since %s", alterReq.vgId, terrstr()); - dndReleaseVnode(pDnode, pVnode); - return -1; - } - - int32_t oldVersion = pVnode->vgVersion; - pVnode->vgVersion = alterReq.vgVersion; - int32_t code = dndWriteVnodesToFile(pDnode); - if (code != 0) { - pVnode->vgVersion = oldVersion; - } - - dndReleaseVnode(pDnode, pVnode); - return code; -} - -int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDropVnodeReq dropReq = {0}; - if (tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - int32_t vgId = dropReq.vgId; - dDebug("vgId:%d, drop vnode req is received", vgId); - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); - if (pVnode == NULL) { - dDebug("vgId:%d, failed to drop since %s", vgId, terrstr()); - terrno = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; - return -1; - } - - pVnode->dropped = 1; - if (dndWriteVnodesToFile(pDnode) != 0) { - pVnode->dropped = 0; - dndReleaseVnode(pDnode, pVnode); - return -1; - } - - dndCloseVnode(pDnode, pVnode); - dndWriteVnodesToFile(pDnode); - - return 0; -} - -int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SSyncVnodeReq syncReq = {0}; - tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &syncReq); - - int32_t vgId = syncReq.vgId; - dDebug("vgId:%d, sync vnode req is received", vgId); - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); - if (pVnode == NULL) { - dDebug("vgId:%d, failed to sync since %s", vgId, terrstr()); - return -1; - } - - if (vnodeSync(pVnode->pImpl) != 0) { - dError("vgId:%d, failed to sync vnode since %s", vgId, terrstr()); - dndReleaseVnode(pDnode, pVnode); - return -1; - } - - dndReleaseVnode(pDnode, pVnode); - return 0; -} - -int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SCompactVnodeReq compatcReq = {0}; - tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &compatcReq); - - int32_t vgId = compatcReq.vgId; - dDebug("vgId:%d, compact vnode req is received", vgId); - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); - if (pVnode == NULL) { - dDebug("vgId:%d, failed to compact since %s", vgId, terrstr()); - return -1; - } - - if (vnodeCompact(pVnode->pImpl) != 0) { - dError("vgId:%d, failed to compact vnode since %s", vgId, terrstr()); - dndReleaseVnode(pDnode, pVnode); - return -1; - } - - dndReleaseVnode(pDnode, pVnode); - return 0; -} - -static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) { vnodeProcessQueryMsg(pVnode->pImpl, pMsg); } - -static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) { vnodeProcessFetchMsg(pVnode->pImpl, pMsg); } - -static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { - SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); - - for (int32_t i = 0; i < numOfMsgs; ++i) { - SRpcMsg *pMsg = NULL; - taosGetQitem(qall, (void **)&pMsg); - void *ptr = taosArrayPush(pArray, &pMsg); - assert(ptr != NULL); - } - - vnodeProcessWMsgs(pVnode->pImpl, pArray); - - for (size_t i = 0; i < numOfMsgs; i++) { - SRpcMsg *pRsp = NULL; - SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); - int32_t code = vnodeApplyWMsg(pVnode->pImpl, pMsg, &pRsp); - if (pRsp != NULL) { - pRsp->ahandle = pMsg->ahandle; - rpcSendResponse(pRsp); - free(pRsp); - } else { - if (code != 0) code = terrno; - SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rpcRsp); - } - } - - for (size_t i = 0; i < numOfMsgs; i++) { - SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); - } - - taosArrayDestroy(pArray); -} - -static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { - SRpcMsg *pMsg = NULL; - - for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(qall, (void **)&pMsg); - - // todo - SRpcMsg *pRsp = NULL; - (void)vnodeApplyWMsg(pVnode->pImpl, pMsg, &pRsp); - } -} - -static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { - SRpcMsg *pMsg = NULL; - - for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(qall, (void **)&pMsg); - - // todo - SRpcMsg *pRsp = NULL; - (void)vnodeProcessSyncReq(pVnode->pImpl, pMsg, &pRsp); - } -} - -static int32_t dndWriteRpcMsgToVnodeQueue(STaosQueue *pQueue, SRpcMsg *pRpcMsg, bool sendRsp) { - int32_t code = 0; - - if (pQueue == NULL) { - code = TSDB_CODE_MSG_NOT_PROCESSED; - } else { - SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg)); - if (pMsg == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - } else { - *pMsg = *pRpcMsg; - if (taosWriteQitem(pQueue, pMsg) != 0) { - code = TSDB_CODE_OUT_OF_MEMORY; - } - } - } - - if (code != TSDB_CODE_SUCCESS && sendRsp) { - if (pRpcMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pRpcMsg->pCont); - } - - return code; -} - -static SVnodeObj *dndAcquireVnodeFromMsg(SDnode *pDnode, SRpcMsg *pMsg) { - SMsgHead *pHead = pMsg->pCont; - pHead->contLen = htonl(pHead->contLen); - pHead->vgId = htonl(pHead->vgId); - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId); - if (pVnode == NULL) { - dError("vgId:%d, failed to acquire vnode while process req", pHead->vgId); - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - } - - return pVnode; -} - -void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); - if (pVnode != NULL) { - (void)dndWriteRpcMsgToVnodeQueue(pVnode->pWriteQ, pMsg, true); - dndReleaseVnode(pDnode, pVnode); - } -} - -void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); - if (pVnode != NULL) { - (void)dndWriteRpcMsgToVnodeQueue(pVnode->pSyncQ, pMsg, true); - dndReleaseVnode(pDnode, pVnode); - } -} - -void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); - if (pVnode != NULL) { - (void)dndWriteRpcMsgToVnodeQueue(pVnode->pQueryQ, pMsg, true); - dndReleaseVnode(pDnode, pVnode); - } -} - -void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); - if (pVnode != NULL) { - (void)dndWriteRpcMsgToVnodeQueue(pVnode->pFetchQ, pMsg, true); - dndReleaseVnode(pDnode, pVnode); - } -} - -int32_t dndPutReqToVQueryQ(SDnode *pDnode, SRpcMsg *pMsg) { - SMsgHead *pHead = pMsg->pCont; - // pHead->vgId = htonl(pHead->vgId); - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId); - if (pVnode == NULL) return -1; - - int32_t code = dndWriteRpcMsgToVnodeQueue(pVnode->pQueryQ, pMsg, false); - dndReleaseVnode(pDnode, pVnode); - return code; -} - -static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SRpcMsg *pMsg) { - SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); - if (pVnode == NULL) return -1; - - int32_t code = taosWriteQitem(pVnode->pApplyQ, pMsg); - dndReleaseVnode(pDnode, pVnode); - return code; -} - -static int32_t dndInitVnodeWorkers(SDnode *pDnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - - int32_t maxFetchThreads = 4; - int32_t minFetchThreads = TMIN(maxFetchThreads, tsNumOfCores); - int32_t minQueryThreads = TMAX((int32_t)(tsNumOfCores * tsRatioOfQueryCores), 1); - int32_t maxQueryThreads = minQueryThreads; - int32_t maxWriteThreads = TMAX(tsNumOfCores, 1); - int32_t maxSyncThreads = TMAX(tsNumOfCores / 2, 1); - - SQWorkerPool *pQPool = &pMgmt->queryPool; - pQPool->name = "vnode-query"; - pQPool->min = minQueryThreads; - pQPool->max = maxQueryThreads; - if (tQWorkerInit(pQPool) != 0) return -1; - - SFWorkerPool *pFPool = &pMgmt->fetchPool; - pFPool->name = "vnode-fetch"; - pFPool->min = minFetchThreads; - pFPool->max = maxFetchThreads; - if (tFWorkerInit(pFPool) != 0) return -1; - - SWWorkerPool *pWPool = &pMgmt->writePool; - pWPool->name = "vnode-write"; - pWPool->max = maxWriteThreads; - if (tWWorkerInit(pWPool) != 0) return -1; - - pWPool = &pMgmt->syncPool; - pWPool->name = "vnode-sync"; - pWPool->max = maxSyncThreads; - if (tWWorkerInit(pWPool) != 0) return -1; - - dDebug("vnode workers is initialized"); - return 0; -} - -static void dndCleanupVnodeWorkers(SDnode *pDnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tFWorkerCleanup(&pMgmt->fetchPool); - tQWorkerCleanup(&pMgmt->queryPool); - tWWorkerCleanup(&pMgmt->writePool); - tWWorkerCleanup(&pMgmt->syncPool); - dDebug("vnode workers is closed"); -} - -static int32_t dndAllocVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - - pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)dndProcessVnodeWriteQueue); - pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)dndProcessVnodeApplyQueue); - pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)dndProcessVnodeSyncQueue); - pVnode->pFetchQ = tFWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)dndProcessVnodeFetchQueue); - pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)dndProcessVnodeQueryQueue); - - if (pVnode->pApplyQ == NULL || pVnode->pWriteQ == NULL || pVnode->pSyncQ == NULL || pVnode->pFetchQ == NULL || - pVnode->pQueryQ == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static void dndFreeVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ); - tFWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); - tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); - tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ); - tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); - pVnode->pWriteQ = NULL; - pVnode->pApplyQ = NULL; - pVnode->pSyncQ = NULL; - pVnode->pFetchQ = NULL; - pVnode->pQueryQ = NULL; -} - -int32_t dndInitVnodes(SDnode *pDnode) { - dInfo("dnode-vnodes start to init"); - - if (dndInitVnodeWorkers(pDnode) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - dError("failed to init vnode workers since %s", terrstr()); - return -1; - } - - if (dndOpenVnodes(pDnode) != 0) { - dError("failed to open vnodes since %s", terrstr()); - return -1; - } - - dInfo("dnode-vnodes is initialized"); - return 0; -} - -void dndCleanupVnodes(SDnode *pDnode) { - dInfo("dnode-vnodes start to clean up"); - dndCloseVnodes(pDnode); - dndCleanupVnodeWorkers(pDnode); - dInfo("dnode-vnodes is cleaned up"); -} - -void dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - SVnodesStat *pStat = &pMgmt->stat; - int32_t totalVnodes = 0; - int32_t masterNum = 0; - int64_t numOfSelectReqs = 0; - int64_t numOfInsertReqs = 0; - int64_t numOfInsertSuccessReqs = 0; - int64_t numOfBatchInsertReqs = 0; - int64_t numOfBatchInsertSuccessReqs = 0; - - taosRLockLatch(&pMgmt->latch); - - void *pIter = taosHashIterate(pMgmt->hash, NULL); - while (pIter) { - SVnodeObj **ppVnode = pIter; - if (ppVnode == NULL || *ppVnode == NULL) continue; - - SVnodeObj *pVnode = *ppVnode; - SVnodeLoad vload = {0}; - vnodeGetLoad(pVnode->pImpl, &vload); - taosArrayPush(pLoads, &vload); - - numOfSelectReqs += vload.numOfSelectReqs; - numOfInsertReqs += vload.numOfInsertReqs; - numOfInsertSuccessReqs += vload.numOfInsertSuccessReqs; - numOfBatchInsertReqs += vload.numOfBatchInsertReqs; - numOfBatchInsertSuccessReqs += vload.numOfBatchInsertSuccessReqs; - totalVnodes++; - if (vload.role == TAOS_SYNC_STATE_LEADER) masterNum++; - - pIter = taosHashIterate(pMgmt->hash, pIter); - } - - taosRUnLockLatch(&pMgmt->latch); - - pStat->totalVnodes = totalVnodes; - pStat->masterNum = masterNum; - pStat->numOfSelectReqs = numOfSelectReqs; - pStat->numOfInsertReqs = numOfInsertReqs; - pStat->numOfInsertSuccessReqs = numOfInsertSuccessReqs; - pStat->numOfBatchInsertReqs = numOfBatchInsertReqs; - pStat->numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs; -} diff --git a/source/dnode/mgmt/daemon/CMakeLists.txt b/source/dnode/mgmt/main/CMakeLists.txt similarity index 62% rename from source/dnode/mgmt/daemon/CMakeLists.txt rename to source/dnode/mgmt/main/CMakeLists.txt index 3238bbf3f0..baa486b91f 100644 --- a/source/dnode/mgmt/daemon/CMakeLists.txt +++ b/source/dnode/mgmt/main/CMakeLists.txt @@ -1,5 +1,5 @@ -aux_source_directory(src DAEMON_SRC) -add_executable(taosd ${DAEMON_SRC}) +aux_source_directory(src EXEC_SRC) +add_executable(taosd ${EXEC_SRC}) target_include_directories( taosd diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/main/inc/dndMain.h similarity index 58% rename from source/dnode/mgmt/impl/inc/dndInt.h rename to source/dnode/mgmt/main/inc/dndMain.h index a8530037da..1958d628a0 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/main/inc/dndMain.h @@ -1,3 +1,4 @@ + /* * Copyright (c) 2019 TAOS Data, Inc. * @@ -13,40 +14,21 @@ * along with this program. If not, see . */ -#ifndef _TD_DND_INT_H_ -#define _TD_DND_INT_H_ +#ifndef _TD_DND_MAIN_H_ +#define _TD_DND_MAIN_H_ + +#include "dnode.h" + +#include "taoserror.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tlog.h" +#include "version.h" #ifdef __cplusplus extern "C" { #endif -#include "os.h" - -#include "cJSON.h" -#include "monitor.h" -#include "tcache.h" -#include "tcrc32c.h" -#include "tdatablock.h" -#include "tglobal.h" -#include "thash.h" -#include "tlockfree.h" -#include "tlog.h" -#include "tmsg.h" -#include "tqueue.h" -#include "trpc.h" -#include "tthread.h" -#include "ttime.h" -#include "tworker.h" - -#include "dnode.h" - -#include "bnode.h" -#include "mnode.h" -#include "qnode.h" -#include "snode.h" -#include "vnode.h" -#include "tfs.h" - #define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} #define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} #define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} @@ -54,20 +36,13 @@ extern "C" { #define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} #define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} -typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat; -typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType; -typedef enum { DND_ENV_INIT = 0, DND_ENV_READY = 1, DND_ENV_CLEANUP = 2 } EEnvStat; -typedef void (*DndMsgFp)(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEps); - -EStat dndGetStat(SDnode *pDnode); -void dndSetStat(SDnode *pDnode, EStat stat); -const char *dndStatStr(EStat stat); - -void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc); -void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup); +void dndDumpCfg(); +void dndPrintVersion(); +void dndGenerateGrant(); +SDnodeOpt dndGetOpt(); #ifdef __cplusplus } #endif -#endif /*_TD_DND_INT_H_*/ +#endif /*_TD_DND_MAIN_H_*/ diff --git a/source/dnode/mgmt/main/src/dndMain.c b/source/dnode/mgmt/main/src/dndMain.c new file mode 100644 index 0000000000..3aff3446da --- /dev/null +++ b/source/dnode/mgmt/main/src/dndMain.c @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndMain.h" + +static struct { + bool dumpConfig; + bool generateGrant; + bool printAuth; + bool printVersion; + char envFile[PATH_MAX]; + char apolloUrl[PATH_MAX]; + SDnode *pDnode; +} global = {0}; + +static void dndSigintHandle(int signum, void *info, void *ctx) { + dInfo("signal:%d is received", signum); + SDnode *pDnode = atomic_val_compare_exchange_ptr(&global.pDnode, 0, global.pDnode); + if (pDnode != NULL) { + dndHandleEvent(pDnode, DND_EVENT_STOP); + } +} + +static void dndSetSignalHandle() { + taosSetSignal(SIGTERM, dndSigintHandle); + taosSetSignal(SIGHUP, dndSigintHandle); + taosSetSignal(SIGINT, dndSigintHandle); + taosSetSignal(SIGABRT, dndSigintHandle); + taosSetSignal(SIGBREAK, dndSigintHandle); +} + +static int32_t dndParseOption(int32_t argc, char const *argv[]) { + for (int32_t i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-c") == 0) { + if (i < argc - 1) { + if (strlen(argv[++i]) >= PATH_MAX) { + printf("config file path overflow"); + return -1; + } + tstrncpy(configDir, argv[i], PATH_MAX); + } else { + printf("'-c' requires a parameter, default is %s\n", configDir); + return -1; + } + } else if (strcmp(argv[i], "-C") == 0) { + global.dumpConfig = true; + } else if (strcmp(argv[i], "-k") == 0) { + global.generateGrant = true; + } else if (strcmp(argv[i], "-V") == 0) { + global.printVersion = true; + } else { + } + } + + return 0; +} + +static int32_t dndRunDnode() { + if (dndInit() != 0) { + dInfo("failed to initialize dnode environment since %s", terrstr()); + return -1; + } + + SDnodeOpt option = dndGetOpt(); + + SDnode *pDnode = dndCreate(&option); + if (pDnode == NULL) { + dError("failed to to create dnode object since %s", terrstr()); + return -1; + } else { + global.pDnode = pDnode; + dndSetSignalHandle(); + } + + dInfo("start the TDengine service"); + int32_t code = dndRun(pDnode); + dInfo("start shutting down the TDengine service"); + + global.pDnode = NULL; + dndClose(pDnode); + dndCleanup(); + taosCloseLog(); + taosCleanupCfg(); + return code; +} + +int main(int argc, char const *argv[]) { + if (!taosCheckSystemIsSmallEnd()) { + dError("failed to start TDengine since on non-small-end machines"); + return -1; + } + + if (dndParseOption(argc, argv) != 0) { + return -1; + } + + if (global.generateGrant) { + dndGenerateGrant(); + return 0; + } + + if (global.printVersion) { + dndPrintVersion(); + return 0; + } + + if (taosCreateLog("taosdlog", 1, configDir, global.envFile, global.apolloUrl, NULL, 0) != 0) { + dError("failed to start TDengine since read log config error"); + return -1; + } + + if (taosInitCfg(configDir, global.envFile, global.apolloUrl, NULL, 0) != 0) { + dError("failed to start TDengine since read config error"); + return -1; + } + + if (global.dumpConfig) { + dndDumpCfg(); + taosCleanupCfg(); + taosCloseLog(); + return 0; + } + + return dndRunDnode(); +} diff --git a/source/dnode/mgmt/daemon/src/dmnUtil.c b/source/dnode/mgmt/main/src/dndUtil.c similarity index 55% rename from source/dnode/mgmt/daemon/src/dmnUtil.c rename to source/dnode/mgmt/main/src/dndUtil.c index d58a5968c0..e07ef68c77 100644 --- a/source/dnode/mgmt/daemon/src/dmnUtil.c +++ b/source/dnode/mgmt/main/src/dndUtil.c @@ -14,15 +14,15 @@ */ #define _DEFAULT_SOURCE -#include "dmnInt.h" +#include "dndMain.h" -void dmnGenerateGrant() { +void dndGenerateGrant() { #if 0 grantParseParameter(); #endif } -void dmnPrintVersion() { +void dndPrintVersion() { #ifdef TD_ENTERPRISE char *releaseName = "enterprise"; #else @@ -32,3 +32,24 @@ void dmnPrintVersion() { printf("gitinfo: %s\n", gitinfo); printf("builuInfo: %s\n", buildinfo); } + +void dndDumpCfg() { + SConfig *pCfg = taosGetCfg(); + cfgDumpCfg(pCfg, 0, 1); +} + +SDnodeOpt dndGetOpt() { + SConfig *pCfg = taosGetCfg(); + SDnodeOpt option = {0}; + + option.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32; + tstrncpy(option.dataDir, tsDataDir, sizeof(option.dataDir)); + tstrncpy(option.firstEp, tsFirst, sizeof(option.firstEp)); + tstrncpy(option.secondEp, tsSecond, sizeof(option.firstEp)); + option.serverPort = tsServerPort; + tstrncpy(option.localFqdn, tsLocalFqdn, sizeof(option.localFqdn)); + snprintf(option.localEp, sizeof(option.localEp), "%s:%u", option.localFqdn, option.serverPort); + option.pDisks = tsDiskCfg; + option.numOfDisks = tsDiskCfgNum; + return option; +} diff --git a/source/dnode/mgmt/mnode/inc/mm.h b/source/dnode/mgmt/mnode/inc/mm.h new file mode 100644 index 0000000000..6ed6c42d93 --- /dev/null +++ b/source/dnode/mgmt/mnode/inc/mm.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_MNODE_H_ +#define _TD_DND_MNODE_H_ + +#include "dnd.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void mmGetMgmtFp(SMgmtWrapper *pMgmt); + +int32_t mmGetUserAuth(SMgmtWrapper *pWrapper, char *user, char *spi, char *encrypt, char *secret, char *ckey); +int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, + SMonGrantInfo *pGrantInfo); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_MNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/mnode/inc/mmInt.h b/source/dnode/mgmt/mnode/inc/mmInt.h new file mode 100644 index 0000000000..1751131764 --- /dev/null +++ b/source/dnode/mgmt/mnode/inc/mmInt.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_MNODE_INT_H_ +#define _TD_DND_MNODE_INT_H_ + +#include "mm.h" +#include "mnode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SMnodeMgmt { + SMnode *pMnode; + SDnode *pDnode; + SMgmtWrapper *pWrapper; + const char *path; + SDnodeWorker readWorker; + SDnodeWorker writeWorker; + SDnodeWorker syncWorker; + SReplica replicas[TSDB_MAX_REPLICA]; + int8_t replica; + int8_t selfIndex; +} SMnodeMgmt; + +// mmFile.c +int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed); +int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed); + +// mmInt.c +int32_t mmOpenFromMsg(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq); +int32_t mmDrop(SMgmtWrapper *pWrapper); +int32_t mmAlter(SMnodeMgmt *pMgmt, SDAlterMnodeReq *pReq); + +// mmMsg.c +void mmInitMsgHandles(SMgmtWrapper *pWrapper); +int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); + +// mmWorker.c +int32_t mmStartWorker(SMnodeMgmt *pMgmt); +void mmStopWorker(SMnodeMgmt *pMgmt); +int32_t mmProcessWriteMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t mmProcessSyncMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t mmProcessReadMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpcMsg); +int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpcMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_MNODE_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/mnode/src/mmFile.c b/source/dnode/mgmt/mnode/src/mmFile.c new file mode 100644 index 0000000000..757b397473 --- /dev/null +++ b/source/dnode/mgmt/mnode/src/mmFile.c @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mmInt.h" + +int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed) { + int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; + int32_t len = 0; + int32_t maxLen = 4096; + char *content = calloc(1, maxLen + 1); + cJSON *root = NULL; + char file[PATH_MAX]; + TdFilePtr pFile = NULL; + + snprintf(file, sizeof(file), "%s%smnode.json", pMgmt->path, TD_DIRSEP); + pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { + dDebug("file %s not exist", file); + code = 0; + goto PRASE_MNODE_OVER; + } + + len = (int32_t)taosReadFile(pFile, content, maxLen); + if (len <= 0) { + dError("failed to read %s since content is null", file); + goto PRASE_MNODE_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s since invalid json format", file); + goto PRASE_MNODE_OVER; + } + + cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); + if (!deployed || deployed->type != cJSON_Number) { + dError("failed to read %s since deployed not found", file); + goto PRASE_MNODE_OVER; + } + *pDeployed = deployed->valueint; + + cJSON *mnodes = cJSON_GetObjectItem(root, "mnodes"); + if (!mnodes || mnodes->type != cJSON_Array) { + dError("failed to read %s since nodes not found", file); + goto PRASE_MNODE_OVER; + } + + pMgmt->replica = cJSON_GetArraySize(mnodes); + if (pMgmt->replica <= 0 || pMgmt->replica > TSDB_MAX_REPLICA) { + dError("failed to read %s since mnodes size %d invalid", file, pMgmt->replica); + goto PRASE_MNODE_OVER; + } + + for (int32_t i = 0; i < pMgmt->replica; ++i) { + cJSON *node = cJSON_GetArrayItem(mnodes, i); + if (node == NULL) break; + + SReplica *pReplica = &pMgmt->replicas[i]; + + cJSON *id = cJSON_GetObjectItem(node, "id"); + if (!id || id->type != cJSON_Number) { + dError("failed to read %s since id not found", file); + goto PRASE_MNODE_OVER; + } + pReplica->id = id->valueint; + + cJSON *fqdn = cJSON_GetObjectItem(node, "fqdn"); + if (!fqdn || fqdn->type != cJSON_String || fqdn->valuestring == NULL) { + dError("failed to read %s since fqdn not found", file); + goto PRASE_MNODE_OVER; + } + tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN); + + cJSON *port = cJSON_GetObjectItem(node, "port"); + if (!port || port->type != cJSON_Number) { + dError("failed to read %s since port not found", file); + goto PRASE_MNODE_OVER; + } + pReplica->port = port->valueint; + } + + code = 0; + dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); + +PRASE_MNODE_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (pFile != NULL) taosCloseFile(&pFile); + + terrno = code; + return code; +} + +int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed) { + char file[PATH_MAX]; + snprintf(file, sizeof(file), "%s%smnode.json.bak", pMgmt->path, TD_DIRSEP); + + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno);; + dError("failed to write %s since %s", file, terrstr()); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 4096; + char *content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", deployed); + len += snprintf(content + len, maxLen - len, " \"mnodes\": [{\n"); + for (int32_t i = 0; i < pMgmt->replica; ++i) { + SReplica *pReplica = &pMgmt->replicas[i]; + len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); + len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); + len += snprintf(content + len, maxLen - len, " \"port\": %u\n", pReplica->port); + if (i < pMgmt->replica - 1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }]\n"); + } + } + len += snprintf(content + len, maxLen - len, "}\n"); + + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); + free(content); + + char realfile[PATH_MAX]; + snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP); + + if (taosRenameFile(file, realfile) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno);; + dError("failed to rename %s since %s", file, terrstr()); + return -1; + } + + dInfo("successed to write %s, deployed:%d", realfile, deployed); + return 0; +} diff --git a/source/dnode/mgmt/mnode/src/mmInt.c b/source/dnode/mgmt/mnode/src/mmInt.c new file mode 100644 index 0000000000..a3c2d40510 --- /dev/null +++ b/source/dnode/mgmt/mnode/src/mmInt.c @@ -0,0 +1,257 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mmInt.h" + +static bool mmDeployRequired(SDnode *pDnode) { + if (pDnode->dnodeId > 0) return false; + if (pDnode->clusterId > 0) return false; + if (strcmp(pDnode->localEp, pDnode->firstEp) != 0) return false; + return true; +} + +static int32_t mmRequire(SMgmtWrapper *pWrapper, bool *required) { + SMnodeMgmt mgmt = {0}; + mgmt.path = pWrapper->path; + if (mmReadFile(&mgmt, required) != 0) { + return -1; + } + + if (!(*required)) { + *required = mmDeployRequired(pWrapper->pDnode); + } + + return 0; +} + +static void mmInitOption(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { + SDnode *pDnode = pMgmt->pDnode; + pOption->pWrapper = pMgmt->pWrapper; + pOption->putToWriteQFp = mmPutMsgToWriteQueue; + pOption->putToReadQFp = mmPutMsgToReadQueue; + pOption->sendReqFp = dndSendReqToDnode; + pOption->sendMnodeReqFp = dndSendReqToMnode; + pOption->sendRspFp = dndSendRsp; + pOption->dnodeId = pDnode->dnodeId; + pOption->clusterId = pDnode->clusterId; +} + +static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { + SDnode *pDnode = pMgmt->pDnode; + + mmInitOption(pMgmt, pOption); + pOption->replica = 1; + pOption->selfIndex = 0; + SReplica *pReplica = &pOption->replicas[0]; + pReplica->id = 1; + pReplica->port = pDnode->serverPort; + tstrncpy(pReplica->fqdn, pDnode->localFqdn, TSDB_FQDN_LEN); + + pMgmt->selfIndex = pOption->selfIndex; + pMgmt->replica = pOption->replica; + memcpy(&pMgmt->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); +} + +static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { + mmInitOption(pMgmt, pOption); + pOption->selfIndex = pMgmt->selfIndex; + pOption->replica = pMgmt->replica; + memcpy(&pOption->replicas, pMgmt->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); +} + +static int32_t mmBuildOptionFromReq(SMnodeMgmt *pMgmt, SMnodeOpt *pOption, SDCreateMnodeReq *pCreate) { + mmInitOption(pMgmt, pOption); + + pOption->replica = pCreate->replica; + pOption->selfIndex = -1; + for (int32_t i = 0; i < pCreate->replica; ++i) { + SReplica *pReplica = &pOption->replicas[i]; + pReplica->id = pCreate->replicas[i].id; + pReplica->port = pCreate->replicas[i].port; + memcpy(pReplica->fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN); + if (pReplica->id == pOption->dnodeId) { + pOption->selfIndex = i; + } + } + + if (pOption->selfIndex == -1) { + dError("failed to build mnode options since %s", terrstr()); + return -1; + } + + pMgmt->selfIndex = pOption->selfIndex; + pMgmt->replica = pOption->replica; + memcpy(&pMgmt->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); + return 0; +} + +static int32_t mmOpenImp(SMnodeMgmt *pMgmt, SDCreateMnodeReq *pReq) { + SMnodeOpt option = {0}; + if (pReq != NULL) { + if (mmBuildOptionFromReq(pMgmt, &option, pReq) != 0) { + return -1; + } + } else { + bool deployed = false; + if (mmReadFile(pMgmt, &deployed) != 0) { + dError("failed to read file since %s", terrstr()); + return -1; + } + + if (!deployed) { + dInfo("mnode start to deploy"); + mmBuildOptionForDeploy(pMgmt, &option); + } else { + dInfo("mnode start to open"); + mmBuildOptionForOpen(pMgmt, &option); + } + } + + pMgmt->pMnode = mndOpen(pMgmt->path, &option); + if (pMgmt->pMnode == NULL) { + dError("failed to open mnode since %s", terrstr()); + return -1; + } + + if (mmStartWorker(pMgmt) != 0) { + dError("failed to start mnode worker since %s", terrstr()); + return -1; + } + + bool deployed = true; + if (mmWriteFile(pMgmt, deployed) != 0) { + dError("failed to write mnode file since %s", terrstr()); + return -1; + } + + return 0; +} + +static void mmCloseImp(SMnodeMgmt *pMgmt) { + if (pMgmt->pMnode != NULL) { + mmStopWorker(pMgmt); + mndClose(pMgmt->pMnode); + pMgmt->pMnode = NULL; + } +} + +int32_t mmAlter(SMnodeMgmt *pMgmt, SDAlterMnodeReq *pReq) { + SMnodeOpt option = {0}; + if (pReq != NULL) { + if (mmBuildOptionFromReq(pMgmt, &option, pReq) != 0) { + return -1; + } + } + + return mndAlter(pMgmt->pMnode, &option); +} + +int32_t mmDrop(SMgmtWrapper *pWrapper) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return 0; + + dInfo("mnode-mgmt start to drop"); + bool deployed = false; + if (mmWriteFile(pMgmt, deployed) != 0) { + dError("failed to drop mnode since %s", terrstr()); + return -1; + } + + mmCloseImp(pMgmt); + taosRemoveDir(pMgmt->path); + pWrapper->pMgmt = NULL; + free(pMgmt); + dInfo("mnode-mgmt is dropped"); + return 0; +} + +static void mmClose(SMgmtWrapper *pWrapper) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return; + + dInfo("mnode-mgmt start to cleanup"); + mmCloseImp(pMgmt); + pWrapper->pMgmt = NULL; + free(pMgmt); + dInfo("mnode-mgmt is cleaned up"); +} + +int32_t mmOpenFromMsg(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq) { + dInfo("mnode-mgmt start to init"); + if (walInit() != 0) { + dError("failed to init wal since %s", terrstr()); + return -1; + } + + SMnodeMgmt *pMgmt = calloc(1, sizeof(SMnodeMgmt)); + if (pMgmt == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pMgmt->path = pWrapper->path; + pMgmt->pDnode = pWrapper->pDnode; + pMgmt->pWrapper = pWrapper; + pWrapper->pMgmt = pMgmt; + + int32_t code = mmOpenImp(pMgmt, pReq); + if (code != 0) { + dError("failed to init mnode-mgmt since %s", terrstr()); + mmClose(pWrapper); + } else { + dInfo("mnode-mgmt is initialized"); + } + + return code; +} + +static int32_t mmOpen(SMgmtWrapper *pWrapper) { + return mmOpenFromMsg(pWrapper, NULL); +} + +static int32_t mmStart(SMgmtWrapper *pWrapper) { + dDebug("mnode mgmt start to run"); + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + return mndStart(pMgmt->pMnode); +} + +void mmGetMgmtFp(SMgmtWrapper *pWrapper) { + SMgmtFp mgmtFp = {0}; + mgmtFp.openFp = mmOpen; + mgmtFp.closeFp = mmClose; + mgmtFp.startFp = mmStart; + mgmtFp.createMsgFp = mmProcessCreateReq; + mgmtFp.dropMsgFp = mmProcessDropReq; + mgmtFp.requiredFp = mmRequire; + + mmInitMsgHandles(pWrapper); + pWrapper->name = "mnode"; + pWrapper->fp = mgmtFp; +} + +int32_t mmGetUserAuth(SMgmtWrapper *pWrapper, char *user, char *spi, char *encrypt, char *secret, char *ckey) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + + int32_t code = mndRetriveAuth(pMgmt->pMnode, user, spi, encrypt, secret, ckey); + dTrace("user:%s, retrieve auth spi:%d encrypt:%d", user, *spi, *encrypt); + return code; +} + +int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, + SMonGrantInfo *pGrantInfo) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + return mndGetMonitorInfo(pMgmt->pMnode, pClusterInfo, pVgroupInfo, pGrantInfo); +} \ No newline at end of file diff --git a/source/dnode/mgmt/mnode/src/mmMsg.c b/source/dnode/mgmt/mnode/src/mmMsg.c new file mode 100644 index 0000000000..8dbea36c22 --- /dev/null +++ b/source/dnode/mgmt/mnode/src/mmMsg.c @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mmInt.h" + +int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDCreateMnodeReq createReq = {0}; + if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (createReq.replica <= 1 || createReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to create mnode since %s", terrstr()); + return -1; + } else { + return mmOpenFromMsg(pWrapper, &createReq); + } +} + +int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDDropMnodeReq dropReq = {0}; + if (tDeserializeSMCreateDropMnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (dropReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to drop mnode since %s", terrstr()); + return -1; + } else { + return mmDrop(pWrapper); + } +} + +int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SDnode *pDnode = pMgmt->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDAlterMnodeReq alterReq = {0}; + if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (alterReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to alter mnode since %s", terrstr()); + return -1; + } else { + return mmAlter(pMgmt, &alterReq); + } +} + +void mmInitMsgHandles(SMgmtWrapper *pWrapper) { + // Requests handled by DNODE + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + + // Requests handled by MNODE + dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, (NodeMsgFp)mmProcessReadMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, (NodeMsgFp)mmProcessReadMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, (NodeMsgFp)mmProcessReadMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, (NodeMsgFp)mmProcessReadMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, (NodeMsgFp)mmProcessReadMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, (NodeMsgFp)mmProcessReadMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, (NodeMsgFp)mmProcessReadMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, (NodeMsgFp)mmProcessReadMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, (NodeMsgFp)mmProcessReadMsg); + + // Requests handled by VNODE + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)mmProcessWriteMsg); +} diff --git a/source/dnode/mgmt/mnode/src/mmWorker.c b/source/dnode/mgmt/mnode/src/mmWorker.c new file mode 100644 index 0000000000..4fa2f60b09 --- /dev/null +++ b/source/dnode/mgmt/mnode/src/mmWorker.c @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mmInt.h" + +static void mmProcessQueue(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { + dTrace("msg:%p, will be processed in mnode queue", pMsg); + SRpcMsg *pRpc = &pMsg->rpcMsg; + int32_t code = -1; + + if (pMsg->rpcMsg.msgType != TDMT_DND_ALTER_MNODE) { + pMsg->pNode = pMgmt->pMnode; + code = mndProcessMsg(pMsg); + } else { + code = mmProcessAlterReq(pMgmt, pMsg); + } + + if (pRpc->msgType & 1U) { + if (pRpc->handle == NULL) return; + if (code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0) code = terrno; + SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .contLen = pMsg->rspLen, .pCont = pMsg->pRsp}; + dndSendRsp(pMgmt->pWrapper, &rsp); + } + } + + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pRpc->pCont); + taosFreeQitem(pMsg); +} + +int32_t mmStartWorker(SMnodeMgmt *pMgmt) { + if (dndInitWorker(pMgmt, &pMgmt->readWorker, DND_WORKER_SINGLE, "mnode-read", 0, 1, mmProcessQueue) != 0) { + dError("failed to start mnode read worker since %s", terrstr()); + return -1; + } + + if (dndInitWorker(pMgmt, &pMgmt->writeWorker, DND_WORKER_SINGLE, "mnode-write", 0, 1, mmProcessQueue) != 0) { + dError("failed to start mnode write worker since %s", terrstr()); + return -1; + } + + if (dndInitWorker(pMgmt, &pMgmt->syncWorker, DND_WORKER_SINGLE, "mnode-sync", 0, 1, mmProcessQueue) != 0) { + dError("failed to start mnode sync worker since %s", terrstr()); + return -1; + } + + return 0; +} + +void mmStopWorker(SMnodeMgmt *pMgmt) { + dndCleanupWorker(&pMgmt->readWorker); + dndCleanupWorker(&pMgmt->writeWorker); + dndCleanupWorker(&pMgmt->syncWorker); +} + +static int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SDnodeWorker *pWorker, SNodeMsg *pMsg) { + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); + return dndWriteMsgToWorker(pWorker, pMsg); +} + +int32_t mmProcessWriteMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->writeWorker, pMsg); +} + +int32_t mmProcessSyncMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg); +} + +int32_t mmProcessReadMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg); +} + +static int32_t mmPutRpcMsgToWorker(SMnodeMgmt *pMgmt, SDnodeWorker *pWorker, SRpcMsg *pRpc) { + SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); + if (pMsg == NULL) { + return -1; + } + + dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); + pMsg->rpcMsg = *pRpc; + + int32_t code = mmPutMsgToWorker(pMgmt, pWorker, pMsg); + if (code != 0) { + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pRpc->pCont); + } + + return code; +} + +int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + return mmPutRpcMsgToWorker(pMgmt, &pMgmt->writeWorker, pRpc); +} + +int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + return mmPutRpcMsgToWorker(pMgmt, &pMgmt->readWorker, pRpc); +} diff --git a/source/dnode/mgmt/daemon/inc/dmnInt.h b/source/dnode/mgmt/qnode/inc/qm.h similarity index 70% rename from source/dnode/mgmt/daemon/inc/dmnInt.h rename to source/dnode/mgmt/qnode/inc/qm.h index 8a571352f0..e28ea3e948 100644 --- a/source/dnode/mgmt/daemon/inc/dmnInt.h +++ b/source/dnode/mgmt/qnode/inc/qm.h @@ -1,4 +1,3 @@ - /* * Copyright (c) 2019 TAOS Data, Inc. * @@ -14,28 +13,19 @@ * along with this program. If not, see . */ -#ifndef _TD_DMN_INT_H_ -#define _TD_DMN_INT_H_ +#ifndef _TD_DND_QNODE_H_ +#define _TD_DND_QNODE_H_ -#include "tconfig.h" -#include "dnode.h" -#include "taoserror.h" -#include "tglobal.h" -#include "tlog.h" -#include "version.h" +#include "dnd.h" #ifdef __cplusplus extern "C" { #endif -SDnodeObjCfg dmnGetObjCfg(); - -void dmnDumpCfg(); -void dmnPrintVersion(); -void dmnGenerateGrant(); +void qmGetMgmtFp(SMgmtWrapper *pMgmt); #ifdef __cplusplus } #endif -#endif /*_TD_DMN_INT_H_*/ +#endif /*_TD_DND_QNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/qnode/inc/qmInt.h b/source/dnode/mgmt/qnode/inc/qmInt.h new file mode 100644 index 0000000000..2629486d60 --- /dev/null +++ b/source/dnode/mgmt/qnode/inc/qmInt.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_QNODE_INT_H_ +#define _TD_DND_QNODE_INT_H_ + +#include "qm.h" +#include "qnode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SQnodeMgmt { + SQnode *pQnode; + SDnode *pDnode; + SMgmtWrapper *pWrapper; + const char *path; + SDnodeWorker queryWorker; + SDnodeWorker fetchWorker; +} SQnodeMgmt; + +// qmInt.c +int32_t qmOpen(SMgmtWrapper *pWrapper); +int32_t qmDrop(SMgmtWrapper *pWrapper); + +// qmMsg.c +void qmInitMsgHandles(SMgmtWrapper *pWrapper); +int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); + +// qmWorker.c +int32_t qmStartWorker(SQnodeMgmt *pMgmt); +void qmStopWorker(SQnodeMgmt *pMgmt); +int32_t qmProcessQueryMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t qmProcessFetchMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_QNODE_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/qnode/src/qmInt.c b/source/dnode/mgmt/qnode/src/qmInt.c new file mode 100644 index 0000000000..78ef238ab8 --- /dev/null +++ b/source/dnode/mgmt/qnode/src/qmInt.c @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "qmInt.h" + +static int32_t qmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); } + +static void qmInitOption(SQnodeMgmt *pMgmt, SQnodeOpt *pOption) { + SDnode *pDnode = pMgmt->pDnode; + pOption->pWrapper = pMgmt->pWrapper; + pOption->sendReqFp = dndSendReqToDnode; + pOption->sendMnodeReqFp = dndSendReqToMnode; + pOption->sendRspFp = dndSendRsp; + pOption->dnodeId = pDnode->dnodeId; + pOption->clusterId = pDnode->clusterId; +} + +static int32_t qmOpenImp(SQnodeMgmt *pMgmt) { + SQnodeOpt option = {0}; + qmInitOption(pMgmt, &option); + + pMgmt->pQnode = qndOpen(&option); + if (pMgmt->pQnode == NULL) { + dError("failed to open qnode since %s", terrstr()); + return -1; + } + + if (qmStartWorker(pMgmt) != 0) { + dError("failed to start qnode worker since %s", terrstr()); + return -1; + } + + bool deployed = true; + if (dndWriteFile(pMgmt->pWrapper, deployed) != 0) { + dError("failed to write qnode file since %s", terrstr()); + return -1; + } + + return 0; +} + +static void qmCloseImp(SQnodeMgmt *pMgmt) { + if (pMgmt->pQnode != NULL) { + qmStopWorker(pMgmt); + qndClose(pMgmt->pQnode); + pMgmt->pQnode = NULL; + } +} + +int32_t qmDrop(SMgmtWrapper *pWrapper) { + SQnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return 0; + + dInfo("qnode-mgmt start to drop"); + bool deployed = false; + if (dndWriteFile(pWrapper, deployed) != 0) { + dError("failed to drop qnode since %s", terrstr()); + return -1; + } + + qmCloseImp(pMgmt); + taosRemoveDir(pMgmt->path); + pWrapper->pMgmt = NULL; + free(pMgmt); + dInfo("qnode-mgmt is dropped"); + return 0; +} + +static void qmClose(SMgmtWrapper *pWrapper) { + SQnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return; + + dInfo("qnode-mgmt start to cleanup"); + qmCloseImp(pMgmt); + pWrapper->pMgmt = NULL; + free(pMgmt); + dInfo("qnode-mgmt is cleaned up"); +} + +int32_t qmOpen(SMgmtWrapper *pWrapper) { + dInfo("qnode-mgmt start to init"); + SQnodeMgmt *pMgmt = calloc(1, sizeof(SQnodeMgmt)); + if (pMgmt == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pMgmt->path = pWrapper->path; + pMgmt->pDnode = pWrapper->pDnode; + pMgmt->pWrapper = pWrapper; + pWrapper->pMgmt = pMgmt; + + int32_t code = qmOpenImp(pMgmt); + if (code != 0) { + dError("failed to init qnode-mgmt since %s", terrstr()); + qmClose(pWrapper); + } else { + dInfo("qnode-mgmt is initialized"); + } + + return code; +} + +void qmGetMgmtFp(SMgmtWrapper *pWrapper) { + SMgmtFp mgmtFp = {0}; + mgmtFp.openFp = qmOpen; + mgmtFp.closeFp = qmClose; + mgmtFp.createMsgFp = qmProcessCreateReq; + mgmtFp.dropMsgFp = qmProcessDropReq; + mgmtFp.requiredFp = qmRequire; + + qmInitMsgHandles(pWrapper); + pWrapper->name = "qnode"; + pWrapper->fp = mgmtFp; +} diff --git a/source/dnode/mgmt/qnode/src/qmMsg.c b/source/dnode/mgmt/qnode/src/qmMsg.c new file mode 100644 index 0000000000..fa7d42b3e3 --- /dev/null +++ b/source/dnode/mgmt/qnode/src/qmMsg.c @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "qmInt.h" + +int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDCreateQnodeReq createReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (createReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to create qnode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pDnode->dnodeId); + return -1; + } else { + return qmOpen(pWrapper); + } +} + +int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDDropQnodeReq dropReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (dropReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to drop qnode since %s", terrstr()); + return -1; + } else { + return qmDrop(pWrapper); + } +} + +void qmInitMsgHandles(SMgmtWrapper *pWrapper) {} diff --git a/source/dnode/mgmt/qnode/src/qmWorker.c b/source/dnode/mgmt/qnode/src/qmWorker.c new file mode 100644 index 0000000000..6285a4bb74 --- /dev/null +++ b/source/dnode/mgmt/qnode/src/qmWorker.c @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "qmInt.h" + +static void qmProcessQueue(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { + dTrace("msg:%p, will be processed in qnode queue", pMsg); + SRpcMsg *pRsp = NULL; + SRpcMsg *pRpc = &pMsg->rpcMsg; + int32_t code = qndProcessMsg(pMgmt->pQnode, pRpc, &pRsp); + + if (pRpc->msgType & 1u) { + if (pRsp != NULL) { + pRsp->ahandle = pRpc->ahandle; + dndSendRsp(pMgmt->pWrapper, pRsp); + free(pRsp); + } else { + if (code != 0) code = terrno; + SRpcMsg rpcRsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code}; + dndSendRsp(pMgmt->pWrapper, &rpcRsp); + } + } + + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pRpc->pCont); + taosFreeQitem(pMsg); +} + +int32_t qmProcessQueryMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SDnodeWorker *pWorker = &pMgmt->queryWorker; + + dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + return dndWriteMsgToWorker(pWorker, pMsg); +} + +int32_t qmProcessFetchMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SDnodeWorker *pWorker = &pMgmt->fetchWorker; + + dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + return dndWriteMsgToWorker(pWorker, pMsg); +} + +int32_t qmStartWorker(SQnodeMgmt *pMgmt) { + if (dndInitWorker(pMgmt, &pMgmt->queryWorker, DND_WORKER_SINGLE, "qnode-query", 0, 1, qmProcessQueue) != 0) { + dError("failed to start qnode query worker since %s", terrstr()); + return -1; + } + + if (dndInitWorker(pMgmt, &pMgmt->fetchWorker, DND_WORKER_SINGLE, "qnode-fetch", 0, 1, qmProcessQueue) != 0) { + dError("failed to start qnode fetch worker since %s", terrstr()); + return -1; + } + + return 0; +} + +void qmStopWorker(SQnodeMgmt *pMgmt) { + dndCleanupWorker(&pMgmt->queryWorker); + dndCleanupWorker(&pMgmt->fetchWorker); +} diff --git a/source/dnode/mgmt/impl/inc/dndTransport.h b/source/dnode/mgmt/snode/inc/sm.h similarity index 68% rename from source/dnode/mgmt/impl/inc/dndTransport.h rename to source/dnode/mgmt/snode/inc/sm.h index e0ea21cba8..82a52e5d1f 100644 --- a/source/dnode/mgmt/impl/inc/dndTransport.h +++ b/source/dnode/mgmt/snode/inc/sm.h @@ -13,21 +13,19 @@ * along with this program. If not, see . */ -#ifndef _TD_DND_TRANSPORT_H_ -#define _TD_DND_TRANSPORT_H_ +#ifndef _TD_DND_SNODE_H_ +#define _TD_DND_SNODE_H_ + +#include "dnd.h" #ifdef __cplusplus extern "C" { #endif -#include "dndEnv.h" -int32_t dndInitTrans(SDnode *pDnode); -void dndCleanupTrans(SDnode *pDnode); -int32_t dndSendReqToMnode(SDnode *pDnode, SRpcMsg *pRpcMsg); -int32_t dndSendReqToDnode(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pRpcMsg); +void smGetMgmtFp(SMgmtWrapper *pWrapper); #ifdef __cplusplus } #endif -#endif /*_TD_DND_TRANSPORT_H_*/ +#endif /*_TD_DND_SNODE_H_*/ diff --git a/source/dnode/mgmt/snode/inc/smInt.h b/source/dnode/mgmt/snode/inc/smInt.h new file mode 100644 index 0000000000..3def27b832 --- /dev/null +++ b/source/dnode/mgmt/snode/inc/smInt.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_SNODE_INT_H_ +#define _TD_DND_SNODE_INT_H_ + +#include "sm.h" +#include "snode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SSnodeMgmt { + SSnode *pSnode; + SDnode *pDnode; + SMgmtWrapper *pWrapper; + const char *path; + SRWLatch latch; + int8_t uniqueWorkerInUse; + SArray *uniqueWorkers; // SArray + SDnodeWorker sharedWorker; +} SSnodeMgmt; + +// smInt.c +int32_t smOpen(SMgmtWrapper *pWrapper); +int32_t smDrop(SMgmtWrapper *pWrapper); + +// smMsg.c +void smInitMsgHandles(SMgmtWrapper *pWrapper); +int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); + +// smWorker.c +int32_t smStartWorker(SSnodeMgmt *pMgmt); +void smStopWorker(SSnodeMgmt *pMgmt); +int32_t smProcessMgmtMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t smProcessUniqueMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t smProcessSharedMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t smProcessExecMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_SNODE_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/snode/src/smInt.c b/source/dnode/mgmt/snode/src/smInt.c new file mode 100644 index 0000000000..0742a43157 --- /dev/null +++ b/source/dnode/mgmt/snode/src/smInt.c @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "smInt.h" + +static int32_t smRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); } + +static void smInitOption(SSnodeMgmt *pMgmt, SSnodeOpt *pOption) { + SDnode *pDnode = pMgmt->pDnode; + pOption->pWrapper = pMgmt->pWrapper; + pOption->sendReqFp = dndSendReqToDnode; + pOption->sendMnodeReqFp = dndSendReqToMnode; + pOption->sendRspFp = dndSendRsp; + pOption->dnodeId = pDnode->dnodeId; + pOption->clusterId = pDnode->clusterId; +} + +static int32_t smOpenImp(SSnodeMgmt *pMgmt) { + SSnodeOpt option = {0}; + smInitOption(pMgmt, &option); + + pMgmt->pSnode = sndOpen(pMgmt->path, &option); + if (pMgmt->pSnode == NULL) { + dError("failed to open snode since %s", terrstr()); + return -1; + } + + if (smStartWorker(pMgmt) != 0) { + dError("failed to start snode worker since %s", terrstr()); + return -1; + } + + bool deployed = true; + if (dndWriteFile(pMgmt->pWrapper, deployed) != 0) { + dError("failed to write snode file since %s", terrstr()); + return -1; + } + + return 0; +} + +static void smCloseImp(SSnodeMgmt *pMgmt) { + if (pMgmt->pSnode != NULL) { + smStopWorker(pMgmt); + sndClose(pMgmt->pSnode); + pMgmt->pSnode = NULL; + } +} + +int32_t smDrop(SMgmtWrapper *pWrapper) { + SSnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return 0; + + dInfo("snode-mgmt start to drop"); + bool deployed = false; + if (dndWriteFile(pWrapper, deployed) != 0) { + dError("failed to drop snode since %s", terrstr()); + return -1; + } + + smCloseImp(pMgmt); + taosRemoveDir(pMgmt->path); + pWrapper->pMgmt = NULL; + free(pMgmt); + dInfo("snode-mgmt is dropped"); + return 0; +} + +static void smClose(SMgmtWrapper *pWrapper) { + SSnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return; + + dInfo("snode-mgmt start to cleanup"); + smCloseImp(pMgmt); + pWrapper->pMgmt = NULL; + free(pMgmt); + dInfo("snode-mgmt is cleaned up"); +} + +int32_t smOpen(SMgmtWrapper *pWrapper) { + dInfo("snode-mgmt start to init"); + SSnodeMgmt *pMgmt = calloc(1, sizeof(SSnodeMgmt)); + if (pMgmt == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pMgmt->path = pWrapper->path; + pMgmt->pDnode = pWrapper->pDnode; + pMgmt->pWrapper = pWrapper; + pWrapper->pMgmt = pMgmt; + + int32_t code = smOpenImp(pMgmt); + if (code != 0) { + dError("failed to init snode-mgmt since %s", terrstr()); + smClose(pWrapper); + } else { + dInfo("snode-mgmt is initialized"); + } + + return code; +} + +void smGetMgmtFp(SMgmtWrapper *pWrapper) { + SMgmtFp mgmtFp = {0}; + mgmtFp.openFp = smOpen; + mgmtFp.closeFp = smClose; + mgmtFp.createMsgFp = smProcessCreateReq; + mgmtFp.dropMsgFp = smProcessDropReq; + mgmtFp.requiredFp = smRequire; + + smInitMsgHandles(pWrapper); + pWrapper->name = "snode"; + pWrapper->fp = mgmtFp; +} diff --git a/source/dnode/mgmt/snode/src/smMsg.c b/source/dnode/mgmt/snode/src/smMsg.c new file mode 100644 index 0000000000..1bff5597bf --- /dev/null +++ b/source/dnode/mgmt/snode/src/smMsg.c @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "smInt.h" + +int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDCreateSnodeReq createReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (createReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to create snode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pDnode->dnodeId); + return -1; + } else { + return smOpen(pWrapper); + } +} + +int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDDropSnodeReq dropReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (dropReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to drop snode since %s", terrstr()); + return -1; + } else { + return smDrop(pWrapper); + } +} + +void smInitMsgHandles(SMgmtWrapper *pWrapper) { + // Requests handled by SNODE + dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, (NodeMsgFp)smProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, (NodeMsgFp)smProcessExecMsg); +} diff --git a/source/dnode/mgmt/snode/src/smWorker.c b/source/dnode/mgmt/snode/src/smWorker.c new file mode 100644 index 0000000000..57d0c09849 --- /dev/null +++ b/source/dnode/mgmt/snode/src/smWorker.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "smInt.h" + +static void smProcessUniqueQueue(SSnodeMgmt *pMgmt, STaosQall *qall, int32_t numOfMsgs) { + for (int32_t i = 0; i < numOfMsgs; i++) { + SNodeMsg *pMsg = NULL; + taosGetQitem(qall, (void **)&pMsg); + + dTrace("msg:%p, will be processed in snode unique queue", pMsg); + sndProcessUMsg(pMgmt->pSnode, &pMsg->rpcMsg); + + dTrace("msg:%p, is freed", pMsg); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); + } +} + +static void smProcessSharedQueue(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { + dTrace("msg:%p, will be processed in snode shared queue", pMsg); + sndProcessSMsg(pMgmt->pSnode, &pMsg->rpcMsg); + + dTrace("msg:%p, is freed", pMsg); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); +} + +int32_t smStartWorker(SSnodeMgmt *pMgmt) { + pMgmt->uniqueWorkers = taosArrayInit(0, sizeof(void *)); + if (pMgmt->uniqueWorkers == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < SND_UNIQUE_THREAD_NUM; i++) { + SDnodeWorker *pUniqueWorker = malloc(sizeof(SDnodeWorker)); + if (pUniqueWorker == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + if (dndInitWorker(pMgmt, pUniqueWorker, DND_WORKER_MULTI, "snode-unique", 1, 1, smProcessSharedQueue) != 0) { + dError("failed to start snode unique worker since %s", terrstr()); + return -1; + } + if (taosArrayPush(pMgmt->uniqueWorkers, &pUniqueWorker) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + + if (dndInitWorker(pMgmt, &pMgmt->sharedWorker, DND_WORKER_SINGLE, "snode-shared", SND_SHARED_THREAD_NUM, + SND_SHARED_THREAD_NUM, smProcessSharedQueue)) { + dError("failed to start snode shared worker since %s", terrstr()); + return -1; + } + + return 0; +} + +void smStopWorker(SSnodeMgmt *pMgmt) { + for (int32_t i = 0; i < taosArrayGetSize(pMgmt->uniqueWorkers); i++) { + SDnodeWorker *worker = taosArrayGetP(pMgmt->uniqueWorkers, i); + dndCleanupWorker(worker); + } + taosArrayDestroy(pMgmt->uniqueWorkers); + dndCleanupWorker(&pMgmt->sharedWorker); +} + +static FORCE_INLINE int32_t smGetSWIdFromMsg(SRpcMsg *pMsg) { + SMsgHead *pHead = pMsg->pCont; + pHead->streamTaskId = htonl(pHead->streamTaskId); + return pHead->streamTaskId % SND_UNIQUE_THREAD_NUM; +} + +static FORCE_INLINE int32_t smGetSWTypeFromMsg(SRpcMsg *pMsg) { + SStreamExecMsgHead *pHead = pMsg->pCont; + pHead->workerType = htonl(pHead->workerType); + return pHead->workerType; +} + +int32_t smProcessMgmtMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SDnodeWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, 0); + if (pWorker == NULL) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + return dndWriteMsgToWorker(pWorker, pMsg); +} + +int32_t smProcessUniqueMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { + int32_t index = smGetSWIdFromMsg(&pMsg->rpcMsg); + SDnodeWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, index); + if (pWorker == NULL) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + return dndWriteMsgToWorker(pWorker, pMsg); +} + +int32_t smProcessSharedMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SDnodeWorker *pWorker = &pMgmt->sharedWorker; + + dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + return dndWriteMsgToWorker(pWorker, pMsg); +} + +int32_t smProcessExecMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { + int32_t workerType = smGetSWTypeFromMsg(&pMsg->rpcMsg); + if (workerType == SND_WORKER_TYPE__SHARED) { + return smProcessSharedMsg(pMgmt, pMsg); + } else { + return smProcessUniqueMsg(pMgmt, pMsg); + } +} diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/test/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/CMakeLists.txt rename to source/dnode/mgmt/test/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/bnode/CMakeLists.txt b/source/dnode/mgmt/test/bnode/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/bnode/CMakeLists.txt rename to source/dnode/mgmt/test/bnode/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/bnode/dbnode.cpp b/source/dnode/mgmt/test/bnode/dbnode.cpp similarity index 88% rename from source/dnode/mgmt/impl/test/bnode/dbnode.cpp rename to source/dnode/mgmt/test/bnode/dbnode.cpp index 75f111587f..ee81780921 100644 --- a/source/dnode/mgmt/impl/test/bnode/dbnode.cpp +++ b/source/dnode/mgmt/test/bnode/dbnode.cpp @@ -13,7 +13,10 @@ class DndTestBnode : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/dnode_test_snode", 9112); } + static void SetUpTestSuite() { + test.Init("/tmp/dnode_test_bnode", 9112); + taosMsleep(1100); + } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; @@ -36,7 +39,7 @@ TEST_F(DndTestBnode, 01_Create_Bnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -62,7 +65,7 @@ TEST_F(DndTestBnode, 01_Create_Bnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } test.Restart(); @@ -76,11 +79,11 @@ TEST_F(DndTestBnode, 01_Create_Bnode) { tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } } -TEST_F(DndTestBnode, 01_Drop_Bnode) { +TEST_F(DndTestBnode, 02_Drop_Bnode) { { SDDropBnodeReq dropReq = {0}; dropReq.dnodeId = 2; @@ -91,7 +94,7 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -117,7 +120,7 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } test.Restart(); @@ -132,7 +135,7 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } { @@ -147,4 +150,4 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) { ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); } -} \ No newline at end of file +} diff --git a/source/dnode/mgmt/impl/test/mnode/CMakeLists.txt b/source/dnode/mgmt/test/mnode/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/mnode/CMakeLists.txt rename to source/dnode/mgmt/test/mnode/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/mnode/dmnode.cpp b/source/dnode/mgmt/test/mnode/dmnode.cpp similarity index 92% rename from source/dnode/mgmt/impl/test/mnode/dmnode.cpp rename to source/dnode/mgmt/test/mnode/dmnode.cpp index 8655bcb774..4072eb90a8 100644 --- a/source/dnode/mgmt/impl/test/mnode/dmnode.cpp +++ b/source/dnode/mgmt/test/mnode/dmnode.cpp @@ -40,7 +40,7 @@ TEST_F(DndTestMnode, 01_Create_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } { @@ -57,7 +57,7 @@ TEST_F(DndTestMnode, 01_Create_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } { @@ -77,7 +77,7 @@ TEST_F(DndTestMnode, 01_Create_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } } @@ -96,7 +96,7 @@ TEST_F(DndTestMnode, 02_Alter_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -113,7 +113,7 @@ TEST_F(DndTestMnode, 02_Alter_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -145,7 +145,7 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -171,7 +171,7 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } { @@ -188,7 +188,7 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } { diff --git a/source/dnode/mgmt/impl/test/qnode/CMakeLists.txt b/source/dnode/mgmt/test/qnode/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/qnode/CMakeLists.txt rename to source/dnode/mgmt/test/qnode/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/qnode/dqnode.cpp b/source/dnode/mgmt/test/qnode/dqnode.cpp similarity index 91% rename from source/dnode/mgmt/impl/test/qnode/dqnode.cpp rename to source/dnode/mgmt/test/qnode/dqnode.cpp index 46c31539a2..343814b159 100644 --- a/source/dnode/mgmt/impl/test/qnode/dqnode.cpp +++ b/source/dnode/mgmt/test/qnode/dqnode.cpp @@ -36,7 +36,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -62,7 +62,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } test.Restart(); @@ -77,7 +77,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } } @@ -92,7 +92,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -118,7 +118,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } test.Restart(); @@ -133,7 +133,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } { diff --git a/source/dnode/mgmt/impl/test/snode/CMakeLists.txt b/source/dnode/mgmt/test/snode/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/snode/CMakeLists.txt rename to source/dnode/mgmt/test/snode/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/snode/dsnode.cpp b/source/dnode/mgmt/test/snode/dsnode.cpp similarity index 91% rename from source/dnode/mgmt/impl/test/snode/dsnode.cpp rename to source/dnode/mgmt/test/snode/dsnode.cpp index ea98dfdd95..8dfa437fd8 100644 --- a/source/dnode/mgmt/impl/test/snode/dsnode.cpp +++ b/source/dnode/mgmt/test/snode/dsnode.cpp @@ -36,7 +36,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -62,7 +62,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } test.Restart(); @@ -77,7 +77,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } } @@ -92,7 +92,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -118,7 +118,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } test.Restart(); @@ -133,7 +133,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } { diff --git a/source/dnode/mgmt/impl/test/sut/CMakeLists.txt b/source/dnode/mgmt/test/sut/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/sut/CMakeLists.txt rename to source/dnode/mgmt/test/sut/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/sut/inc/client.h b/source/dnode/mgmt/test/sut/inc/client.h similarity index 100% rename from source/dnode/mgmt/impl/test/sut/inc/client.h rename to source/dnode/mgmt/test/sut/inc/client.h diff --git a/source/dnode/mgmt/impl/test/sut/inc/server.h b/source/dnode/mgmt/test/sut/inc/server.h similarity index 89% rename from source/dnode/mgmt/impl/test/sut/inc/server.h rename to source/dnode/mgmt/test/sut/inc/server.h index 99554a7aa7..ad2d4a76e9 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/server.h +++ b/source/dnode/mgmt/test/sut/inc/server.h @@ -24,11 +24,11 @@ class TestServer { bool DoStart(); private: - SDnodeObjCfg BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp); + SDnodeOpt BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp); private: SDnode* pDnode; - pthread_t* threadId; + TdThread threadId; char path[PATH_MAX]; char fqdn[TSDB_FQDN_LEN]; char firstEp[TSDB_EP_LEN]; diff --git a/source/dnode/mgmt/impl/test/sut/inc/sut.h b/source/dnode/mgmt/test/sut/inc/sut.h similarity index 100% rename from source/dnode/mgmt/impl/test/sut/inc/sut.h rename to source/dnode/mgmt/test/sut/inc/sut.h diff --git a/source/dnode/mgmt/impl/test/sut/src/client.cpp b/source/dnode/mgmt/test/sut/src/client.cpp similarity index 100% rename from source/dnode/mgmt/impl/test/sut/src/client.cpp rename to source/dnode/mgmt/test/sut/src/client.cpp diff --git a/source/dnode/mgmt/impl/test/sut/src/server.cpp b/source/dnode/mgmt/test/sut/src/server.cpp similarity index 57% rename from source/dnode/mgmt/impl/test/sut/src/server.cpp rename to source/dnode/mgmt/test/sut/src/server.cpp index 985625b41c..c5379c6d17 100644 --- a/source/dnode/mgmt/impl/test/sut/src/server.cpp +++ b/source/dnode/mgmt/test/sut/src/server.cpp @@ -16,36 +16,37 @@ #include "sut.h" void* serverLoop(void* param) { - while (1) { - taosMsleep(100); - pthread_testcancel(); - } + SDnode* pDnode = (SDnode*)param; + dndRun(pDnode); + return NULL; } -SDnodeObjCfg TestServer::BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - SDnodeObjCfg cfg = {0}; - cfg.numOfSupportVnodes = 16; - cfg.serverPort = port; - strcpy(cfg.dataDir, path); - snprintf(cfg.localEp, TSDB_EP_LEN, "%s:%u", fqdn, port); - snprintf(cfg.localFqdn, TSDB_FQDN_LEN, "%s", fqdn); - snprintf(cfg.firstEp, TSDB_EP_LEN, "%s", firstEp); - return cfg; +SDnodeOpt TestServer::BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + SDnodeOpt option = {0}; + option.numOfSupportVnodes = 16; + option.serverPort = port; + strcpy(option.dataDir, path); + snprintf(option.localEp, TSDB_EP_LEN, "%s:%u", fqdn, port); + snprintf(option.localFqdn, TSDB_FQDN_LEN, "%s", fqdn); + snprintf(option.firstEp, TSDB_EP_LEN, "%s", firstEp); + return option; } bool TestServer::DoStart() { - SDnodeObjCfg cfg = BuildOption(path, fqdn, port, firstEp); + SDnodeOpt option = BuildOption(path, fqdn, port, firstEp); taosMkDir(path); - pDnode = dndCreate(&cfg); - if (pDnode != NULL) { + pDnode = dndCreate(&option); + if (pDnode == NULL) { return false; } - threadId = taosCreateThread(serverLoop, NULL); - if (threadId != NULL) { - return false; - } + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + taosThreadCreate(&threadId, &thAttr, serverLoop, pDnode); + taosThreadAttrDestroy(&thAttr); + taosMsleep(2100); return true; } @@ -67,10 +68,8 @@ bool TestServer::Start(const char* path, const char* fqdn, uint16_t port, const } void TestServer::Stop() { - if (threadId != NULL) { - taosDestoryThread(threadId); - threadId = NULL; - } + dndHandleEvent(pDnode, DND_EVENT_STOP); + taosThreadJoin(threadId, NULL); if (pDnode != NULL) { dndClose(pDnode); diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/test/sut/src/sut.cpp similarity index 98% rename from source/dnode/mgmt/impl/test/sut/src/sut.cpp rename to source/dnode/mgmt/test/sut/src/sut.cpp index 2e38ea7513..14197153b4 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/test/sut/src/sut.cpp @@ -21,8 +21,8 @@ void Testbase::InitLog(const char* path) { mDebugFlag = 143; cDebugFlag = 0; jniDebugFlag = 0; - tmrDebugFlag = 143; - uDebugFlag = 143; + tmrDebugFlag = 135; + uDebugFlag = 135; rpcDebugFlag = 143; qDebugFlag = 0; wDebugFlag = 0; @@ -49,7 +49,6 @@ void Testbase::Init(const char* path, int16_t port) { InitLog("/tmp/td"); server.Start(path, fqdn, port, firstEp); client.Init("root", "taosdata", fqdn, port); - taosMsleep(1100); tFreeSTableMetaRsp(&metaRsp); showId = 0; diff --git a/source/dnode/mgmt/impl/test/vnode/CMakeLists.txt b/source/dnode/mgmt/test/vnode/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/vnode/CMakeLists.txt rename to source/dnode/mgmt/test/vnode/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/vnode/vnode.cpp b/source/dnode/mgmt/test/vnode/vnode.cpp similarity index 100% rename from source/dnode/mgmt/impl/test/vnode/vnode.cpp rename to source/dnode/mgmt/test/vnode/vnode.cpp diff --git a/source/dnode/mgmt/impl/inc/dndQnode.h b/source/dnode/mgmt/vnode/inc/vm.h similarity index 53% rename from source/dnode/mgmt/impl/inc/dndQnode.h rename to source/dnode/mgmt/vnode/inc/vm.h index 2a25dca1c6..60d9cfc3a1 100644 --- a/source/dnode/mgmt/impl/inc/dndQnode.h +++ b/source/dnode/mgmt/vnode/inc/vm.h @@ -13,24 +13,34 @@ * along with this program. If not, see . */ -#ifndef _TD_DND_QNODE_H_ -#define _TD_DND_QNODE_H_ +#ifndef _TD_DND_VNODES_H_ +#define _TD_DND_VNODES_H_ + +#include "dnd.h" #ifdef __cplusplus extern "C" { #endif -#include "dndEnv.h" -int32_t dndInitQnode(SDnode *pDnode); -void dndCleanupQnode(SDnode *pDnode); +typedef struct { + int32_t openVnodes; + int32_t totalVnodes; + int32_t masterNum; + int64_t numOfSelectReqs; + int64_t numOfInsertReqs; + int64_t numOfInsertSuccessReqs; + int64_t numOfBatchInsertReqs; + int64_t numOfBatchInsertSuccessReqs; +} SVnodesStat; -void dndProcessQnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessQnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -int32_t dndProcessCreateQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); -int32_t dndProcessDropQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); +void vmGetMgmtFp(SMgmtWrapper *pWrapper); + +void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads); +int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo); +void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo); #ifdef __cplusplus } #endif -#endif /*_TD_DND_QNODE_H_*/ \ No newline at end of file +#endif /*_TD_DND_VNODES_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/vnode/inc/vmInt.h b/source/dnode/mgmt/vnode/inc/vmInt.h new file mode 100644 index 0000000000..2020a3d219 --- /dev/null +++ b/source/dnode/mgmt/vnode/inc/vmInt.h @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_VNODES_INT_H_ +#define _TD_DND_VNODES_INT_H_ + +#include "sync.h" +#include "vm.h" +#include "vnode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { VND_WRITE_QUEUE, VND_QUERY_QUEUE, VND_FETCH_QUEUE, VND_APPLY_QUEUE, VND_SYNC_QUEUE } EVndQueueType; + +typedef struct SVnodesMgmt { + SHashObj *hash; + SRWLatch latch; + SVnodesStat state; + STfs *pTfs; + SQWorkerPool queryPool; + SFWorkerPool fetchPool; + SWWorkerPool syncPool; + SWWorkerPool writePool; + const char *path; + SDnode *pDnode; + SMgmtWrapper *pWrapper; + SDnodeWorker mgmtWorker; +} SVnodesMgmt; + +typedef struct { + int32_t vgId; + int32_t vgVersion; + int8_t dropped; + uint64_t dbUid; + char db[TSDB_DB_FNAME_LEN]; + char path[PATH_MAX + 20]; +} SWrapperCfg; + +typedef struct { + int32_t vgId; + int32_t refCount; + int32_t vgVersion; + int8_t dropped; + int8_t accessState; + uint64_t dbUid; + char *db; + char *path; + SVnode *pImpl; + STaosQueue *pWriteQ; + STaosQueue *pSyncQ; + STaosQueue *pApplyQ; + STaosQueue *pQueryQ; + STaosQueue *pFetchQ; + SMgmtWrapper *pWrapper; +} SVnodeObj; + +typedef struct { + int32_t vnodeNum; + int32_t opened; + int32_t failed; + int32_t threadIndex; + TdThread thread; + SVnodesMgmt *pMgmt; + SWrapperCfg *pCfgs; +} SVnodeThread; + +// vmInt.c +SVnodeObj *vmAcquireVnode(SVnodesMgmt *pMgmt, int32_t vgId); +void vmReleaseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); +int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl); +void vmCloseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); + +// vmMsg.c +void vmInitMsgHandles(SMgmtWrapper *pWrapper); +int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); +int32_t vmProcessAlterVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); +int32_t vmProcessDropVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); +int32_t vmProcessSyncVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); +int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); + +// vmFile.c +int32_t vmGetVnodesFromFile(SVnodesMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes); +int32_t vmWriteVnodesToFile(SVnodesMgmt *pMgmt); +SVnodeObj **vmGetVnodesFromHash(SVnodesMgmt *pMgmt, int32_t *numOfVnodes); + +// vmWorker.c +int32_t vmStartWorker(SVnodesMgmt *pMgmt); +void vmStopWorker(SVnodesMgmt *pMgmt); +int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); +void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); + +int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); +int32_t vmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); +int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); + +int32_t vmProcessWriteMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); +int32_t vmProcessSyncMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); +int32_t vmProcessQueryMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); +int32_t vmProcessFetchMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); +int32_t vmProcessMgmtMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_VNODES_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/vnode/src/vmFile.c b/source/dnode/mgmt/vnode/src/vmFile.c new file mode 100644 index 0000000000..0fe868dfa4 --- /dev/null +++ b/source/dnode/mgmt/vnode/src/vmFile.c @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "vmInt.h" + +SVnodeObj **vmGetVnodesFromHash(SVnodesMgmt *pMgmt, int32_t *numOfVnodes) { + taosRLockLatch(&pMgmt->latch); + + int32_t num = 0; + int32_t size = taosHashGetSize(pMgmt->hash); + SVnodeObj **pVnodes = calloc(size, sizeof(SVnodeObj *)); + + void *pIter = taosHashIterate(pMgmt->hash, NULL); + while (pIter) { + SVnodeObj **ppVnode = pIter; + SVnodeObj *pVnode = *ppVnode; + if (pVnode && num < size) { + int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); + dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); + pVnodes[num] = (*ppVnode); + num++; + pIter = taosHashIterate(pMgmt->hash, pIter); + } else { + taosHashCancelIterate(pMgmt->hash, pIter); + } + } + + taosRUnLockLatch(&pMgmt->latch); + *numOfVnodes = num; + + return pVnodes; +} + +int32_t vmGetVnodesFromFile(SVnodesMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) { + int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; + int32_t len = 0; + int32_t maxLen = 30000; + char *content = calloc(1, maxLen + 1); + cJSON *root = NULL; + FILE *fp = NULL; + char file[PATH_MAX]; + SWrapperCfg *pCfgs = NULL; + TdFilePtr pFile = NULL; + + snprintf(file, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP); + + pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { + dDebug("file %s not exist", file); + code = 0; + goto PRASE_VNODE_OVER; + } + + len = (int32_t)taosReadFile(pFile, content, maxLen); + if (len <= 0) { + dError("failed to read %s since content is null", file); + goto PRASE_VNODE_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s since invalid json format", file); + goto PRASE_VNODE_OVER; + } + + cJSON *vnodes = cJSON_GetObjectItem(root, "vnodes"); + if (!vnodes || vnodes->type != cJSON_Array) { + dError("failed to read %s since vnodes not found", file); + goto PRASE_VNODE_OVER; + } + + int32_t vnodesNum = cJSON_GetArraySize(vnodes); + if (vnodesNum > 0) { + pCfgs = calloc(vnodesNum, sizeof(SWrapperCfg)); + if (pCfgs == NULL) { + dError("failed to read %s since out of memory", file); + goto PRASE_VNODE_OVER; + } + + for (int32_t i = 0; i < vnodesNum; ++i) { + cJSON *vnode = cJSON_GetArrayItem(vnodes, i); + SWrapperCfg *pCfg = &pCfgs[i]; + + cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId"); + if (!vgId || vgId->type != cJSON_Number) { + dError("failed to read %s since vgId not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->vgId = vgId->valueint; + snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCfg->vgId); + + cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped"); + if (!dropped || dropped->type != cJSON_Number) { + dError("failed to read %s since dropped not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->dropped = dropped->valueint; + + cJSON *vgVersion = cJSON_GetObjectItem(vnode, "vgVersion"); + if (!vgVersion || vgVersion->type != cJSON_Number) { + dError("failed to read %s since vgVersion not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->vgVersion = vgVersion->valueint; + + cJSON *dbUid = cJSON_GetObjectItem(vnode, "dbUid"); + if (!dbUid || dbUid->type != cJSON_String) { + dError("failed to read %s since dbUid not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->dbUid = atoll(dbUid->valuestring); + + cJSON *db = cJSON_GetObjectItem(vnode, "db"); + if (!db || db->type != cJSON_String) { + dError("failed to read %s since db not found", file); + goto PRASE_VNODE_OVER; + } + tstrncpy(pCfg->db, db->valuestring, TSDB_DB_FNAME_LEN); + } + + *ppCfgs = pCfgs; + } + + *numOfVnodes = vnodesNum; + code = 0; + dInfo("succcessed to read file %s", file); + +PRASE_VNODE_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (pFile != NULL) taosCloseFile(&pFile); + + terrno = code; + return code; +} + +int32_t vmWriteVnodesToFile(SVnodesMgmt *pMgmt) { + char file[PATH_MAX]; + char realfile[PATH_MAX]; + snprintf(file, sizeof(file), "%s%svnodes.json.bak", pMgmt->path, TD_DIRSEP); + snprintf(realfile, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP); + + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to write %s since %s", file, terrstr()); + return -1; + } + + int32_t numOfVnodes = 0; + SVnodeObj **pVnodes = vmGetVnodesFromHash(pMgmt, &numOfVnodes); + + int32_t len = 0; + int32_t maxLen = 65536; + char *content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"vnodes\": [\n"); + for (int32_t i = 0; i < numOfVnodes; ++i) { + SVnodeObj *pVnode = pVnodes[i]; + len += snprintf(content + len, maxLen - len, " {\n"); + len += snprintf(content + len, maxLen - len, " \"vgId\": %d,\n", pVnode->vgId); + len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pVnode->dropped); + len += snprintf(content + len, maxLen - len, " \"vgVersion\": %d,\n", pVnode->vgVersion); + len += snprintf(content + len, maxLen - len, " \"dbUid\": \"%" PRIu64 "\",\n", pVnode->dbUid); + len += snprintf(content + len, maxLen - len, " \"db\": \"%s\"\n", pVnode->db); + if (i < numOfVnodes - 1) { + len += snprintf(content + len, maxLen - len, " },\n"); + } else { + len += snprintf(content + len, maxLen - len, " }\n"); + } + } + len += snprintf(content + len, maxLen - len, " ]\n"); + len += snprintf(content + len, maxLen - len, "}\n"); + + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); + free(content); + terrno = 0; + + for (int32_t i = 0; i < numOfVnodes; ++i) { + SVnodeObj *pVnode = pVnodes[i]; + vmReleaseVnode(pMgmt, pVnode); + } + + if (pVnodes != NULL) { + free(pVnodes); + } + + dDebug("successed to write %s", realfile); + return taosRenameFile(file, realfile); +} \ No newline at end of file diff --git a/source/dnode/mgmt/vnode/src/vmInt.c b/source/dnode/mgmt/vnode/src/vmInt.c new file mode 100644 index 0000000000..c5e79765c7 --- /dev/null +++ b/source/dnode/mgmt/vnode/src/vmInt.c @@ -0,0 +1,413 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "vmInt.h" + +SVnodeObj *vmAcquireVnode(SVnodesMgmt *pMgmt, int32_t vgId) { + SVnodeObj *pVnode = NULL; + int32_t refCount = 0; + + taosRLockLatch(&pMgmt->latch); + taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); + if (pVnode == NULL) { + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + } else { + refCount = atomic_add_fetch_32(&pVnode->refCount, 1); + } + taosRUnLockLatch(&pMgmt->latch); + + if (pVnode != NULL) { + dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); + } + + return pVnode; +} + +void vmReleaseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { + if (pVnode == NULL) return; + + taosRLockLatch(&pMgmt->latch); + int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); + taosRUnLockLatch(&pMgmt->latch); + dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount); +} + +int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { + SVnodeObj *pVnode = calloc(1, sizeof(SVnodeObj)); + if (pVnode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pVnode->vgId = pCfg->vgId; + pVnode->refCount = 0; + pVnode->dropped = 0; + pVnode->accessState = TSDB_VN_ALL_ACCCESS; + pVnode->pWrapper = pMgmt->pWrapper; + pVnode->pImpl = pImpl; + pVnode->vgVersion = pCfg->vgVersion; + pVnode->dbUid = pCfg->dbUid; + pVnode->db = tstrdup(pCfg->db); + pVnode->path = tstrdup(pCfg->path); + + if (pVnode->path == NULL || pVnode->db == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (vmAllocQueue(pMgmt, pVnode) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + taosWLockLatch(&pMgmt->latch); + int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); + taosWUnLockLatch(&pMgmt->latch); + + if (code != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + return code; +} + +void vmCloseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { + taosWLockLatch(&pMgmt->latch); + taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); + taosWUnLockLatch(&pMgmt->latch); + + vmReleaseVnode(pMgmt, pVnode); + while (pVnode->refCount > 0) taosMsleep(10); + while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10); + while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10); + while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10); + while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10); + while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10); + + vmFreeQueue(pMgmt, pVnode); + vnodeClose(pVnode->pImpl); + pVnode->pImpl = NULL; + + dDebug("vgId:%d, vnode is closed", pVnode->vgId); + + if (pVnode->dropped) { + dDebug("vgId:%d, vnode is destroyed for dropped:%d", pVnode->vgId, pVnode->dropped); + vnodeDestroy(pVnode->path); + } + + free(pVnode->path); + free(pVnode->db); + free(pVnode); +} + +static void *vmOpenVnodeFunc(void *param) { + SVnodeThread *pThread = param; + SVnodesMgmt *pMgmt = pThread->pMgmt; + SDnode *pDnode = pMgmt->pDnode; + + dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); + setThreadName("open-vnodes"); + + for (int32_t v = 0; v < pThread->vnodeNum; ++v) { + SWrapperCfg *pCfg = &pThread->pCfgs[v]; + + char stepDesc[TSDB_STEP_DESC_LEN] = {0}; + snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pCfg->vgId, + pMgmt->state.openVnodes, pMgmt->state.totalVnodes); + dndReportStartup(pDnode, "open-vnodes", stepDesc); + + SVnodeCfg cfg = {.pWrapper = pMgmt->pWrapper, .pTfs = pMgmt->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid}; + SVnode *pImpl = vnodeOpen(pCfg->path, &cfg); + if (pImpl == NULL) { + dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); + pThread->failed++; + } else { + vmOpenVnode(pMgmt, pCfg, pImpl); + dDebug("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex); + pThread->opened++; + } + + atomic_add_fetch_32(&pMgmt->state.openVnodes, 1); + } + + dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, + pThread->failed); + return NULL; +} + +static int32_t vmOpenVnodes(SVnodesMgmt *pMgmt) { + SDnode *pDnode = pMgmt->pDnode; + taosInitRWLatch(&pMgmt->latch); + + pMgmt->hash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + if (pMgmt->hash == NULL) { + dError("failed to init vnode hash"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + SWrapperCfg *pCfgs = NULL; + int32_t numOfVnodes = 0; + if (vmGetVnodesFromFile(pMgmt, &pCfgs, &numOfVnodes) != 0) { + dInfo("failed to get vnode list from disk since %s", terrstr()); + return -1; + } + + pMgmt->state.totalVnodes = numOfVnodes; + +#if 0 + int32_t threadNum = tsNumOfCores; +#else + int32_t threadNum = 1; +#endif + int32_t vnodesPerThread = numOfVnodes / threadNum + 1; + + SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread)); + for (int32_t t = 0; t < threadNum; ++t) { + threads[t].threadIndex = t; + threads[t].pMgmt = pMgmt; + threads[t].pCfgs = calloc(vnodesPerThread, sizeof(SWrapperCfg)); + } + + for (int32_t v = 0; v < numOfVnodes; ++v) { + int32_t t = v % threadNum; + SVnodeThread *pThread = &threads[t]; + pThread->pCfgs[pThread->vnodeNum++] = pCfgs[v]; + } + + dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes); + + for (int32_t t = 0; t < threadNum; ++t) { + SVnodeThread *pThread = &threads[t]; + if (pThread->vnodeNum == 0) continue; + + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + if (taosThreadCreate(&pThread->thread, &thAttr, vmOpenVnodeFunc, pThread) != 0) { + dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno)); + } + + taosThreadAttrDestroy(&thAttr); + } + + for (int32_t t = 0; t < threadNum; ++t) { + SVnodeThread *pThread = &threads[t]; + if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) { + taosThreadJoin(pThread->thread, NULL); + } + free(pThread->pCfgs); + } + free(threads); + free(pCfgs); + + if (pMgmt->state.openVnodes != pMgmt->state.totalVnodes) { + dError("there are total vnodes:%d, opened:%d", pMgmt->state.totalVnodes, pMgmt->state.openVnodes); + return -1; + } else { + dInfo("total vnodes:%d open successfully", pMgmt->state.totalVnodes); + return 0; + } +} + +static void vmCloseVnodes(SVnodesMgmt *pMgmt) { + dInfo("start to close all vnodes"); + + int32_t numOfVnodes = 0; + SVnodeObj **pVnodes = vmGetVnodesFromHash(pMgmt, &numOfVnodes); + + for (int32_t i = 0; i < numOfVnodes; ++i) { + vmCloseVnode(pMgmt, pVnodes[i]); + } + + if (pVnodes != NULL) { + free(pVnodes); + } + + if (pMgmt->hash != NULL) { + taosHashCleanup(pMgmt->hash); + pMgmt->hash = NULL; + } + + dInfo("total vnodes:%d are all closed", numOfVnodes); +} + +static void vmCleanup(SMgmtWrapper *pWrapper) { + SVnodesMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return; + + dInfo("vnodes-mgmt start to cleanup"); + vmCloseVnodes(pMgmt); + vmStopWorker(pMgmt); + vnodeCleanup(); + // walCleanUp(); + free(pMgmt); + pWrapper->pMgmt = NULL; + dInfo("vnodes-mgmt is cleaned up"); +} + +static int32_t vmInit(SMgmtWrapper *pWrapper) { + SDnode *pDnode = pWrapper->pDnode; + SVnodesMgmt *pMgmt = calloc(1, sizeof(SVnodesMgmt)); + int32_t code = -1; + SVnodeOpt vnodeOpt = {0}; + + dInfo("vnodes-mgmt start to init"); + if (pMgmt == NULL) goto _OVER; + + pMgmt->path = pWrapper->path; + pMgmt->pDnode = pWrapper->pDnode; + pMgmt->pWrapper = pWrapper; + taosInitRWLatch(&pMgmt->latch); + + SDiskCfg dCfg = {0}; + tstrncpy(dCfg.dir, pDnode->dataDir, TSDB_FILENAME_LEN); + dCfg.level = 0; + dCfg.primary = 1; + SDiskCfg *pDisks = pDnode->pDisks; + int32_t numOfDisks = pDnode->numOfDisks; + if (numOfDisks <= 0 || pDisks == NULL) { + pDisks = &dCfg; + numOfDisks = 1; + } + + pMgmt->pTfs = tfsOpen(pDisks, numOfDisks); + if (pMgmt->pTfs == NULL) { + dError("failed to init tfs since %s", terrstr()); + goto _OVER; + } + + if (walInit() != 0) { + dError("failed to init wal since %s", terrstr()); + goto _OVER; + } + + vnodeOpt.nthreads = tsNumOfCommitThreads; + vnodeOpt.putToQueryQFp = vmPutMsgToQueryQueue; + vnodeOpt.putToFetchQFp = vmPutMsgToQueryQueue; + vnodeOpt.sendReqFp = dndSendReqToDnode; + vnodeOpt.sendMnodeReqFp = dndSendReqToMnode; + vnodeOpt.sendRspFp = dndSendRsp; + if (vnodeInit(&vnodeOpt) != 0) { + dError("failed to init vnode since %s", terrstr()); + goto _OVER; + } + + if (vmStartWorker(pMgmt) != 0) { + dError("failed to init workers since %s", terrstr()) goto _OVER; + } + + if (vmOpenVnodes(pMgmt) != 0) { + dError("failed to open vnodes since %s", terrstr()); + return -1; + } + + code = 0; + +_OVER: + if (code == 0) { + pWrapper->pMgmt = pMgmt; + dInfo("vnodes-mgmt is initialized"); + } else { + dError("failed to init vnodes-mgmt since %s", terrstr()); + vmCleanup(pWrapper); + } + + return 0; +} + +static int32_t vmRequire(SMgmtWrapper *pWrapper, bool *required) { + SDnode *pDnode = pWrapper->pDnode; + *required = pDnode->numOfSupportVnodes > 0; + return 0; +} + +void vmGetMgmtFp(SMgmtWrapper *pWrapper) { + SMgmtFp mgmtFp = {0}; + mgmtFp.openFp = vmInit; + mgmtFp.closeFp = vmCleanup; + mgmtFp.requiredFp = vmRequire; + + vmInitMsgHandles(pWrapper); + pWrapper->name = "vnodes"; + pWrapper->fp = mgmtFp; +} + +int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo) { + SVnodesMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return -1; + + return tfsGetMonitorInfo(pMgmt->pTfs, pInfo); +} + +void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo) { + SVnodesMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return; + + SVnodesStat *pStat = &pMgmt->state; + pInfo->req_select = pStat->numOfSelectReqs; + pInfo->req_insert = pStat->numOfInsertReqs; + pInfo->req_insert_success = pStat->numOfInsertSuccessReqs; + pInfo->req_insert_batch = pStat->numOfBatchInsertReqs; + pInfo->req_insert_batch_success = pStat->numOfBatchInsertSuccessReqs; + pInfo->errors = tsNumOfErrorLogs; + pInfo->vnodes_num = pStat->totalVnodes; + pInfo->masters = pStat->masterNum; +} + +void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads) { + SVnodesMgmt *pMgmt = pWrapper->pMgmt; + SVnodesStat *pStat = &pMgmt->state; + int32_t totalVnodes = 0; + int32_t masterNum = 0; + int64_t numOfSelectReqs = 0; + int64_t numOfInsertReqs = 0; + int64_t numOfInsertSuccessReqs = 0; + int64_t numOfBatchInsertReqs = 0; + int64_t numOfBatchInsertSuccessReqs = 0; + + taosRLockLatch(&pMgmt->latch); + + void *pIter = taosHashIterate(pMgmt->hash, NULL); + while (pIter) { + SVnodeObj **ppVnode = pIter; + if (ppVnode == NULL || *ppVnode == NULL) continue; + + SVnodeObj *pVnode = *ppVnode; + SVnodeLoad vload = {0}; + vnodeGetLoad(pVnode->pImpl, &vload); + taosArrayPush(pLoads, &vload); + + numOfSelectReqs += vload.numOfSelectReqs; + numOfInsertReqs += vload.numOfInsertReqs; + numOfInsertSuccessReqs += vload.numOfInsertSuccessReqs; + numOfBatchInsertReqs += vload.numOfBatchInsertReqs; + numOfBatchInsertSuccessReqs += vload.numOfBatchInsertSuccessReqs; + totalVnodes++; + if (vload.role == TAOS_SYNC_STATE_LEADER) masterNum++; + + pIter = taosHashIterate(pMgmt->hash, pIter); + } + + taosRUnLockLatch(&pMgmt->latch); + + pStat->totalVnodes = totalVnodes; + pStat->masterNum = masterNum; + pStat->numOfSelectReqs = numOfSelectReqs; + pStat->numOfInsertReqs = numOfInsertReqs; + pStat->numOfInsertSuccessReqs = numOfInsertSuccessReqs; + pStat->numOfBatchInsertReqs = numOfBatchInsertReqs; + pStat->numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs; +} \ No newline at end of file diff --git a/source/dnode/mgmt/vnode/src/vmMsg.c b/source/dnode/mgmt/vnode/src/vmMsg.c new file mode 100644 index 0000000000..53423845d4 --- /dev/null +++ b/source/dnode/mgmt/vnode/src/vmMsg.c @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "vmInt.h" + +static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { + pCfg->vgId = pCreate->vgId; + pCfg->wsize = pCreate->cacheBlockSize; + pCfg->ssize = pCreate->cacheBlockSize; + pCfg->lsize = pCreate->cacheBlockSize; + pCfg->isHeapAllocator = true; + pCfg->ttl = 4; + pCfg->keep = pCreate->daysToKeep0; + pCfg->streamMode = pCreate->streamMode; + pCfg->isWeak = true; + pCfg->tsdbCfg.keep = pCreate->daysToKeep0; + pCfg->tsdbCfg.keep1 = pCreate->daysToKeep2; + pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0; + pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize; + pCfg->metaCfg.lruSize = pCreate->cacheBlockSize; + pCfg->walCfg.fsyncPeriod = pCreate->fsyncPeriod; + pCfg->walCfg.level = pCreate->walLevel; + pCfg->walCfg.retentionPeriod = 10; + pCfg->walCfg.retentionSize = 128; + pCfg->walCfg.rollPeriod = 128; + pCfg->walCfg.segSize = 128; + pCfg->walCfg.vgId = pCreate->vgId; + pCfg->hashBegin = pCreate->hashBegin; + pCfg->hashEnd = pCreate->hashEnd; + pCfg->hashMethod = pCreate->hashMethod; +} + +static void vmGenerateWrapperCfg(SVnodesMgmt *pMgmt, SCreateVnodeReq *pCreate, SWrapperCfg *pCfg) { + memcpy(pCfg->db, pCreate->db, TSDB_DB_FNAME_LEN); + pCfg->dbUid = pCreate->dbUid; + pCfg->dropped = 0; + snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCreate->vgId); + pCfg->vgId = pCreate->vgId; + pCfg->vgVersion = pCreate->vgVersion; +} + +int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pReq = &pMsg->rpcMsg; + SCreateVnodeReq createReq = {0}; + if (tDeserializeSCreateVnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + dDebug("vgId:%d, create vnode req is received", createReq.vgId); + + SVnodeCfg vnodeCfg = {0}; + vmGenerateVnodeCfg(&createReq, &vnodeCfg); + + SWrapperCfg wrapperCfg = {0}; + vmGenerateWrapperCfg(pMgmt, &createReq, &wrapperCfg); + + if (createReq.dnodeId != pMgmt->pDnode->dnodeId) { + terrno = TSDB_CODE_DND_VNODE_INVALID_OPTION; + dDebug("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); + return -1; + } + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, createReq.vgId); + if (pVnode != NULL) { + dDebug("vgId:%d, already exist", createReq.vgId); + vmReleaseVnode(pMgmt, pVnode); + terrno = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED; + return -1; + } + + vnodeCfg.pWrapper = pMgmt->pWrapper; + vnodeCfg.pTfs = pMgmt->pTfs; + vnodeCfg.dbId = wrapperCfg.dbUid; + SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg); + if (pImpl == NULL) { + dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); + return -1; + } + + int32_t code = vmOpenVnode(pMgmt, &wrapperCfg, pImpl); + if (code != 0) { + dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr()); + vnodeClose(pImpl); + vnodeDestroy(wrapperCfg.path); + terrno = code; + return code; + } + + code = vmWriteVnodesToFile(pMgmt); + if (code != 0) { + vnodeClose(pImpl); + vnodeDestroy(wrapperCfg.path); + terrno = code; + return code; + } + + return 0; +} + +int32_t vmProcessAlterVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pReq = &pMsg->rpcMsg; + SAlterVnodeReq alterReq = {0}; + if (tDeserializeSCreateVnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + dDebug("vgId:%d, alter vnode req is received", alterReq.vgId); + + SVnodeCfg vnodeCfg = {0}; + vmGenerateVnodeCfg(&alterReq, &vnodeCfg); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, alterReq.vgId); + if (pVnode == NULL) { + dDebug("vgId:%d, failed to alter vnode since %s", alterReq.vgId, terrstr()); + return -1; + } + + if (alterReq.vgVersion == pVnode->vgVersion) { + vmReleaseVnode(pMgmt, pVnode); + dDebug("vgId:%d, no need to alter vnode cfg for version unchanged ", alterReq.vgId); + return 0; + } + + if (vnodeAlter(pVnode->pImpl, &vnodeCfg) != 0) { + dError("vgId:%d, failed to alter vnode since %s", alterReq.vgId, terrstr()); + vmReleaseVnode(pMgmt, pVnode); + return -1; + } + + int32_t oldVersion = pVnode->vgVersion; + pVnode->vgVersion = alterReq.vgVersion; + int32_t code = vmWriteVnodesToFile(pMgmt); + if (code != 0) { + pVnode->vgVersion = oldVersion; + } + + vmReleaseVnode(pMgmt, pVnode); + return code; +} + +int32_t vmProcessDropVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pReq = &pMsg->rpcMsg; + SDropVnodeReq dropReq = {0}; + if (tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + int32_t vgId = dropReq.vgId; + dDebug("vgId:%d, drop vnode req is received", vgId); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); + if (pVnode == NULL) { + dDebug("vgId:%d, failed to drop since %s", vgId, terrstr()); + terrno = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; + return -1; + } + + pVnode->dropped = 1; + if (vmWriteVnodesToFile(pMgmt) != 0) { + pVnode->dropped = 0; + vmReleaseVnode(pMgmt, pVnode); + return -1; + } + + vmCloseVnode(pMgmt, pVnode); + vmWriteVnodesToFile(pMgmt); + + return 0; +} + +int32_t vmProcessSyncVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pReq = &pMsg->rpcMsg; + SSyncVnodeReq syncReq = {0}; + tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &syncReq); + + int32_t vgId = syncReq.vgId; + dDebug("vgId:%d, sync vnode req is received", vgId); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); + if (pVnode == NULL) { + dDebug("vgId:%d, failed to sync since %s", vgId, terrstr()); + return -1; + } + + if (vnodeSync(pVnode->pImpl) != 0) { + dError("vgId:%d, failed to sync vnode since %s", vgId, terrstr()); + vmReleaseVnode(pMgmt, pVnode); + return -1; + } + + vmReleaseVnode(pMgmt, pVnode); + return 0; +} + +int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pReq = &pMsg->rpcMsg; + SCompactVnodeReq compatcReq = {0}; + tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &compatcReq); + + int32_t vgId = compatcReq.vgId; + dDebug("vgId:%d, compact vnode req is received", vgId); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); + if (pVnode == NULL) { + dDebug("vgId:%d, failed to compact since %s", vgId, terrstr()); + return -1; + } + + if (vnodeCompact(pVnode->pImpl) != 0) { + dError("vgId:%d, failed to compact vnode since %s", vgId, terrstr()); + vmReleaseVnode(pMgmt, pVnode); + return -1; + } + + vmReleaseVnode(pMgmt, pVnode); + return 0; +} + +void vmInitMsgHandles(SMgmtWrapper *pWrapper) { + // Requests handled by VNODE + dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_EXEC, (NodeMsgFp)vmProcessFetchMsg); + + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, (NodeMsgFp)vmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, (NodeMsgFp)vmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, (NodeMsgFp)vmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, (NodeMsgFp)vmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, (NodeMsgFp)vmProcessMgmtMsg); +} diff --git a/source/dnode/mgmt/vnode/src/vmWorker.c b/source/dnode/mgmt/vnode/src/vmWorker.c new file mode 100644 index 0000000000..fe01b19d2d --- /dev/null +++ b/source/dnode/mgmt/vnode/src/vmWorker.c @@ -0,0 +1,340 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "vmInt.h" + +static void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { + SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; + dndSendRsp(pWrapper, &rsp); +} + +static void vmProcessMgmtQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + int32_t code = -1; + tmsg_t msgType = pMsg->rpcMsg.msgType; + dTrace("msg:%p, will be processed in vnode-mgmt queue", pMsg); + + switch (msgType) { + case TDMT_DND_CREATE_VNODE: + code = vmProcessCreateVnodeReq(pMgmt, pMsg); + break; + case TDMT_DND_ALTER_VNODE: + code = vmProcessAlterVnodeReq(pMgmt, pMsg); + break; + case TDMT_DND_DROP_VNODE: + code = vmProcessDropVnodeReq(pMgmt, pMsg); + break; + case TDMT_DND_SYNC_VNODE: + code = vmProcessSyncVnodeReq(pMgmt, pMsg); + break; + case TDMT_DND_COMPACT_VNODE: + code = vmProcessCompactVnodeReq(pMgmt, pMsg); + break; + default: + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + dError("msg:%p, not processed in vnode-mgmt queue", pMsg); + } + + if (msgType & 1u) { + if (code != 0 && terrno != 0) code = terrno; + vmSendRsp(pMgmt->pWrapper, pMsg, code); + } + + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); +} + +static void vmProcessQueryQueue(SVnodeObj *pVnode, SNodeMsg *pMsg) { + dTrace("msg:%p, will be processed in vnode-query queue", pMsg); + int32_t code = vnodeProcessQueryMsg(pVnode->pImpl, &pMsg->rpcMsg); + if (code != 0) { + vmSendRsp(pVnode->pWrapper, pMsg, code); + } + + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); +} + +static void vmProcessFetchQueue(SVnodeObj *pVnode, SNodeMsg *pMsg) { + dTrace("msg:%p, will be processed in vnode-fetch queue", pMsg); + int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, &pMsg->rpcMsg); + if (code != 0) { + vmSendRsp(pVnode->pWrapper, pMsg, code); + } + + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); +} + +static void vmProcessWriteQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { + SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SNodeMsg *)); + if (pArray == NULL) { + dError("failed to process %d msgs in write-queue since %s", numOfMsgs, terrstr()); + return; + } + + for (int32_t i = 0; i < numOfMsgs; ++i) { + SNodeMsg *pMsg = NULL; + if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; + + dTrace("msg:%p, will be processed in vnode-write queue", pMsg); + if (taosArrayPush(pArray, &pMsg) == NULL) { + dTrace("msg:%p, failed to process since %s", pMsg, terrstr()); + vmSendRsp(pVnode->pWrapper, pMsg, TSDB_CODE_OUT_OF_MEMORY); + } + } + + vnodeProcessWMsgs(pVnode->pImpl, pArray); + + numOfMsgs = taosArrayGetSize(pArray); + for (int32_t i = 0; i < numOfMsgs; i++) { + SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); + SRpcMsg *pRpc = &pMsg->rpcMsg; + SRpcMsg *pRsp = NULL; + + int32_t code = vnodeApplyWMsg(pVnode->pImpl, pRpc, &pRsp); + if (pRsp != NULL) { + pRsp->ahandle = pRpc->ahandle; + dndSendRsp(pVnode->pWrapper, pRsp); + free(pRsp); + } else { + if (code != 0 && terrno != 0) code = terrno; + vmSendRsp(pVnode->pWrapper, pMsg, code); + } + } + + for (int32_t i = 0; i < numOfMsgs; i++) { + SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); + dTrace("msg:%p, is freed", pMsg); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); + } + + taosArrayDestroy(pArray); +} + +static void vmProcessApplyQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { + SNodeMsg *pMsg = NULL; + + for (int32_t i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pMsg); + + // todo + SRpcMsg *pRsp = NULL; + (void)vnodeApplyWMsg(pVnode->pImpl, &pMsg->rpcMsg, &pRsp); + } +} + +static void vmProcessSyncQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { + SNodeMsg *pMsg = NULL; + + for (int32_t i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pMsg); + + // todo + SRpcMsg *pRsp = NULL; + (void)vnodeProcessSyncReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp); + } +} + +static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EVndQueueType qtype) { + SRpcMsg *pRpc = &pMsg->rpcMsg; + int32_t code = -1; + + SMsgHead *pHead = pRpc->pCont; + pHead->contLen = htonl(pHead->contLen); + pHead->vgId = htonl(pHead->vgId); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); + if (pVnode == NULL) { + dError("vgId:%d, failed to write msg:%p to queue since %s", pHead->vgId, pMsg, terrstr()); + return -1; + } + + switch (qtype) { + case VND_QUERY_QUEUE: + dTrace("msg:%p, will be written into vnode-query queue", pMsg); + code = taosWriteQitem(pVnode->pQueryQ, pMsg); + break; + case VND_FETCH_QUEUE: + dTrace("msg:%p, will be written into vnode-fetch queue", pMsg); + code = taosWriteQitem(pVnode->pFetchQ, pMsg); + break; + case VND_WRITE_QUEUE: + dTrace("msg:%p, will be written into vnode-write queue", pMsg); + code = taosWriteQitem(pVnode->pWriteQ, pMsg); + case VND_SYNC_QUEUE: + dTrace("msg:%p, will be written into vnode-sync queue", pMsg); + code = taosWriteQitem(pVnode->pSyncQ, pMsg); + default: + terrno = TSDB_CODE_INVALID_PARA; + break; + } + + vmReleaseVnode(pMgmt, pVnode); + return code; +} + +int32_t vmProcessSyncMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + return vmPutNodeMsgToQueue(pMgmt, pMsg, VND_SYNC_QUEUE); +} + +int32_t vmProcessWriteMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + return vmPutNodeMsgToQueue(pMgmt, pMsg, VND_WRITE_QUEUE); +} + +int32_t vmProcessQueryMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + return vmPutNodeMsgToQueue(pMgmt, pMsg, VND_QUERY_QUEUE); +} + +int32_t vmProcessFetchMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + return vmPutNodeMsgToQueue(pMgmt, pMsg, VND_FETCH_QUEUE); +} + +int32_t vmProcessMgmtMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SDnodeWorker *pWorker = &pMgmt->mgmtWorker; + dTrace("msg:%p, will be written to vnode-mgmt queue, worker:%s", pMsg, pWorker->name); + return dndWriteMsgToWorker(pWorker, pMsg); +} + +static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EVndQueueType qtype) { + SVnodesMgmt *pMgmt = pWrapper->pMgmt; + int32_t code = -1; + SMsgHead *pHead = pRpc->pCont; + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); + if (pVnode == NULL) return -1; + + SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); + if (pMsg != NULL) { + dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); + pMsg->rpcMsg = *pRpc; + switch (qtype) { + case VND_QUERY_QUEUE: + dTrace("msg:%p, will be put into vnode-query queue", pMsg); + code = taosWriteQitem(pVnode->pQueryQ, pMsg); + break; + case VND_FETCH_QUEUE: + dTrace("msg:%p, will be put into vnode-fetch queue", pMsg); + code = taosWriteQitem(pVnode->pFetchQ, pMsg); + break; + case VND_APPLY_QUEUE: + dTrace("msg:%p, will be put into vnode-apply queue", pMsg); + code = taosWriteQitem(pVnode->pApplyQ, pMsg); + break; + case VND_WRITE_QUEUE: + case VND_SYNC_QUEUE: + default: + terrno = TSDB_CODE_INVALID_PARA; + break; + } + } + vmReleaseVnode(pMgmt, pVnode); + return code; +} + +int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + return vmPutRpcMsgToQueue(pWrapper, pRpc, VND_QUERY_QUEUE); +} + +int32_t vmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + return vmPutRpcMsgToQueue(pWrapper, pRpc, VND_FETCH_QUEUE); +} + +int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + return vmPutRpcMsgToQueue(pWrapper, pRpc, VND_APPLY_QUEUE); +} + +int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { + pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessWriteQueue); + pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessApplyQueue); + pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue); + pVnode->pFetchQ = tFWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)vmProcessFetchQueue); + pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue); + + if (pVnode->pApplyQ == NULL || pVnode->pWriteQ == NULL || pVnode->pSyncQ == NULL || pVnode->pFetchQ == NULL || + pVnode->pQueryQ == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + dDebug("vgId:%d, vnode queue is alloced", pVnode->vgId); + return 0; +} + +void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { + tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ); + tFWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); + tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); + tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ); + tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); + pVnode->pWriteQ = NULL; + pVnode->pApplyQ = NULL; + pVnode->pSyncQ = NULL; + pVnode->pFetchQ = NULL; + pVnode->pQueryQ = NULL; + dDebug("vgId:%d, vnode queue is freed", pVnode->vgId); +} + +int32_t vmStartWorker(SVnodesMgmt *pMgmt) { + int32_t maxFetchThreads = 4; + int32_t minFetchThreads = TMIN(maxFetchThreads, tsNumOfCores); + int32_t minQueryThreads = TMAX((int32_t)(tsNumOfCores * tsRatioOfQueryCores), 1); + int32_t maxQueryThreads = minQueryThreads; + int32_t maxWriteThreads = TMAX(tsNumOfCores, 1); + int32_t maxSyncThreads = TMAX(tsNumOfCores / 2, 1); + + SQWorkerPool *pQPool = &pMgmt->queryPool; + pQPool->name = "vnode-query"; + pQPool->min = minQueryThreads; + pQPool->max = maxQueryThreads; + if (tQWorkerInit(pQPool) != 0) return -1; + + SFWorkerPool *pFPool = &pMgmt->fetchPool; + pFPool->name = "vnode-fetch"; + pFPool->min = minFetchThreads; + pFPool->max = maxFetchThreads; + if (tFWorkerInit(pFPool) != 0) return -1; + + SWWorkerPool *pWPool = &pMgmt->writePool; + pWPool->name = "vnode-write"; + pWPool->max = maxWriteThreads; + if (tWWorkerInit(pWPool) != 0) return -1; + + pWPool = &pMgmt->syncPool; + pWPool->name = "vnode-sync"; + pWPool->max = maxSyncThreads; + if (tWWorkerInit(pWPool) != 0) return -1; + + if (dndInitWorker(pMgmt, &pMgmt->mgmtWorker, DND_WORKER_SINGLE, "vnode-mgmt", 1, 1, vmProcessMgmtQueue) != 0) { + dError("failed to start vnode-mgmt worker since %s", terrstr()); + return -1; + } + + dDebug("vnode workers is initialized"); + return 0; +} + +void vmStopWorker(SVnodesMgmt *pMgmt) { + dndCleanupWorker(&pMgmt->mgmtWorker); + tFWorkerCleanup(&pMgmt->fetchPool); + tQWorkerCleanup(&pMgmt->queryPool); + tWWorkerCleanup(&pMgmt->writePool); + tWWorkerCleanup(&pMgmt->syncPool); + dDebug("vnode workers is closed"); +} diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 8ea9cc141f..909486aaac 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -702,16 +702,6 @@ typedef struct { int32_t tEncodeSStreamObj(SCoder* pEncoder, const SStreamObj* pObj); int32_t tDecodeSStreamObj(SCoder* pDecoder, SStreamObj* pObj); -typedef struct SMnodeMsg { - char user[TSDB_USER_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int32_t acctId; - SMnode* pMnode; - int64_t createdTime; - SRpcMsg rpcMsg; - int32_t contLen; - void* pCont; -} SMnodeMsg; #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 0fe24cd757..88dd3a5c22 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -38,11 +38,11 @@ extern "C" { #define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }} #define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }} -typedef int32_t (*MndMsgFp)(SMnodeMsg *pMsg); +typedef int32_t (*MndMsgFp)(SNodeMsg *pMsg); typedef int32_t (*MndInitFp)(SMnode *pMnode); typedef void (*MndCleanupFp)(SMnode *pMnode); -typedef int32_t (*ShowMetaFp)(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); -typedef int32_t (*ShowRetrieveFp)(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +typedef int32_t (*ShowMetaFp)(SNodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); +typedef int32_t (*ShowRetrieveFp)(SNodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter); typedef struct SMnodeLoad { @@ -110,7 +110,7 @@ typedef struct SMnode { char *path; int64_t checkTime; SSdb *pSdb; - SDnode *pDnode; + SMgmtWrapper *pWrapper; SArray *pSteps; SShowMgmt showMgmt; SProfileMgmt profileMgmt; @@ -119,16 +119,14 @@ typedef struct SMnode { SHashObj *infosMeta; SGrantInfo grant; MndMsgFp msgFp[TDMT_MAX]; - SendReqToDnodeFp sendReqToDnodeFp; - SendReqToMnodeFp sendReqToMnodeFp; - SendRedirectRspFp sendRedirectRspFp; - PutReqToMWriteQFp putReqToMWriteQFp; - PutReqToMReadQFp putReqToMReadQFp; + SendReqFp sendReqFp; + SendMnodeReqFp sendMnodeReqFp; + PutToQueueFp putToWriteQFp; + PutToQueueFp putToReadQFp; } SMnode; int32_t mndSendReqToDnode(SMnode *pMnode, SEpSet *pEpSet, SRpcMsg *rpcMsg); int32_t mndSendReqToMnode(SMnode *pMnode, SRpcMsg *pMsg); -void mndSendRedirectRsp(SMnode *pMnode, SRpcMsg *pMsg); void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); uint64_t mndGenerateUid(char *name, int32_t len); diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index a0218eaf65..5c1b0991be 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -47,7 +47,7 @@ void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen); void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); -void mndTransProcessRsp(SMnodeMsg *pRsp); +void mndTransProcessRsp(SNodeMsg *pRsp); void mndTransPullup(SMnode *pMnode); #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 85e9a15bd4..f42829eddf 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -28,7 +28,7 @@ SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId); void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup); SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup); int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); -SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup); +SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup); int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index aa87eb43a1..a6e6d57345 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -26,9 +26,9 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw); static int32_t mndAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct); static int32_t mndAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct); static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew); -static int32_t mndProcessCreateAcctReq(SMnodeMsg *pReq); -static int32_t mndProcessAlterAcctReq(SMnodeMsg *pReq); -static int32_t mndProcessDropAcctReq(SMnodeMsg *pReq); +static int32_t mndProcessCreateAcctReq(SNodeMsg *pReq); +static int32_t mndProcessAlterAcctReq(SNodeMsg *pReq); +static int32_t mndProcessDropAcctReq(SNodeMsg *pReq); int32_t mndInitAcct(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_ACCT, @@ -185,19 +185,19 @@ static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew) { return 0; } -static int32_t mndProcessCreateAcctReq(SMnodeMsg *pReq) { +static int32_t mndProcessCreateAcctReq(SNodeMsg *pReq) { terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; } -static int32_t mndProcessAlterAcctReq(SMnodeMsg *pReq) { +static int32_t mndProcessAlterAcctReq(SNodeMsg *pReq) { terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; } -static int32_t mndProcessDropAcctReq(SMnodeMsg *pReq) { +static int32_t mndProcessDropAcctReq(SNodeMsg *pReq) { terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index ab8b9350b0..20dd7f1378 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -17,7 +17,7 @@ #include "mndAuth.h" #include "mndUser.h" -static int32_t mndProcessAuthReq(SMnodeMsg *pReq); +static int32_t mndProcessAuthReq(SNodeMsg *pReq); int32_t mndInitAuth(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_AUTH, mndProcessAuthReq); @@ -45,7 +45,7 @@ int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, cha return 0; } -static int32_t mndProcessAuthReq(SMnodeMsg *pReq) { +static int32_t mndProcessAuthReq(SNodeMsg *pReq) { SAuthReq authReq = {0}; if (tDeserializeSAuthReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &authReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -56,15 +56,15 @@ static int32_t mndProcessAuthReq(SMnodeMsg *pReq) { memcpy(authRsp.user, authReq.user, TSDB_USER_LEN); int32_t code = - mndRetriveAuth(pReq->pMnode, authRsp.user, &authRsp.spi, &authRsp.encrypt, authRsp.secret, authRsp.ckey); + mndRetriveAuth(pReq->pNode, authRsp.user, &authRsp.spi, &authRsp.encrypt, authRsp.secret, authRsp.ckey); mTrace("user:%s, auth req received, spi:%d encrypt:%d ruser:%s", pReq->user, authRsp.spi, authRsp.encrypt, authRsp.user); int32_t contLen = tSerializeSAuthReq(NULL, 0, &authRsp); void *pRsp = rpcMallocCont(contLen); tSerializeSAuthReq(pRsp, contLen, &authRsp); - pReq->pCont = pRsp; - pReq->contLen = contLen; + pReq->pRsp = pRsp; + pReq->rspLen = contLen; return code; } diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index a992135464..d9a0df555c 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -29,12 +29,12 @@ static SSdbRow *mndBnodeActionDecode(SSdbRaw *pRaw); static int32_t mndBnodeActionInsert(SSdb *pSdb, SBnodeObj *pObj); static int32_t mndBnodeActionDelete(SSdb *pSdb, SBnodeObj *pObj); static int32_t mndBnodeActionUpdate(SSdb *pSdb, SBnodeObj *pOld, SBnodeObj *pNew); -static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessDropBnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessCreateBnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessDropBnodeRsp(SMnodeMsg *pRsp); -static int32_t mndGetBnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveBnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateBnodeReq(SNodeMsg *pReq); +static int32_t mndProcessDropBnodeReq(SNodeMsg *pReq); +static int32_t mndProcessCreateBnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessDropBnodeRsp(SNodeMsg *pRsp); +static int32_t mndGetBnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveBnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextBnode(SMnode *pMnode, void *pIter); int32_t mndInitBnode(SMnode *pMnode) { @@ -203,7 +203,7 @@ static int32_t mndSetCreateBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_BNODE; - action.acceptableCode = TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pReq); @@ -230,7 +230,7 @@ static int32_t mndSetCreateBnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_BNODE; - action.acceptableCode = TSDB_CODE_DND_BNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pReq); @@ -240,7 +240,7 @@ static int32_t mndSetCreateBnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S return 0; } -static int32_t mndCreateBnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode, SMCreateBnodeReq *pCreate) { +static int32_t mndCreateBnode(SMnode *pMnode, SNodeMsg *pReq, SDnodeObj *pDnode, SMCreateBnodeReq *pCreate) { int32_t code = -1; SBnodeObj bnodeObj = {0}; @@ -266,8 +266,8 @@ CREATE_BNODE_OVER: return code; } -static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateBnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SBnodeObj *pObj = NULL; SDnodeObj *pDnode = NULL; @@ -353,7 +353,7 @@ static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBn action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_BNODE; - action.acceptableCode = TSDB_CODE_DND_BNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pReq); @@ -363,7 +363,7 @@ static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBn return 0; } -static int32_t mndDropBnode(SMnode *pMnode, SMnodeMsg *pReq, SBnodeObj *pObj) { +static int32_t mndDropBnode(SMnode *pMnode, SNodeMsg *pReq, SBnodeObj *pObj) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_BNODE, &pReq->rpcMsg); @@ -382,8 +382,8 @@ DROP_BNODE_OVER: return code; } -static int32_t mndProcessDropBnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropBnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SBnodeObj *pObj = NULL; @@ -430,18 +430,18 @@ DROP_BNODE_OVER: return code; } -static int32_t mndProcessCreateBnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessCreateBnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessDropBnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessDropBnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndGetBnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetBnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -480,8 +480,8 @@ static int32_t mndGetBnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveBnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveBnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 3410a386da..ce47080645 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -26,8 +26,8 @@ static int32_t mndClusterActionInsert(SSdb *pSdb, SClusterObj *pCluster); static int32_t mndClusterActionDelete(SSdb *pSdb, SClusterObj *pCluster); static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pOldCluster, SClusterObj *pNewCluster); static int32_t mndCreateDefaultCluster(SMnode *pMnode); -static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveClusters(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetClusterMeta(SNodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveClusters(SNodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextCluster(SMnode *pMnode, void *pIter); int32_t mndInitCluster(SMnode *pMnode) { @@ -163,7 +163,7 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) { return sdbWrite(pMnode->pSdb, pRaw); } -static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) { +static int32_t mndGetClusterMeta(SNodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) { int32_t cols = 0; SSchema *pSchema = pMeta->pSchemas; @@ -201,8 +201,8 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp return 0; } -static int32_t mndRetrieveClusters(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndRetrieveClusters(SNodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 4a9d23aa15..d86665b832 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -34,9 +34,9 @@ static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer); -static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg); -static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveConsumer(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessConsumerMetaMsg(SNodeMsg *pMsg); +static int32_t mndGetConsumerMeta(SNodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveConsumer(SNodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter); int32_t mndInitConsumer(SMnode *pMnode) { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 6625c7f104..3c9e0b0a1d 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -31,14 +31,14 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw); static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb); static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb); static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew); -static int32_t mndProcessCreateDbReq(SMnodeMsg *pReq); -static int32_t mndProcessAlterDbReq(SMnodeMsg *pReq); -static int32_t mndProcessDropDbReq(SMnodeMsg *pReq); -static int32_t mndProcessUseDbReq(SMnodeMsg *pReq); -static int32_t mndProcessSyncDbReq(SMnodeMsg *pReq); -static int32_t mndProcessCompactDbReq(SMnodeMsg *pReq); -static int32_t mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveDbs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateDbReq(SNodeMsg *pReq); +static int32_t mndProcessAlterDbReq(SNodeMsg *pReq); +static int32_t mndProcessDropDbReq(SNodeMsg *pReq); +static int32_t mndProcessUseDbReq(SNodeMsg *pReq); +static int32_t mndProcessSyncDbReq(SNodeMsg *pReq); +static int32_t mndProcessCompactDbReq(SNodeMsg *pReq); +static int32_t mndGetDbMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextDb(SMnode *pMnode, void *pIter); int32_t mndInitDb(SMnode *pMnode) { @@ -384,7 +384,7 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pReq, SCreateDbReq *pCreate, SUserObj *pUser) { +static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate, SUserObj *pUser) { SDbObj dbObj = {0}; memcpy(dbObj.name, pCreate->db, TSDB_DB_FNAME_LEN); memcpy(dbObj.acct, pUser->acct, TSDB_USER_LEN); @@ -458,8 +458,8 @@ CREATE_DB_OVER: return code; } -static int32_t mndProcessCreateDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateDbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -622,7 +622,7 @@ static int32_t mndSetUpdateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pOld, SDbObj *pNew) { +static int32_t mndUpdateDb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pOld, SDbObj *pNew) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_ALTER_DB, &pReq->rpcMsg); if (pTrans == NULL) goto UPDATE_DB_OVER; @@ -642,8 +642,8 @@ UPDATE_DB_OVER: return code; } -static int32_t mndProcessAlterDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -802,7 +802,33 @@ static int32_t mndSetDropDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *p return 0; } -static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pDb) { +static int32_t mndBuildDropDbRsp(SDbObj *pDb, int32_t *pRspLen, void **ppRsp, bool useRpcMalloc) { + SDropDbRsp dropRsp = {0}; + if (pDb != NULL) { + memcpy(dropRsp.db, pDb->name, TSDB_DB_FNAME_LEN); + dropRsp.uid = pDb->uid; + } + + int32_t rspLen = tSerializeSDropDbRsp(NULL, 0, &dropRsp); + void *pRsp = NULL; + if (useRpcMalloc) { + pRsp = rpcMallocCont(rspLen); + } else { + pRsp = malloc(rspLen); + } + + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + tSerializeSDropDbRsp(pRsp, rspLen, &dropRsp); + *pRspLen = rspLen; + *ppRsp = pRsp; + return 0; +} + +static int32_t mndDropDb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pDb) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_DB, &pReq->rpcMsg); if (pTrans == NULL) goto DROP_DB_OVER; @@ -814,18 +840,9 @@ static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pDb) { if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto DROP_DB_OVER; if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto DROP_DB_OVER; - SDropDbRsp dropRsp = {0}; - memcpy(dropRsp.db, pDb->name, TSDB_DB_FNAME_LEN); - dropRsp.uid = pDb->uid; - - int32_t rspLen = tSerializeSDropDbRsp(NULL, 0, &dropRsp); - void *pRsp = malloc(rspLen); - if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto DROP_DB_OVER; - } - tSerializeSDropDbRsp(pRsp, rspLen, &dropRsp); - + int32_t rspLen = 0; + void *pRsp = NULL; + if (mndBuildDropDbRsp(pDb, &rspLen, &pRsp, false) < 0) goto DROP_DB_OVER; mndTransSetRpcRsp(pTrans, pRsp, rspLen); if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_DB_OVER; @@ -837,8 +854,8 @@ DROP_DB_OVER: return code; } -static int32_t mndProcessDropDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropDbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -854,7 +871,7 @@ static int32_t mndProcessDropDbReq(SMnodeMsg *pReq) { pDb = mndAcquireDb(pMnode, dropReq.db); if (pDb == NULL) { if (dropReq.ignoreNotExists) { - code = 0; + code = mndBuildDropDbRsp(pDb, &pReq->rspLen, &pReq->pRsp, true); goto DROP_DB_OVER; } else { terrno = TSDB_CODE_MND_DB_NOT_EXIST; @@ -948,8 +965,8 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) { sdbCancelFetch(pSdb, pIter); } -static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -1017,8 +1034,8 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { tSerializeSUseDbRsp(pRsp, contLen, &usedbRsp); - pReq->pCont = pRsp; - pReq->contLen = contLen; + pReq->pRsp = pRsp; + pReq->rspLen = contLen; USE_DB_OVER: if (code != 0) { @@ -1101,8 +1118,8 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, return 0; } -static int32_t mndProcessSyncDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessSyncDbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -1142,8 +1159,8 @@ SYNC_DB_OVER: return code; } -static int32_t mndProcessCompactDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCompactDbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -1183,8 +1200,8 @@ SYNC_DB_OVER: return code; } -static int32_t mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetDbMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -1324,8 +1341,8 @@ char *mnGetDbStr(char *src) { return pos; } -static int32_t mndRetrieveDbs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SDbObj *pDb = NULL; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index e231717e61..bf79c89a1c 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -49,17 +49,17 @@ static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode); static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode); static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOld, SDnodeObj *pNew); -static int32_t mndProcessCreateDnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessDropDnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessConfigDnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessStatusReq(SMnodeMsg *pReq); +static int32_t mndProcessCreateDnodeReq(SNodeMsg *pReq); +static int32_t mndProcessDropDnodeReq(SNodeMsg *pReq); +static int32_t mndProcessConfigDnodeReq(SNodeMsg *pReq); +static int32_t mndProcessConfigDnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessStatusReq(SNodeMsg *pReq); -static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveConfigs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetConfigMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter); -static int32_t mndGetDnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveDnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetDnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter); int32_t mndInitDnode(SMnode *pMnode) { @@ -296,8 +296,8 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) { return 0; } -static int32_t mndProcessStatusReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessStatusReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SStatusReq statusReq = {0}; SDnodeObj *pDnode = NULL; int32_t code = -1; @@ -424,8 +424,8 @@ static int32_t mndProcessStatusReq(SMnodeMsg *pReq) { tSerializeSStatusRsp(pHead, contLen, &statusRsp); taosArrayDestroy(statusRsp.pDnodeEps); - pReq->contLen = contLen; - pReq->pCont = pHead; + pReq->rspLen = contLen; + pReq->pRsp = pHead; } pDnode->lastAccessTime = curMs; @@ -437,7 +437,7 @@ PROCESS_STATUS_MSG_OVER: return code; } -static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pReq, SCreateDnodeReq *pCreate) { +static int32_t mndCreateDnode(SMnode *pMnode, SNodeMsg *pReq, SCreateDnodeReq *pCreate) { SDnodeObj dnodeObj = {0}; dnodeObj.id = sdbGetMaxId(pMnode->pSdb, SDB_DNODE); dnodeObj.createdTime = taosGetTimestampMs(); @@ -471,8 +471,8 @@ static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pReq, SCreateDnodeReq * return 0; } -static int32_t mndProcessCreateDnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateDnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SDnodeObj *pDnode = NULL; @@ -521,7 +521,7 @@ CREATE_DNODE_OVER: return code; } -static int32_t mndDropDnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode) { +static int32_t mndDropDnode(SMnode *pMnode, SNodeMsg *pReq, SDnodeObj *pDnode) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_DNODE, &pReq->rpcMsg); if (pTrans == NULL) { mError("dnode:%d, failed to drop since %s", pDnode->id, terrstr()); @@ -547,8 +547,8 @@ static int32_t mndDropDnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode) return 0; } -static int32_t mndProcessDropDnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropDnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SDnodeObj *pDnode = NULL; @@ -596,8 +596,8 @@ DROP_DNODE_OVER: return code; } -static int32_t mndProcessConfigDnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessConfigDnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SMCfgDnodeReq cfgReq = {0}; if (tDeserializeSMCfgDnodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &cfgReq) != 0) { @@ -628,11 +628,11 @@ static int32_t mndProcessConfigDnodeReq(SMnodeMsg *pReq) { return 0; } -static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessConfigDnodeRsp(SNodeMsg *pRsp) { mInfo("app:%p config rsp from dnode", pRsp->rpcMsg.ahandle); } -static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { +static int32_t mndGetConfigMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { int32_t cols = 0; SSchema *pSchema = pMeta->pSchemas; @@ -663,8 +663,8 @@ static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp return 0; } -static int32_t mndRetrieveConfigs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; int32_t totalRows = 0; int32_t numOfRows = 0; char *cfgOpts[TSDB_CONFIG_NUMBER] = {0}; @@ -709,8 +709,8 @@ static int32_t mndRetrieveConfigs(SMnodeMsg *pReq, SShowObj *pShow, char *data, static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter) {} -static int32_t mndGetDnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetDnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -773,8 +773,8 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveDnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index b2daf848c5..b43d887dd7 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -29,13 +29,13 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw); static int32_t mndFuncActionInsert(SSdb *pSdb, SFuncObj *pFunc); static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc); static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew); -static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pReq, SCreateFuncReq *pCreate); -static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pReq, SFuncObj *pFunc); -static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq); -static int32_t mndProcessDropFuncReq(SMnodeMsg *pReq); -static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq); -static int32_t mndGetFuncMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveFuncs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndCreateFunc(SMnode *pMnode, SNodeMsg *pReq, SCreateFuncReq *pCreate); +static int32_t mndDropFunc(SMnode *pMnode, SNodeMsg *pReq, SFuncObj *pFunc); +static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq); +static int32_t mndProcessDropFuncReq(SNodeMsg *pReq); +static int32_t mndProcessRetrieveFuncReq(SNodeMsg *pReq); +static int32_t mndGetFuncMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveFuncs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextFunc(SMnode *pMnode, void *pIter); int32_t mndInitFunc(SMnode *pMnode) { @@ -181,7 +181,7 @@ static void mndReleaseFunc(SMnode *pMnode, SFuncObj *pFunc) { sdbRelease(pSdb, pFunc); } -static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pReq, SCreateFuncReq *pCreate) { +static int32_t mndCreateFunc(SMnode *pMnode, SNodeMsg *pReq, SCreateFuncReq *pCreate) { int32_t code = -1; STrans *pTrans = NULL; @@ -234,7 +234,7 @@ CREATE_FUNC_OVER: return code; } -static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pReq, SFuncObj *pFunc) { +static int32_t mndDropFunc(SMnode *pMnode, SNodeMsg *pReq, SFuncObj *pFunc) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_FUNC, &pReq->rpcMsg); if (pTrans == NULL) goto DROP_FUNC_OVER; @@ -262,8 +262,8 @@ DROP_FUNC_OVER: return code; } -static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SFuncObj *pFunc = NULL; @@ -339,8 +339,8 @@ CREATE_FUNC_OVER: return code; } -static int32_t mndProcessDropFuncReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropFuncReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SFuncObj *pFunc = NULL; @@ -394,8 +394,8 @@ DROP_FUNC_OVER: return code; } -static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessRetrieveFuncReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SRetrieveFuncReq retrieveReq = {0}; SRetrieveFuncRsp retrieveRsp = {0}; @@ -451,8 +451,8 @@ static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq) { tSerializeSRetrieveFuncRsp(pRsp, contLen, &retrieveRsp); - pReq->pCont = pRsp; - pReq->contLen = contLen; + pReq->pRsp = pRsp; + pReq->rspLen = contLen; code = 0; @@ -463,8 +463,8 @@ RETRIEVE_FUNC_OVER: return code; } -static int32_t mndGetFuncMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetFuncMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -545,8 +545,8 @@ static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int16_t le return tDataTypes[type].name; } -static int32_t mndRetrieveFuncs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveFuncs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SFuncObj *pFunc = NULL; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 9a6297a0f4..adeb1c7b65 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -30,13 +30,13 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw); static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj); static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj); static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOld, SMnodeObj *pNew); -static int32_t mndProcessCreateMnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessDropMnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessCreateMnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessAlterMnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessDropMnodeRsp(SMnodeMsg *pRsp); -static int32_t mndGetMnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveMnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateMnodeReq(SNodeMsg *pReq); +static int32_t mndProcessDropMnodeReq(SNodeMsg *pReq); +static int32_t mndProcessCreateMnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessAlterMnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessDropMnodeRsp(SNodeMsg *pRsp); +static int32_t mndGetMnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveMnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter); int32_t mndInitMnode(SMnode *pMnode) { @@ -320,7 +320,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_ALTER_MNODE; - action.acceptableCode = TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pReq); @@ -345,7 +345,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_MNODE; - action.acceptableCode = TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pReq); return -1; @@ -355,7 +355,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno return 0; } -static int32_t mndCreateMnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode, SMCreateMnodeReq *pCreate) { +static int32_t mndCreateMnode(SMnode *pMnode, SNodeMsg *pReq, SDnodeObj *pDnode, SMCreateMnodeReq *pCreate) { int32_t code = -1; SMnodeObj mnodeObj = {0}; @@ -380,8 +380,8 @@ CREATE_MNODE_OVER: return code; } -static int32_t mndProcessCreateMnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateMnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SMnodeObj *pObj = NULL; SDnodeObj *pDnode = NULL; @@ -490,7 +490,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_ALTER_MNODE; - action.acceptableCode = TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pReq); @@ -517,7 +517,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_MNODE; - action.acceptableCode = TSDB_CODE_DND_MNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pReq); return -1; @@ -527,7 +527,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode return 0; } -static int32_t mndDropMnode(SMnode *pMnode, SMnodeMsg *pReq, SMnodeObj *pObj) { +static int32_t mndDropMnode(SMnode *pMnode, SNodeMsg *pReq, SMnodeObj *pObj) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_MNODE, &pReq->rpcMsg); @@ -547,8 +547,8 @@ DROP_MNODE_OVER: return code; } -static int32_t mndProcessDropMnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropMnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SMnodeObj *pObj = NULL; @@ -595,23 +595,23 @@ DROP_MNODE_OVER: return code; } -static int32_t mndProcessCreateMnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessCreateMnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessAlterMnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessAlterMnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessDropMnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessDropMnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndGetMnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetMnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -663,8 +663,8 @@ static int32_t mndGetMnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveMnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveMnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index ac9e99ebd4..38157cf220 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -32,7 +32,7 @@ static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset); static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset); static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOffset, SMqOffsetObj *pNewOffset); -static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pReq); +static int32_t mndProcessCommitOffsetReq(SNodeMsg *pReq); int32_t mndInitOffset(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_OFFSET, @@ -152,10 +152,10 @@ int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicNam return 0; } -static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pMsg) { +static int32_t mndProcessCommitOffsetReq(SNodeMsg *pMsg) { char key[TSDB_PARTITION_KEY_LEN]; - SMnode *pMnode = pMsg->pMnode; + SMnode *pMnode = pMsg->pNode; char *msgStr = pMsg->rpcMsg.pCont; SMqCMCommitOffsetReq commitOffsetReq; SCoder decoder; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index dbf299e8da..7ca1984457 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -50,14 +50,14 @@ static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId); static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); static void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter); static void mndCancelGetNextConn(SMnode *pMnode, void *pIter); -static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq); -static int32_t mndProcessConnectReq(SMnodeMsg *pReq); -static int32_t mndProcessKillQueryReq(SMnodeMsg *pReq); -static int32_t mndProcessKillConnReq(SMnodeMsg *pReq); -static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveConns(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); -static int32_t mndGetQueryMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq); +static int32_t mndProcessConnectReq(SNodeMsg *pReq); +static int32_t mndProcessKillQueryReq(SNodeMsg *pReq); +static int32_t mndProcessKillConnReq(SNodeMsg *pReq); +static int32_t mndGetConnsMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveConns(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetQueryMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveQueries(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter); int32_t mndInitProfile(SMnode *pMnode) { @@ -177,8 +177,8 @@ static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) { } } -static int32_t mndProcessConnectReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessConnectReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SUserObj *pUser = NULL; SDbObj *pDb = NULL; SConnObj *pConn = NULL; @@ -206,8 +206,9 @@ static int32_t mndProcessConnectReq(SMnodeMsg *pReq) { } if (connReq.db[0]) { - snprintf(pReq->db, TSDB_DB_FNAME_LEN, "%d%s%s", pUser->acctId, TS_PATH_DELIMITER, connReq.db); - pDb = mndAcquireDb(pMnode, pReq->db); + char db[TSDB_DB_FNAME_LEN]; + snprintf(db, TSDB_DB_FNAME_LEN, "%d%s%s", pUser->acctId, TS_PATH_DELIMITER, connReq.db); + pDb = mndAcquireDb(pMnode, db); if (pDb == NULL) { terrno = TSDB_CODE_MND_INVALID_DB; mError("user:%s, failed to login from %s while use db:%s since %s", pReq->user, ip, connReq.db, terrstr()); @@ -237,8 +238,8 @@ static int32_t mndProcessConnectReq(SMnodeMsg *pReq) { if (pRsp == NULL) goto CONN_OVER; tSerializeSConnectRsp(pRsp, contLen, &connectRsp); - pReq->contLen = contLen; - pReq->pCont = pRsp; + pReq->rspLen = contLen; + pReq->pRsp = pRsp; mDebug("user:%s, login from %s, conn:%d, app:%s", info.user, ip, pConn->id, connReq.app); @@ -338,8 +339,8 @@ static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) { return NULL; } -static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SClientHbBatchReq batchReq = {0}; if (tDeserializeSClientHbBatchReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &batchReq) != 0) { @@ -423,12 +424,12 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { } taosArrayDestroy(batchRsp.rsps); - pReq->contLen = tlen; - pReq->pCont = buf; + pReq->rspLen = tlen; + pReq->pRsp = buf; return 0; #if 0 - SMnode *pMnode = pReq->pMnode; + SMnode *pMnode = pReq->pNode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; SHeartBeatReq *pHeartbeat = pReq->rpcMsg.pCont; @@ -495,13 +496,13 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { mndReleaseConn(pMnode, pConn); pReq->contLen = sizeof(SConnectRsp); - pReq->pCont = pRsp; + pReq->pRsp = pRsp; return 0; #endif } -static int32_t mndProcessKillQueryReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessKillQueryReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; SUserObj *pUser = mndAcquireUser(pMnode, pReq->user); @@ -534,8 +535,8 @@ static int32_t mndProcessKillQueryReq(SMnodeMsg *pReq) { } } -static int32_t mndProcessKillConnReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessKillConnReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; SUserObj *pUser = mndAcquireUser(pMnode, pReq->user); @@ -566,8 +567,8 @@ static int32_t mndProcessKillConnReq(SMnodeMsg *pReq) { } } -static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetConnsMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; SUserObj *pUser = mndAcquireUser(pMnode, pReq->user); @@ -641,8 +642,8 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveConns(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveConns(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; int32_t numOfRows = 0; SConnObj *pConn = NULL; int32_t cols = 0; @@ -700,8 +701,8 @@ static int32_t mndRetrieveConns(SMnodeMsg *pReq, SShowObj *pShow, char *data, in return numOfRows; } -static int32_t mndGetQueryMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetQueryMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; SUserObj *pUser = mndAcquireUser(pMnode, pReq->user); @@ -815,8 +816,8 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveQueries(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; int32_t numOfRows = 0; SConnObj *pConn = NULL; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 980e1a5a3f..3c66160c91 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -29,14 +29,14 @@ static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw); static int32_t mndQnodeActionInsert(SSdb *pSdb, SQnodeObj *pObj); static int32_t mndQnodeActionDelete(SSdb *pSdb, SQnodeObj *pObj); static int32_t mndQnodeActionUpdate(SSdb *pSdb, SQnodeObj *pOld, SQnodeObj *pNew); -static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessDropQnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessCreateQnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessDropQnodeRsp(SMnodeMsg *pRsp); -static int32_t mndGetQnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveQnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateQnodeReq(SNodeMsg *pReq); +static int32_t mndProcessDropQnodeReq(SNodeMsg *pReq); +static int32_t mndProcessCreateQnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessDropQnodeRsp(SNodeMsg *pRsp); +static int32_t mndGetQnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveQnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextQnode(SMnode *pMnode, void *pIter); -static int32_t mndProcessQnodeListReq(SMnodeMsg *pReq); +static int32_t mndProcessQnodeListReq(SNodeMsg *pReq); int32_t mndInitQnode(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_QNODE, @@ -205,7 +205,7 @@ static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_QNODE; - action.acceptableCode = TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pReq); @@ -232,7 +232,7 @@ static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_QNODE; - action.acceptableCode = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pReq); @@ -242,7 +242,7 @@ static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S return 0; } -static int32_t mndCreateQnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode, SMCreateQnodeReq *pCreate) { +static int32_t mndCreateQnode(SMnode *pMnode, SNodeMsg *pReq, SDnodeObj *pDnode, SMCreateQnodeReq *pCreate) { int32_t code = -1; SQnodeObj qnodeObj = {0}; @@ -268,8 +268,8 @@ CREATE_QNODE_OVER: return code; } -static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateQnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SQnodeObj *pObj = NULL; SDnodeObj *pDnode = NULL; @@ -355,7 +355,7 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_QNODE; - action.acceptableCode = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pReq); @@ -365,7 +365,7 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn return 0; } -static int32_t mndDropQnode(SMnode *pMnode, SMnodeMsg *pReq, SQnodeObj *pObj) { +static int32_t mndDropQnode(SMnode *pMnode, SNodeMsg *pReq, SQnodeObj *pObj) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_QNODE, &pReq->rpcMsg); @@ -384,8 +384,8 @@ DROP_QNODE_OVER: return code; } -static int32_t mndProcessDropQnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropQnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SQnodeObj *pObj = NULL; @@ -432,12 +432,11 @@ DROP_QNODE_OVER: return code; } - -static int32_t mndProcessQnodeListReq(SMnodeMsg *pReq) { +static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) { int32_t code = -1; SQnodeListReq qlistReq = {0}; int32_t numOfRows = 0; - SMnode *pMnode = pReq->pMnode; + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; SQnodeObj *pObj = NULL; SQnodeListRsp qlistRsp = {0}; @@ -483,8 +482,8 @@ static int32_t mndProcessQnodeListReq(SMnodeMsg *pReq) { tSerializeSQnodeListRsp(pRsp, rspLen, &qlistRsp); - pReq->contLen = rspLen; - pReq->pCont = pRsp; + pReq->rspLen = rspLen; + pReq->pRsp = pRsp; code = 0; QNODE_LIST_OVER: @@ -494,19 +493,18 @@ QNODE_LIST_OVER: return code; } - -static int32_t mndProcessCreateQnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessCreateQnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessDropQnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessDropQnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndGetQnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetQnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -545,8 +543,8 @@ static int32_t mndGetQnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveQnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveQnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index faef757e76..c28c0d76c4 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -31,6 +31,53 @@ #include "tname.h" #include "tuuid.h" +int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet* pEpSet) { + SCoder encoder; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); + tEncodeSStreamTask(&encoder, pTask); + int32_t tlen = sizeof(SMsgHead) + encoder.pos; + tCoderClear(&encoder); + void* buf = malloc(tlen); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + ((SMsgHead*)buf)->streamTaskId = pTask->taskId; + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, tlen, TD_ENCODER); + tEncodeSStreamTask(&encoder, pTask); + tCoderClear(&encoder); + + STransAction action = {0}; + memcpy(&action.epSet, pEpSet, sizeof(SEpSet)); + action.pCont = buf; + action.contLen = tlen; + action.msgType = TDMT_SND_TASK_DEPLOY; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + rpcFreeCont(buf); + return -1; + } + return 0; +} + +int32_t mndAssignTaskToVg(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SSubplan* plan, const SVgObj* pVgroup) { + int32_t msgLen; + plan->execNode.nodeId = pVgroup->vgId; + plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup); + + if (qSubPlanToString(plan, &pTask->qmsg, &msgLen) < 0) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } + mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet); + return 0; +} + +int32_t mndAssignTaskToSnode(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SSubplan* plan, + const SSnodeObj* pSnode) { + return 0; +} + int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { SSdb* pSdb = pMnode->pSdb; SVgObj* pVgroup = NULL; @@ -44,7 +91,6 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { int32_t totLevel = LIST_LENGTH(pPlan->pSubplans); pStream->tasks = taosArrayInit(totLevel, sizeof(SArray)); - int32_t msgLen; for (int32_t level = 0; level < totLevel; level++) { SArray* taskOneLevel = taosArrayInit(0, sizeof(SStreamTask)); SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, level); @@ -67,43 +113,16 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { // send to vnode SStreamTask* pTask = streamTaskNew(pStream->uid, level); - - plan->execNode.nodeId = pVgroup->vgId; - plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup); - if (qSubPlanToString(plan, &pTask->qmsg, &msgLen) < 0) { + // TODO: set to + pTask->parallel = 4; + if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); - terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } taosArrayPush(taskOneLevel, pTask); - - SCoder encoder; - tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); - tEncodeSStreamTask(&encoder, pTask); - int32_t tlen = sizeof(SMsgHead) + encoder.pos; - tCoderClear(&encoder); - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - ((SMsgHead*)buf)->streamTaskId = pTask->taskId; - void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, tlen, TD_ENCODER); - tEncodeSStreamTask(&encoder, pTask); - tCoderClear(&encoder); - - STransAction action = {0}; - action.epSet = plan->execNode.epSet; - action.pCont = buf; - action.contLen = tlen; - action.msgType = TDMT_VND_TASK_DEPLOY; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - rpcFreeCont(buf); - return -1; - } } + } else if (plan->subplanType == SUBPLAN_TYPE_SCAN) { // duplicatable @@ -113,88 +132,26 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { // if has snode, set to shared thread num in snode parallel = SND_SHARED_THREAD_NUM; - for (int32_t i = 0; i < parallel; i++) { - SStreamTask* pTask = streamTaskNew(pStream->uid, level); - - // TODO:get snode id and ep - plan->execNode.nodeId = pVgroup->vgId; - plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup); - - if (qSubPlanToString(plan, &pTask->qmsg, &msgLen) < 0) { - qDestroyQueryPlan(pPlan); - terrno = TSDB_CODE_QRY_INVALID_INPUT; - return -1; - } - - taosArrayPush(taskOneLevel, pTask); - - SCoder encoder; - tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); - tEncodeSStreamTask(&encoder, pTask); - int32_t tlen = sizeof(SMsgHead) + encoder.pos; - tCoderClear(&encoder); - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - ((SMsgHead*)buf)->streamTaskId = pTask->taskId; - void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, tlen, TD_ENCODER); - tEncodeSStreamTask(&encoder, pTask); - tCoderClear(&encoder); - - STransAction action = {0}; - action.epSet = plan->execNode.epSet; - action.pCont = buf; - action.contLen = tlen; - action.msgType = TDMT_SND_TASK_DEPLOY; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - rpcFreeCont(buf); - return -1; - } + SStreamTask* pTask = streamTaskNew(pStream->uid, level); + pTask->parallel = parallel; + // TODO:get snode id and ep + if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) { + sdbRelease(pSdb, pVgroup); + qDestroyQueryPlan(pPlan); + return -1; } + taosArrayPush(taskOneLevel, pTask); } else { // not duplicatable SStreamTask* pTask = streamTaskNew(pStream->uid, level); - // TODO:get snode id and ep - plan->execNode.nodeId = pVgroup->vgId; - plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup); - - if (qSubPlanToString(plan, &pTask->qmsg, &msgLen) < 0) { + // TODO: get snode + if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); - terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } taosArrayPush(taskOneLevel, pTask); - - SCoder encoder; - tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); - tEncodeSStreamTask(&encoder, pTask); - int32_t tlen = sizeof(SMsgHead) + encoder.pos; - tCoderClear(&encoder); - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - ((SMsgHead*)buf)->streamTaskId = pTask->taskId; - void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, tlen, TD_ENCODER); - tEncodeSStreamTask(&encoder, pTask); - tCoderClear(&encoder); - - STransAction action = {0}; - action.epSet = plan->execNode.epSet; - action.pCont = buf; - action.contLen = tlen; - action.msgType = TDMT_SND_TASK_DEPLOY; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - rpcFreeCont(buf); - return -1; - } } taosArrayPush(pStream->tasks, taskOneLevel); } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index ad97888ac5..33a8b7fbe5 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -22,10 +22,10 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowReq *pReq); static void mndFreeShowObj(SShowObj *pShow); static SShowObj *mndAcquireShowObj(SMnode *pMnode, int64_t showId); static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove); -static int32_t mndProcessShowReq(SMnodeMsg *pReq); -static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq); +static int32_t mndProcessShowReq(SNodeMsg *pReq); +static int32_t mndProcessRetrieveReq(SNodeMsg *pReq); static bool mndCheckRetrieveFinished(SShowObj *pShow); -static int32_t mndProcessRetrieveSysTableReq(SMnodeMsg *pReq); +static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq); int32_t mndInitShow(SMnode *pMnode) { SShowMgmt *pMgmt = &pMnode->showMgmt; @@ -117,8 +117,8 @@ static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) { taosCacheRelease(pMgmt->cache, (void **)(&pShow), forceRemove); } -static int32_t mndProcessShowReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessShowReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SShowMgmt *pMgmt = &pMnode->showMgmt; int32_t code = -1; SShowReq showReq = {0}; @@ -161,8 +161,8 @@ static int32_t mndProcessShowReq(SMnodeMsg *pReq) { int32_t bufLen = tSerializeSShowRsp(NULL, 0, &showRsp); void *pBuf = rpcMallocCont(bufLen); tSerializeSShowRsp(pBuf, bufLen, &showRsp); - pReq->contLen = bufLen; - pReq->pCont = pBuf; + pReq->rspLen = bufLen; + pReq->pRsp = pBuf; mndReleaseShowObj(pShow, false); } else { mndReleaseShowObj(pShow, true); @@ -178,8 +178,8 @@ SHOW_OVER: return code; } -static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessRetrieveReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SShowMgmt *pMgmt = &pMnode->showMgmt; int32_t rowsToRead = 0; int32_t size = 0; @@ -248,8 +248,8 @@ static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { pRsp->numOfRows = htonl(rowsRead); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision - pReq->pCont = pRsp; - pReq->contLen = size; + pReq->pRsp = pRsp; + pReq->rspLen = size; if (rowsRead == 0 || rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) { pRsp->completed = 1; @@ -263,8 +263,8 @@ static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { return TSDB_CODE_SUCCESS; } -static int32_t mndProcessRetrieveSysTableReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SShowMgmt *pMgmt = &pMnode->showMgmt; int32_t rowsToRead = 0; int32_t size = 0; @@ -348,8 +348,8 @@ static int32_t mndProcessRetrieveSysTableReq(SMnodeMsg *pReq) { pRsp->numOfRows = htonl(rowsRead); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision - pReq->pCont = pRsp; - pReq->contLen = size; + pReq->pRsp = pRsp; + pReq->rspLen = size; if (rowsRead == 0 || rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) { pRsp->completed = 1; diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 6040aa088c..dda02fecf2 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -29,12 +29,12 @@ static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw); static int32_t mndSnodeActionInsert(SSdb *pSdb, SSnodeObj *pObj); static int32_t mndSnodeActionDelete(SSdb *pSdb, SSnodeObj *pObj); static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOld, SSnodeObj *pNew); -static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessDropSnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessCreateSnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessDropSnodeRsp(SMnodeMsg *pRsp); -static int32_t mndGetSnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveSnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateSnodeReq(SNodeMsg *pReq); +static int32_t mndProcessDropSnodeReq(SNodeMsg *pReq); +static int32_t mndProcessCreateSnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessDropSnodeRsp(SNodeMsg *pRsp); +static int32_t mndGetSnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveSnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextSnode(SMnode *pMnode, void *pIter); int32_t mndInitSnode(SMnode *pMnode) { @@ -203,7 +203,7 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_SNODE; - action.acceptableCode = TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pReq); @@ -230,7 +230,7 @@ static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_SNODE; - action.acceptableCode = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pReq); @@ -240,7 +240,7 @@ static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S return 0; } -static int32_t mndCreateSnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode, SMCreateSnodeReq *pCreate) { +static int32_t mndCreateSnode(SMnode *pMnode, SNodeMsg *pReq, SDnodeObj *pDnode, SMCreateSnodeReq *pCreate) { int32_t code = -1; SSnodeObj snodeObj = {0}; @@ -267,8 +267,8 @@ CREATE_SNODE_OVER: return code; } -static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateSnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SSnodeObj *pObj = NULL; SDnodeObj *pDnode = NULL; @@ -355,7 +355,7 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_SNODE; - action.acceptableCode = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pReq); @@ -365,7 +365,7 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn return 0; } -static int32_t mndDropSnode(SMnode *pMnode, SMnodeMsg *pReq, SSnodeObj *pObj) { +static int32_t mndDropSnode(SMnode *pMnode, SNodeMsg *pReq, SSnodeObj *pObj) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_SNODE, &pReq->rpcMsg); @@ -385,8 +385,8 @@ DROP_SNODE_OVER: return code; } -static int32_t mndProcessDropSnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropSnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SSnodeObj *pObj = NULL; @@ -433,18 +433,18 @@ DROP_SNODE_OVER: return code; } -static int32_t mndProcessCreateSnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessCreateSnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessDropSnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessDropSnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndGetSnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetSnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -483,8 +483,8 @@ static int32_t mndGetSnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveSnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveSnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index d8091dfa76..121720f48f 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -33,15 +33,15 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw); static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb); static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb); static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew); -static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq); -static int32_t mndProcessMAlterStbReq(SMnodeMsg *pReq); -static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq); -static int32_t mndProcessVCreateStbRsp(SMnodeMsg *pRsp); -static int32_t mndProcessVAlterStbRsp(SMnodeMsg *pRsp); -static int32_t mndProcessVDropStbRsp(SMnodeMsg *pRsp); -static int32_t mndProcessTableMetaReq(SMnodeMsg *pReq); -static int32_t mndGetStbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq); +static int32_t mndProcessMAlterStbReq(SNodeMsg *pReq); +static int32_t mndProcessMDropStbReq(SNodeMsg *pReq); +static int32_t mndProcessVCreateStbRsp(SNodeMsg *pRsp); +static int32_t mndProcessVAlterStbRsp(SNodeMsg *pRsp); +static int32_t mndProcessVDropStbRsp(SNodeMsg *pRsp); +static int32_t mndProcessTableMetaReq(SNodeMsg *pReq); +static int32_t mndGetStbMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextStb(SMnode *pMnode, void *pIter); int32_t mndInitStb(SMnode *pMnode) { @@ -490,7 +490,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) { +static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) { SStbObj stbObj = {0}; memcpy(stbObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); memcpy(stbObj.db, pDb->name, TSDB_DB_FNAME_LEN); @@ -551,8 +551,8 @@ CREATE_STB_OVER: return code; } -static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SStbObj *pTopicStb = NULL; SStbObj *pStb = NULL; @@ -623,7 +623,7 @@ CREATE_STB_OVER: return code; } -static int32_t mndProcessVCreateStbRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessVCreateStbRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } @@ -980,7 +980,7 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndAlterStb(SMnode *pMnode, SMnodeMsg *pReq, const SMAltertbReq *pAlter, SDbObj *pDb, SStbObj *pOld) { +static int32_t mndAlterStb(SMnode *pMnode, SNodeMsg *pReq, const SMAltertbReq *pAlter, SDbObj *pDb, SStbObj *pOld) { SStbObj stbObj = {0}; taosRLockLatch(&pOld->lock); memcpy(&stbObj, pOld, sizeof(SStbObj)); @@ -1043,8 +1043,8 @@ ALTER_STB_OVER: return code; } -static int32_t mndProcessMAlterStbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessMAlterStbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SDbObj *pDb = NULL; SStbObj *pStb = NULL; @@ -1096,7 +1096,7 @@ ALTER_STB_OVER: return code; } -static int32_t mndProcessVAlterStbRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessVAlterStbRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } @@ -1160,7 +1160,7 @@ static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * return 0; } -static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pDb, SStbObj *pStb) { +static int32_t mndDropStb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pDb, SStbObj *pStb) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK,TRN_TYPE_DROP_STB, &pReq->rpcMsg); if (pTrans == NULL) goto DROP_STB_OVER; @@ -1180,8 +1180,8 @@ DROP_STB_OVER: return code; } -static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessMDropStbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SDbObj *pDb = NULL; @@ -1237,7 +1237,7 @@ DROP_STB_OVER: return code; } -static int32_t mndProcessVDropStbRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessVDropStbRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } @@ -1311,8 +1311,8 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char return code; } -static int32_t mndProcessTableMetaReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessTableMetaReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; STableInfoReq infoReq = {0}; STableMetaRsp metaRsp = {0}; @@ -1347,8 +1347,8 @@ static int32_t mndProcessTableMetaReq(SMnodeMsg *pReq) { } tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp); - pReq->pCont = pRsp; - pReq->contLen = rspLen; + pReq->pRsp = pRsp; + pReq->rspLen = rspLen; code = 0; mDebug("stb:%s.%s, meta is retrieved", infoReq.dbFName, infoReq.tbName); @@ -1436,8 +1436,8 @@ static int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs return 0; } -static int32_t mndGetStbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetStbMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; if (mndGetNumOfStbs(pMnode, pShow->db, &pShow->numOfRows) != 0) { @@ -1499,8 +1499,8 @@ static void mndExtractTableName(char *tableId, char *name) { } } -static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SStbObj *pStb = NULL; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index af0f354c11..e7cc12bb96 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -32,12 +32,12 @@ static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream); -static int32_t mndProcessCreateStreamReq(SMnodeMsg *pReq); -/*static int32_t mndProcessDropStreamReq(SMnodeMsg *pReq);*/ -/*static int32_t mndProcessDropStreamInRsp(SMnodeMsg *pRsp);*/ -static int32_t mndProcessStreamMetaReq(SMnodeMsg *pReq); -static int32_t mndGetStreamMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveStream(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateStreamReq(SNodeMsg *pReq); +/*static int32_t mndProcessDropStreamReq(SNodeMsg *pReq);*/ +/*static int32_t mndProcessDropStreamInRsp(SNodeMsg *pRsp);*/ +static int32_t mndProcessStreamMetaReq(SNodeMsg *pReq); +static int32_t mndGetStreamMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveStream(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextStream(SMnode *pMnode, void *pIter); int32_t mndInitStream(SMnode *pMnode) { @@ -209,7 +209,7 @@ static int32_t mndCheckCreateStreamReq(SCMCreateStreamReq *pCreate) { return 0; } -static int32_t mndCreateStream(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) { +static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) { mDebug("stream:%s to create", pCreate->name); SStreamObj streamObj = {0}; tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN); @@ -220,8 +220,8 @@ static int32_t mndCreateStream(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateStreamR streamObj.dbUid = pDb->uid; streamObj.version = 1; streamObj.sql = pCreate->sql; - streamObj.physicalPlan = pCreate->physicalPlan; - streamObj.logicalPlan = pCreate->logicalPlan; + streamObj.physicalPlan = ""; + streamObj.logicalPlan = ""; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg); if (pTrans == NULL) { @@ -254,8 +254,8 @@ static int32_t mndCreateStream(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateStreamR return 0; } -static int32_t mndProcessCreateStreamReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateStreamReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SStreamObj *pStream = NULL; SDbObj *pDb = NULL; @@ -346,8 +346,8 @@ static int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfS return 0; } -static int32_t mndGetStreamMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetStreamMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; if (mndGetNumOfStreams(pMnode, pShow->db, &pShow->numOfRows) != 0) { @@ -390,8 +390,8 @@ static int32_t mndGetStreamMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp return 0; } -static int32_t mndRetrieveStream(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveStream(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SStreamObj *pStream = NULL; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 1dcd07829d..25ec5f7cd4 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -48,14 +48,14 @@ static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *); static int32_t mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *); static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubscribeObj *pNewSub); -static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg); -static int32_t mndProcessSubscribeRsp(SMnodeMsg *pMsg); -static int32_t mndProcessSubscribeInternalReq(SMnodeMsg *pMsg); -static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pMsg); -static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg); -static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg); -static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg); -static int32_t mndProcessResetOffsetReq(SMnodeMsg *pMsg); +static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg); +static int32_t mndProcessSubscribeRsp(SNodeMsg *pMsg); +static int32_t mndProcessSubscribeInternalReq(SNodeMsg *pMsg); +static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pMsg); +static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg); +static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg); +static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg); +static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg); static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup, const SMqConsumerEp *pConsumerEp); @@ -211,8 +211,8 @@ static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMq } #if 0 -static int32_t mndProcessResetOffsetReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; uint8_t *str = pMsg->rpcMsg.pCont; SMqCMResetOffsetReq req; @@ -249,14 +249,14 @@ static int32_t mndProcessResetOffsetReq(SMnodeMsg *pMsg) { } #endif -static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont; SMqCMGetSubEpRsp rsp = {0}; int64_t consumerId = be64toh(pReq->consumerId); int32_t epoch = ntohl(pReq->epoch); - SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pMnode, consumerId); + SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pNode, consumerId); if (pConsumer == NULL) { terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; return -1; @@ -327,8 +327,8 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { tEncodeSMqCMGetSubEpRsp(&abuf, &rsp); tDeleteSMqCMGetSubEpRsp(&rsp); mndReleaseConsumer(pMnode, pConsumer); - pMsg->pCont = buf; - pMsg->contLen = tlen; + pMsg->pRsp = buf; + pMsg->rspLen = tlen; return 0; } @@ -356,8 +356,8 @@ static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { return pRebSub; } -static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; SSdb *pSdb = pMnode->pSdb; SMqConsumerObj *pConsumer; void *pIter = NULL; @@ -420,7 +420,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { .pCont = pRebMsg, .contLen = sizeof(SMqDoRebalanceMsg), }; - pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg); + (*pMnode->putToWriteQFp)(pMnode->pWrapper, &rpcMsg); } else { taosHashCleanup(pRebMsg->rebSubHash); rpcFreeCont(pRebMsg); @@ -428,8 +428,8 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; SMqDoRebalanceMsg *pReq = pMsg->rpcMsg.pCont; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_REBALANCE, &pMsg->rpcMsg); void *pIter = NULL; @@ -994,8 +994,8 @@ void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub) { sdbRelease(pSdb, pSub); } -static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; char *msgStr = pMsg->rpcMsg.pCont; SCMSubscribeReq subscribe; tDeserializeSCMSubscribeReq(msgStr, &subscribe); @@ -1156,7 +1156,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 453535c6e7..7993d7df9d 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -79,8 +79,8 @@ static char* mndBuildTelemetryReport(SMnode* pMnode) { return pCont; } -static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) { - SMnode* pMnode = pReq->pMnode; +static int32_t mndProcessTelemTimer(SNodeMsg* pReq) { + SMnode* pMnode = pReq->pNode; STelemMgmt* pMgmt = &pMnode->telemMgmt; if (!pMgmt->enable) return 0; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index e913cae1ac..e94ae4f8ec 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -31,12 +31,12 @@ static int32_t mndTopicActionInsert(SSdb *pSdb, SMqTopicObj *pTopic); static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic); static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pTopic, SMqTopicObj *pNewTopic); -static int32_t mndProcessCreateTopicReq(SMnodeMsg *pReq); -static int32_t mndProcessDropTopicReq(SMnodeMsg *pReq); -static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pRsp); -static int32_t mndProcessTopicMetaReq(SMnodeMsg *pReq); -static int32_t mndGetTopicMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveTopic(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateTopicReq(SNodeMsg *pReq); +static int32_t mndProcessDropTopicReq(SNodeMsg *pReq); +static int32_t mndProcessDropTopicInRsp(SNodeMsg *pRsp); +static int32_t mndProcessTopicMetaReq(SNodeMsg *pReq); +static int32_t mndGetTopicMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter); int32_t mndInitTopic(SMnode *pMnode) { @@ -246,7 +246,7 @@ static int32_t mndGetPlanString(SCMCreateTopicReq *pCreate, char **pStr) { SQueryPlan* pPlan = NULL; if (TSDB_CODE_SUCCESS == code) { - SPlanContext cxt = { .pAstRoot = pAst, .streamQuery = true }; + SPlanContext cxt = { .pAstRoot = pAst, .topicQuery = true }; code = qCreateQueryPlan(&cxt, &pPlan, NULL); } @@ -259,7 +259,7 @@ static int32_t mndGetPlanString(SCMCreateTopicReq *pCreate, char **pStr) { return code; } -static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb) { +static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb) { mDebug("topic:%s to create", pCreate->name); SMqTopicObj topicObj = {0}; tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN); @@ -312,8 +312,8 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateTopicReq return 0; } -static int32_t mndProcessCreateTopicReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateTopicReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SMqTopicObj *pTopic = NULL; SDbObj *pDb = NULL; @@ -377,7 +377,7 @@ CREATE_TOPIC_OVER: return code; } -static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pReq, SMqTopicObj *pTopic) { +static int32_t mndDropTopic(SMnode *pMnode, SNodeMsg *pReq, SMqTopicObj *pTopic) { // TODO: cannot drop when subscribed STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_TOPIC, &pReq->rpcMsg); if (pTrans == NULL) { @@ -404,8 +404,8 @@ static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pReq, SMqTopicObj *pTopic return 0; } -static int32_t mndProcessDropTopicReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropTopicReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SMDropTopicReq dropReq = {0}; if (tDeserializeSMDropTopicReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { @@ -439,7 +439,7 @@ static int32_t mndProcessDropTopicReq(SMnodeMsg *pReq) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessDropTopicInRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } @@ -471,8 +471,8 @@ static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTo return 0; } -static int32_t mndGetTopicMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetTopicMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; if (mndGetNumOfTopics(pMnode, pShow->db, &pShow->numOfRows) != 0) { @@ -515,8 +515,8 @@ static int32_t mndGetTopicMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveTopic(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SMqTopicObj *pTopic = NULL; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index a009e01a52..d3b36aed31 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -55,11 +55,11 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans); static void mndTransExecute(SMnode *pMnode, STrans *pTrans); static void mndTransSendRpcRsp(STrans *pTrans); -static int32_t mndProcessTransReq(SMnodeMsg *pReq); -static int32_t mndProcessKillTransReq(SMnodeMsg *pReq); +static int32_t mndProcessTransReq(SNodeMsg *pReq); +static int32_t mndProcessKillTransReq(SNodeMsg *pReq); -static int32_t mndGetTransMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveTrans(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetTransMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveTrans(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextTrans(SMnode *pMnode, void *pIter); int32_t mndInitTrans(SMnode *pMnode) { @@ -760,7 +760,7 @@ static void mndTransSendRpcRsp(STrans *pTrans) { } free(pTrans->rpcRsp); - mDebug("trans:%d, send rsp, code:0x%x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage, + mDebug("trans:%d, send rsp, code:0x%04x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage, pTrans->rpcAHandle); SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, .code = pTrans->code, @@ -774,8 +774,8 @@ static void mndTransSendRpcRsp(STrans *pTrans) { } } -void mndTransProcessRsp(SMnodeMsg *pRsp) { - SMnode *pMnode = pRsp->pMnode; +void mndTransProcessRsp(SNodeMsg *pRsp) { + SMnode *pMnode = pRsp->pNode; int64_t signature = (int64_t)(pRsp->rpcMsg.ahandle); int32_t transId = (int32_t)(signature >> 32); int32_t action = (int32_t)((signature << 32) >> 32); @@ -816,7 +816,7 @@ void mndTransProcessRsp(SMnodeMsg *pRsp) { } } - mDebug("trans:%d, action:%d response is received, code:0x%x, accept:0x%x", transId, action, pRsp->rpcMsg.code, + mDebug("trans:%d, action:%d response is received, code:0x%04x, accept:0x%04x", transId, action, pRsp->rpcMsg.code, pAction->acceptableCode); mndTransExecute(pMnode, pTrans); @@ -928,13 +928,13 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA mDebug("trans:%d, all %d actions execute successfully", pTrans->id, numOfActions); return 0; } else { - mError("trans:%d, all %d actions executed, code:0x%x", pTrans->id, numOfActions, errCode); + mError("trans:%d, all %d actions executed, code:0x%04x", pTrans->id, numOfActions, errCode & 0XFFFF); mndTransResetActions(pMnode, pTrans, pArray); terrno = errCode; return errCode; } } else { - mDebug("trans:%d, %d of %d actions executed, code:0x%x", pTrans->id, numOfReceived, numOfActions, errCode); + mDebug("trans:%d, %d of %d actions executed, code:0x%04x", pTrans->id, numOfReceived, numOfActions, errCode & 0XFFFF); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } } @@ -1111,7 +1111,7 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) { mError("trans:%d, failed to write sdb since %s", pTrans->id, terrstr()); } - mDebug("trans:%d, finished, code:0x%x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes); + mDebug("trans:%d, finished, code:0x%04x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes); return continueExec; } @@ -1157,8 +1157,8 @@ static void mndTransExecute(SMnode *pMnode, STrans *pTrans) { mndTransSendRpcRsp(pTrans); } -static int32_t mndProcessTransReq(SMnodeMsg *pReq) { - mndTransPullup(pReq->pMnode); +static int32_t mndProcessTransReq(SNodeMsg *pReq) { + mndTransPullup(pReq->pNode); return 0; } @@ -1199,8 +1199,8 @@ static int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) { return 0; } -static int32_t mndProcessKillTransReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessKillTransReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SKillTransReq killReq = {0}; int32_t code = -1; SUserObj *pUser = NULL; @@ -1257,8 +1257,8 @@ void mndTransPullup(SMnode *pMnode) { sdbWriteFile(pMnode->pSdb); } -static int32_t mndGetTransMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetTransMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -1320,8 +1320,8 @@ static int32_t mndGetTransMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveTrans(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveTrans(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; STrans *pTrans = NULL; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index b1cdf484b4..b7d32c01d9 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -30,13 +30,13 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw); static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser); static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser); static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew); -static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate, SMnodeMsg *pReq); -static int32_t mndProcessCreateUserReq(SMnodeMsg *pReq); -static int32_t mndProcessAlterUserReq(SMnodeMsg *pReq); -static int32_t mndProcessDropUserReq(SMnodeMsg *pReq); -static int32_t mndProcessGetUserAuthReq(SMnodeMsg *pReq); -static int32_t mndGetUserMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveUsers(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate, SNodeMsg *pReq); +static int32_t mndProcessCreateUserReq(SNodeMsg *pReq); +static int32_t mndProcessAlterUserReq(SNodeMsg *pReq); +static int32_t mndProcessDropUserReq(SNodeMsg *pReq); +static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq); +static int32_t mndGetUserMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextUser(SMnode *pMnode, void *pIter); int32_t mndInitUser(SMnode *pMnode) { @@ -261,7 +261,7 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) { sdbRelease(pSdb, pUser); } -static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate, SMnodeMsg *pReq) { +static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate, SNodeMsg *pReq) { SUserObj userObj = {0}; taosEncryptPass_c((uint8_t *)pCreate->pass, strlen(pCreate->pass), userObj.pass); tstrncpy(userObj.user, pCreate->user, TSDB_USER_LEN); @@ -295,8 +295,8 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate return 0; } -static int32_t mndProcessCreateUserReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateUserReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SUserObj *pOperUser = NULL; @@ -349,7 +349,7 @@ CREATE_USER_OVER: return code; } -static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SMnodeMsg *pReq) { +static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SNodeMsg *pReq) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_ALTER_USER,&pReq->rpcMsg); if (pTrans == NULL) { mError("user:%s, failed to update since %s", pOld->user, terrstr()); @@ -397,8 +397,8 @@ static SHashObj *mndDupDbHash(SHashObj *pOld) { return pNew; } -static int32_t mndProcessAlterUserReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SUserObj *pOperUser = NULL; @@ -510,7 +510,7 @@ ALTER_USER_OVER: return code; } -static int32_t mndDropUser(SMnode *pMnode, SMnodeMsg *pReq, SUserObj *pUser) { +static int32_t mndDropUser(SMnode *pMnode, SNodeMsg *pReq, SUserObj *pUser) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK,TRN_TYPE_DROP_USER, &pReq->rpcMsg); if (pTrans == NULL) { mError("user:%s, failed to drop since %s", pUser->user, terrstr()); @@ -536,8 +536,8 @@ static int32_t mndDropUser(SMnode *pMnode, SMnodeMsg *pReq, SUserObj *pUser) { return 0; } -static int32_t mndProcessDropUserReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropUserReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SUserObj *pOperUser = NULL; @@ -585,8 +585,8 @@ DROP_USER_OVER: return code; } -static int32_t mndProcessGetUserAuthReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SGetUserAuthReq authReq = {0}; @@ -635,8 +635,8 @@ static int32_t mndProcessGetUserAuthReq(SMnodeMsg *pReq) { tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp); - pReq->pCont = pRsp; - pReq->contLen = contLen; + pReq->pRsp = pRsp; + pReq->rspLen = contLen; code = 0; GET_AUTH_OVER: @@ -647,8 +647,8 @@ GET_AUTH_OVER: return code; } -static int32_t mndGetUserMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetUserMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -693,8 +693,8 @@ static int32_t mndGetUserMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *p return 0; } -static int32_t mndRetrieveUsers(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SUserObj *pUser = NULL; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index f7b177f170..f7d5226e77 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -29,17 +29,17 @@ static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup); static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew); -static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pRsp); +static int32_t mndProcessCreateVnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessAlterVnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessDropVnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessSyncVnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessCompactVnodeRsp(SNodeMsg *pRsp); -static int32_t mndGetVgroupMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveVgroups(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetVgroupMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter); -static int32_t mndGetVnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveVnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetVnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter); int32_t mndInitVgroup(SMnode *pMnode) { @@ -433,12 +433,12 @@ ALLOC_VGROUP_OVER: return code; } -SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup) { +SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup) { SEpSet epset = {0}; for (int32_t v = 0; v < pVgroup->replica; ++v) { - SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + const SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pDnode == NULL) continue; if (pVgid->role == TAOS_SYNC_STATE_LEADER) { @@ -452,24 +452,24 @@ SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup) { return epset; } -static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessCreateVnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessAlterVnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessDropVnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pRsp) { return 0; } +static int32_t mndProcessSyncVnodeRsp(SNodeMsg *pRsp) { return 0; } -static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pRsp) { return 0; } +static int32_t mndProcessCompactVnodeRsp(SNodeMsg *pRsp) { return 0; } static bool mndGetVgroupMaxReplicaFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { SVgObj *pVgroup = pObj; @@ -500,8 +500,8 @@ static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pRep return 0; } -static int32_t mndGetVgroupMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetVgroupMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; if (mndGetVgroupMaxReplica(pMnode, pShow->db, &pShow->replica, &pShow->numOfRows) != 0) { @@ -551,8 +551,8 @@ static int32_t mndGetVgroupMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp return 0; } -static int32_t mndRetrieveVgroups(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SVgObj *pVgroup = NULL; @@ -624,8 +624,8 @@ int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { return numOfVnodes; } -static int32_t mndGetVnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetVnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -664,8 +664,8 @@ static int32_t mndGetVnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveVnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SVgObj *pVgroup = NULL; diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index f2baea1cd9..39f2155109 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -44,27 +44,21 @@ #define TELEM_TIMER_MS 86400000 int32_t mndSendReqToDnode(SMnode *pMnode, SEpSet *pEpSet, SRpcMsg *pMsg) { - if (pMnode == NULL || pMnode->sendReqToDnodeFp == NULL) { + if (pMnode == NULL || pMnode->sendReqFp == NULL) { terrno = TSDB_CODE_MND_NOT_READY; return -1; } - return (*pMnode->sendReqToDnodeFp)(pMnode->pDnode, pEpSet, pMsg); + return (*pMnode->sendReqFp)(pMnode->pWrapper, pEpSet, pMsg); } int32_t mndSendReqToMnode(SMnode *pMnode, SRpcMsg *pMsg) { - if (pMnode == NULL || pMnode->sendReqToDnodeFp == NULL) { + if (pMnode == NULL || pMnode->sendReqFp == NULL) { terrno = TSDB_CODE_MND_NOT_READY; return -1; } - return (*pMnode->sendReqToMnodeFp)(pMnode->pDnode, pMsg); -} - -void mndSendRedirectRsp(SMnode *pMnode, SRpcMsg *pMsg) { - if (pMnode != NULL && pMnode->sendRedirectRspFp != NULL) { - (*pMnode->sendRedirectRspFp)(pMnode->pDnode, pMsg); - } + return (*pMnode->sendMnodeReqFp)(pMnode->pWrapper, pMsg); } static void *mndBuildTimerMsg(int32_t *pContLen) { @@ -86,7 +80,7 @@ static void mndPullupTrans(void *param, void *tmrId) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS_TIMER, .pCont = pReq, .contLen = contLen}; - pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg); + pMnode->putToWriteQFp(pMnode->pWrapper, &rpcMsg); } taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer); @@ -102,7 +96,7 @@ static void mndCalMqRebalance(void *param, void *tmrId) { .pCont = pReq, .contLen = contLen, }; - pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); + pMnode->putToReadQFp(pMnode->pWrapper, &rpcMsg); } taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer); @@ -114,7 +108,7 @@ static void mndPullupTelem(void *param, void *tmrId) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen}; - pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); + pMnode->putToReadQFp(pMnode->pWrapper, &rpcMsg); } taosTmrReset(mndPullupTelem, TELEM_TIMER_MS, pMnode, pMnode->timer, &pMnode->telemTimer); @@ -236,7 +230,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { } else { if (mndAllocStep(pMnode, "mnode-sdb-read", mndReadSdb, NULL) != 0) return -1; } - if (mndAllocStep(pMnode, "mnode-timer", mndInitTimer, NULL) != 0) return -1; + // if (mndAllocStep(pMnode, "mnode-timer", mndInitTimer, NULL) != 0) return -1; if (mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile) != 0) return -1; if (mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow) != 0) return -1; if (mndAllocStep(pMnode, "mnode-sync", mndInitSync, mndCleanupSync) != 0) return -1; @@ -292,15 +286,14 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->replica = pOption->replica; pMnode->selfIndex = pOption->selfIndex; memcpy(&pMnode->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); - pMnode->pDnode = pOption->pDnode; - pMnode->putReqToMWriteQFp = pOption->putReqToMWriteQFp; - pMnode->putReqToMReadQFp = pOption->putReqToMReadQFp; - pMnode->sendReqToDnodeFp = pOption->sendReqToDnodeFp; - pMnode->sendReqToMnodeFp = pOption->sendReqToMnodeFp; - pMnode->sendRedirectRspFp = pOption->sendRedirectRspFp; + pMnode->pWrapper = pOption->pWrapper; + pMnode->putToWriteQFp = pOption->putToWriteQFp; + pMnode->putToReadQFp = pOption->putToReadQFp; + pMnode->sendReqFp = pOption->sendReqFp; + pMnode->sendMnodeReqFp = pOption->sendMnodeReqFp; - if (pMnode->sendReqToDnodeFp == NULL || pMnode->sendReqToMnodeFp == NULL || pMnode->sendRedirectRspFp == NULL || - pMnode->putReqToMWriteQFp == NULL || pMnode->dnodeId < 0 || pMnode->clusterId < 0) { + if (pMnode->sendReqFp == NULL || pMnode->sendMnodeReqFp == NULL || + pMnode->putToWriteQFp == NULL || pMnode->dnodeId < 0 || pMnode->clusterId < 0) { terrno = TSDB_CODE_MND_INVALID_OPTIONS; return -1; } @@ -386,108 +379,49 @@ int32_t mndAlter(SMnode *pMnode, const SMnodeOpt *pOption) { return 0; } -void mndDestroy(const char *path) { - mDebug("start to destroy mnode at %s", path); - taosRemoveDir(path); - mDebug("mnode is destroyed"); +int32_t mndStart(SMnode *pMnode) { + return mndInitTimer(pMnode); } -SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { - SMnodeMsg *pMsg = taosAllocateQitem(sizeof(SMnodeMsg)); - if (pMsg == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("failed to create msg since %s, app:%p RPC:%p", terrstr(), pRpcMsg->ahandle, pRpcMsg->handle); - return NULL; - } +int32_t mndProcessMsg(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; + SRpcMsg *pRpc = &pMsg->rpcMsg; + tmsg_t msgType = pMsg->rpcMsg.msgType; + void *ahandle = pMsg->rpcMsg.ahandle; + bool isReq = (pRpc->msgType & 1U); - if (pRpcMsg->msgType != TDMT_MND_TRANS_TIMER && pRpcMsg->msgType != TDMT_MND_MQ_TIMER && - pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE && pRpcMsg->msgType != TDMT_MND_TELEM_TIMER) { - SRpcConnInfo connInfo = {0}; - if ((pRpcMsg->msgType & 1U) && rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) { - taosFreeQitem(pMsg); - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - mError("failed to create msg since %s, app:%p RPC:%p", terrstr(), pRpcMsg->ahandle, pRpcMsg->handle); - return NULL; - } - memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN); - } - - pMsg->pMnode = pMnode; - pMsg->rpcMsg = *pRpcMsg; - pMsg->createdTime = taosGetTimestampSec(); - - if (pRpcMsg != NULL) { - mTrace("msg:%p, is created, app:%p RPC:%p user:%s", pMsg, pRpcMsg->ahandle, pRpcMsg->handle, pMsg->user); - } - return pMsg; -} - -void mndCleanupMsg(SMnodeMsg *pMsg) { - mTrace("msg:%p, is destroyed", pMsg); - rpcFreeCont(pMsg->rpcMsg.pCont); - pMsg->rpcMsg.pCont = NULL; - taosFreeQitem(pMsg); -} - -void mndSendRsp(SMnodeMsg *pMsg, int32_t code) { - SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .code = code}; - rpcSendResponse(&rpcRsp); -} - -void mndProcessMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - int32_t code = 0; - tmsg_t msgType = pMsg->rpcMsg.msgType; - void *ahandle = pMsg->rpcMsg.ahandle; - bool isReq = (msgType & 1U); - - mTrace("msg:%p, type:%s will be processed, app:%p", pMsg, TMSG_INFO(msgType), ahandle); + mTrace("msg:%p, will be processed, type:%s app:%p", pMsg, TMSG_INFO(msgType), ahandle); if (isReq && !mndIsMaster(pMnode)) { - code = TSDB_CODE_APP_NOT_READY; + terrno = TSDB_CODE_APP_NOT_READY; mDebug("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); - goto PROCESS_RPC_END; + return -1; } - if (isReq && pMsg->rpcMsg.pCont == NULL) { - code = TSDB_CODE_MND_INVALID_MSG_LEN; + if (isReq && (pRpc->contLen == 0 || pRpc->pCont == NULL)) { + terrno = TSDB_CODE_MND_INVALID_MSG_LEN; mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); - goto PROCESS_RPC_END; + return -1; } MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(msgType)]; if (fp == NULL) { - code = TSDB_CODE_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_MSG_NOT_PROCESSED; mError("msg:%p, failed to process since no msg handle, app:%p", pMsg, ahandle); - goto PROCESS_RPC_END; + return -1; } - code = (*fp)(pMsg); + int32_t code = (*fp)(pMsg); if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { + terrno = code; mTrace("msg:%p, in progress, app:%p", pMsg, ahandle); - return; } else if (code != 0) { - code = terrno; mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); - goto PROCESS_RPC_END; } else { mTrace("msg:%p, is processed, app:%p", pMsg, ahandle); } -PROCESS_RPC_END: - if (isReq) { - if (pMsg->rpcMsg.handle == NULL) return; - - if (code == TSDB_CODE_APP_NOT_READY) { - mndSendRedirectRsp(pMnode, &pMsg->rpcMsg); - } else if (code != 0) { - SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .contLen = pMsg->contLen, .pCont = pMsg->pCont, .code = code}; - rpcSendResponse(&rpcRsp); - } else { - SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .contLen = pMsg->contLen, .pCont = pMsg->pCont}; - rpcSendResponse(&rpcRsp); - } - } + return code; } void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) { diff --git a/source/dnode/mnode/impl/test/bnode/bnode.cpp b/source/dnode/mnode/impl/test/bnode/bnode.cpp index d7d15df35a..4d691692ff 100644 --- a/source/dnode/mnode/impl/test/bnode/bnode.cpp +++ b/source/dnode/mnode/impl/test/bnode/bnode.cpp @@ -317,4 +317,4 @@ TEST_F(MndTestBnode, 04_Drop_Bnode_Rollback) { ASSERT_NE(retry, retryMax); } -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans.cpp index cea93017f4..9ddb3594bb 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans.cpp @@ -204,6 +204,8 @@ TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) { ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); } + taosMsleep(1000); + { // show trans test.SendShowMetaReq(TSDB_MGMT_TABLE_TRANS, ""); @@ -241,6 +243,7 @@ TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) { EXPECT_EQ(test.GetShowRows(), 0); } + uInfo("======== re-create trans"); // re-create trans { SMCreateQnodeReq createReq = {0}; @@ -255,10 +258,14 @@ TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) { ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); } + uInfo("======== kill and restart server") KillThenRestartServer(); + uInfo("======== server2 start") server2.DoStart(); + uInfo("======== server2 started") + { int32_t retry = 0; int32_t retryMax = 20; diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index a1c3f5b0d4..a257b343c2 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -30,7 +30,7 @@ SQnode *qndOpen(const SQnodeOpt *pOption) { } if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, pQnode, - (putReqToQueryQFp)qnodePutReqToVQueryQ, (sendReqToDnodeFp)qnodeSendReqToDnode)) { + (putReqToQueryQFp)qnodePutReqToVQueryQ, (sendReqFp)qnodeSendReqToDnode)) { tfree(pQnode); return NULL; } diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 2ecaeb00e9..f4129e37ce 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -38,13 +38,6 @@ void sndClose(SSnode *pSnode) { int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad) { return 0; } -/*int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {*/ -/**pRsp = NULL;*/ -/*return 0;*/ -/*}*/ - -void sndDestroy(const char *path) {} - SStreamMeta *sndMetaNew() { SStreamMeta *pMeta = calloc(1, sizeof(SStreamMeta)); if (pMeta == NULL) { @@ -64,7 +57,7 @@ void sndMetaDelete(SStreamMeta *pMeta) { } int32_t sndMetaDeployTask(SStreamMeta *pMeta, SStreamTask *pTask) { - pTask->executor = qCreateStreamExecTaskInfo(pTask->qmsg, NULL); + pTask->runner.executor = qCreateStreamExecTaskInfo(pTask->qmsg, NULL); return taosHashPut(pMeta->pHash, &pTask->taskId, sizeof(int32_t), pTask, sizeof(void *)); } @@ -84,16 +77,16 @@ int32_t sndMetaRemoveTask(SStreamMeta *pMeta, int32_t taskId) { } static int32_t sndProcessTaskExecReq(SSnode *pSnode, SRpcMsg *pMsg) { - SMsgHead *pHead = pMsg->pCont; - int32_t taskId = pHead->streamTaskId; - SStreamTask *pTask = sndMetaGetTask(pSnode->pMeta, taskId); + SStreamExecMsgHead *pHead = pMsg->pCont; + int32_t taskId = pHead->streamTaskId; + SStreamTask *pTask = sndMetaGetTask(pSnode->pMeta, taskId); if (pTask == NULL) { return -1; } return 0; } -int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) { +void sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) { // stream deploy // stream stop/resume // operator exec @@ -101,7 +94,7 @@ int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) { void *msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); SStreamTask *pTask = malloc(sizeof(SStreamTask)); if (pTask == NULL) { - return -1; + ASSERT(0); } SCoder decoder; tCoderInit(&decoder, TD_LITTLE_ENDIAN, msg, pMsg->contLen - sizeof(SMsgHead), TD_DECODER); @@ -114,15 +107,13 @@ int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) { } else { ASSERT(0); } - return 0; } -int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) { +void sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) { // operator exec if (pMsg->msgType == TDMT_SND_TASK_EXEC) { sndProcessTaskExecReq(pSnode, pMsg); } else { ASSERT(0); } - return 0; } diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index 8d8ed2e427..6391eaffea 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -55,6 +55,8 @@ int tqCommit(STQ*); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessSetConnReq(STQ* pTq, char* msg); int32_t tqProcessRebReq(STQ* pTq, char* msg); +int32_t tqProcessTaskExec(STQ* pTq, SRpcMsg* msg); + int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen); #ifdef __cplusplus diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index b6dd90a4e2..1ffd4e0d78 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -30,11 +30,8 @@ extern "C" { #endif /* ------------------------ TYPES EXPOSED ------------------------ */ -typedef struct SVnode SVnode; -typedef struct SDnode SDnode; -typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); -typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); - +typedef struct SMgmtWrapper SMgmtWrapper; +typedef struct SVnode SVnode; typedef struct { // TODO int32_t reserved; @@ -43,7 +40,7 @@ typedef struct { typedef struct { int32_t vgId; uint64_t dbId; - SDnode *pDnode; + void *pWrapper; STfs *pTfs; uint64_t wsize; uint64_t ssize; @@ -63,9 +60,12 @@ typedef struct { } SVnodeCfg; typedef struct { - uint16_t nthreads; // number of commit threads. 0 for no threads and a schedule queue should be given (TODO) - PutReqToVQueryQFp putReqToVQueryQFp; - SendReqToDnodeFp sendReqToDnodeFp; + uint16_t nthreads; // number of commit threads. 0 for no threads and a schedule queue should be given (TODO) + PutToQueueFp putToQueryQFp; + PutToQueueFp putToFetchQFp; + SendReqFp sendReqFp; + SendMnodeReqFp sendMnodeReqFp; + SendRspFp sendRspFp; } SVnodeOpt; typedef struct { @@ -126,9 +126,8 @@ void vnodeDestroy(const char *path); * * @param pVnode The vnode object. * @param pMsgs The array of SRpcMsg - * @return int 0 for success, -1 for failure */ -int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs); +void vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs); /** * @brief Apply a write request message. diff --git a/source/dnode/vnode/src/inc/tsdbDef.h b/source/dnode/vnode/src/inc/tsdbDef.h index 5e4c852621..f7fdc818d0 100644 --- a/source/dnode/vnode/src/inc/tsdbDef.h +++ b/source/dnode/vnode/src/inc/tsdbDef.h @@ -16,6 +16,7 @@ #ifndef _TD_TSDB_DEF_H_ #define _TD_TSDB_DEF_H_ +#include "tsdbDBDef.h" #include "tmallocator.h" #include "meta.h" #include "tcompression.h" @@ -27,7 +28,6 @@ #include "ttime.h" #include "tsdb.h" -#include "tsdbDBDef.h" #include "tsdbCommit.h" #include "tsdbFS.h" #include "tsdbFile.h" @@ -46,7 +46,7 @@ extern "C" { struct STsdb { int32_t vgId; bool repoLocked; - pthread_mutex_t mutex; + TdThreadMutex mutex; char * path; STsdbCfg config; STsdbMemTable * mem; @@ -63,6 +63,7 @@ struct STsdb { #define REPO_ID(r) ((r)->vgId) #define REPO_CFG(r) (&(r)->config) #define REPO_FS(r) (r)->fs +#define REPO_TFS(r) (r)->pTfs #define IS_REPO_LOCKED(r) (r)->repoLocked #define REPO_SMA_ENV(r, t) ((TSDB_SMA_TYPE_ROLLUP == (t)) ? (r)->pRSmaEnv : (r)->pTSmaEnv) diff --git a/source/dnode/vnode/src/inc/tsdbFS.h b/source/dnode/vnode/src/inc/tsdbFS.h index 96c5f8468f..8156cbae00 100644 --- a/source/dnode/vnode/src/inc/tsdbFS.h +++ b/source/dnode/vnode/src/inc/tsdbFS.h @@ -42,29 +42,29 @@ typedef struct { typedef struct { STsdbFSMeta meta; // FS meta SArray * df; // data file array - SArray * sf; // sma data file array v2(t|r)1900.index_name_1 + SArray * sf; // sma data file array v2f1900.index_name_1 } SFSStatus; /** * @brief Directory structure of .tsma data files. * - * /vnode2/tsdb $ tree .sma/ - * .sma/ - * ├── v2t100.index_name_1 - * ├── v2t101.index_name_1 - * ├── v2t102.index_name_1 - * ├── v2t1900.index_name_3 - * ├── v2t1901.index_name_3 - * ├── v2t1902.index_name_3 - * ├── v2t200.index_name_2 - * ├── v2t201.index_name_2 - * └── v2t202.index_name_2 + * /vnode2/tsdb $ tree tsma/ + * tsma/ + * ├── v2f100.index_name_1 + * ├── v2f101.index_name_1 + * ├── v2f102.index_name_1 + * ├── v2f1900.index_name_3 + * ├── v2f1901.index_name_3 + * ├── v2f1902.index_name_3 + * ├── v2f200.index_name_2 + * ├── v2f201.index_name_2 + * └── v2f202.index_name_2 * * 0 directories, 9 files */ typedef struct { - pthread_rwlock_t lock; + TdThreadRwlock lock; SFSStatus *cstatus; // current status SHashObj * metaCache; // meta cache @@ -108,7 +108,7 @@ SDFileSet *tsdbFSIterNext(SFSIter *pIter); int tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta); static FORCE_INLINE int tsdbRLockFS(STsdbFS *pFs) { - int code = pthread_rwlock_rdlock(&(pFs->lock)); + int code = taosThreadRwlockRdlock(&(pFs->lock)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); return -1; @@ -117,7 +117,7 @@ static FORCE_INLINE int tsdbRLockFS(STsdbFS *pFs) { } static FORCE_INLINE int tsdbWLockFS(STsdbFS *pFs) { - int code = pthread_rwlock_wrlock(&(pFs->lock)); + int code = taosThreadRwlockWrlock(&(pFs->lock)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); return -1; @@ -126,7 +126,7 @@ static FORCE_INLINE int tsdbWLockFS(STsdbFS *pFs) { } static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) { - int code = pthread_rwlock_unlock(&(pFs->lock)); + int code = taosThreadRwlockUnlock(&(pFs->lock)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); return -1; diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h index 649b5a2d47..f934b0263d 100644 --- a/source/dnode/vnode/src/inc/tsdbSma.h +++ b/source/dnode/vnode/src/inc/tsdbSma.h @@ -20,13 +20,15 @@ typedef struct SSmaStat SSmaStat; typedef struct SSmaEnv SSmaEnv; struct SSmaEnv { - pthread_rwlock_t lock; - TDBEnv dbEnv; - char * path; + TdThreadRwlock lock; + SDiskID did; + TDBEnv dbEnv; // TODO: If it's better to put it in smaIndex level? + char * path; // relative path SSmaStat * pStat; }; #define SMA_ENV_LOCK(env) ((env)->lock) +#define SMA_ENV_DID(env) ((env)->did) #define SMA_ENV_ENV(env) ((env)->dbEnv) #define SMA_ENV_PATH(env) ((env)->path) #define SMA_ENV_STAT(env) ((env)->pStat) @@ -49,7 +51,7 @@ static FORCE_INLINE int32_t tsdbEncodeTSmaKey(tb_uid_t tableUid, col_id_t colId, } static FORCE_INLINE int tsdbRLockSma(SSmaEnv *pEnv) { - int code = pthread_rwlock_rdlock(&(pEnv->lock)); + int code = taosThreadRwlockRdlock(&(pEnv->lock)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); return -1; @@ -58,7 +60,7 @@ static FORCE_INLINE int tsdbRLockSma(SSmaEnv *pEnv) { } static FORCE_INLINE int tsdbWLockSma(SSmaEnv *pEnv) { - int code = pthread_rwlock_wrlock(&(pEnv->lock)); + int code = taosThreadRwlockWrlock(&(pEnv->lock)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); return -1; @@ -67,7 +69,7 @@ static FORCE_INLINE int tsdbWLockSma(SSmaEnv *pEnv) { } static FORCE_INLINE int tsdbUnLockSma(SSmaEnv *pEnv) { - int code = pthread_rwlock_unlock(&(pEnv->lock)); + int code = taosThreadRwlockUnlock(&(pEnv->lock)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); return -1; diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 6f4f0049e3..de9b7bac83 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -46,16 +46,18 @@ typedef struct SVnodeTask { typedef struct SVnodeMgr { td_mode_flag_t vnodeInitFlag; // For commit - bool stop; - uint16_t nthreads; - pthread_t* threads; - pthread_mutex_t mutex; - pthread_cond_t hasTask; + bool stop; + uint16_t nthreads; + TdThread* threads; + TdThreadMutex mutex; + TdThreadCond hasTask; TD_DLIST(SVnodeTask) queue; // For vnode Mgmt - SDnode* pDnode; - PutReqToVQueryQFp putReqToVQueryQFp; - SendReqToDnodeFp sendReqToDnodeFp; + PutToQueueFp putToQueryQFp; + PutToQueueFp putToFetchQFp; + SendReqFp sendReqFp; + SendMnodeReqFp sendMnodeReqFp; + SendRspFp sendRspFp; } SVnodeMgr; extern SVnodeMgr vnodeMgr; @@ -79,14 +81,17 @@ struct SVnode { SWal* pWal; tsem_t canCommit; SQHandle* pQuery; - SDnode* pDnode; + void* pWrapper; STfs* pTfs; }; int vnodeScheduleTask(SVnodeTask* task); -int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq); -void vnodeSendReqToDnode(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq); +int32_t vnodePutToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq); +int32_t vnodePutToVFetchQ(SVnode* pVnode, struct SRpcMsg* pReq); +int32_t vnodeSendReq(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq); +int32_t vnodeSendMnodeReq(SVnode* pVnode, struct SRpcMsg* pReq); +void vnodeSendRsp(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pRsp); #define vFatal(...) \ do { \ diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index 99a2b272ed..d4cadc75e4 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -33,7 +33,7 @@ typedef struct { struct SMetaDB { #if IMPL_WITH_LOCK - pthread_rwlock_t rwlock; + TdThreadRwlock rwlock; #endif // DB DB *pTbDB; @@ -317,7 +317,7 @@ static SMetaDB *metaNewDB() { } #if IMPL_WITH_LOCK - pthread_rwlock_init(&pDB->rwlock, NULL); + taosThreadRwlockInit(&pDB->rwlock, NULL); #endif return pDB; @@ -326,7 +326,7 @@ static SMetaDB *metaNewDB() { static void metaFreeDB(SMetaDB *pDB) { if (pDB) { #if IMPL_WITH_LOCK - pthread_rwlock_destroy(&pDB->rwlock); + taosThreadRwlockDestroy(&pDB->rwlock); #endif free(pDB); } @@ -965,18 +965,18 @@ SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) { static void metaDBWLock(SMetaDB *pDB) { #if IMPL_WITH_LOCK - pthread_rwlock_wrlock(&(pDB->rwlock)); + taosThreadRwlockWrlock(&(pDB->rwlock)); #endif } static void metaDBRLock(SMetaDB *pDB) { #if IMPL_WITH_LOCK - pthread_rwlock_rdlock(&(pDB->rwlock)); + taosThreadRwlockRdlock(&(pDB->rwlock)); #endif } static void metaDBULock(SMetaDB *pDB) { #if IMPL_WITH_LOCK - pthread_rwlock_unlock(&(pDB->rwlock)); + taosThreadRwlockUnlock(&(pDB->rwlock)); #endif } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8cdc250e8d..02fecb49b7 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -433,3 +433,8 @@ int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { return 0; } + +int32_t tqProcessTaskExec(STQ* pTq, SRpcMsg* msg) { + // + return 0; +} diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index a2342ec85a..0c4b933c19 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "tdatablock.h" #include "vnode.h" STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { @@ -128,10 +129,13 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { int j = 0; for (int32_t i = 0; i < colNumNeed; i++) { - int32_t colId = *(int32_t*)taosArrayGet(pHandle->pColIdList, i); + int16_t colId = *(int16_t*)taosArrayGet(pHandle->pColIdList, i); while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) { j++; } + if (j >= pSchemaWrapper->nCols) { + continue; + } SSchema* pColSchema = &pSchemaWrapper->pSchema[j]; SColumnInfoData colInfo = {0}; int sz = numOfRows * pColSchema->bytes; @@ -145,6 +149,8 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { taosArrayDestroy(pArray); return NULL; } + + blockDataEnsureColumnCapacity(&colInfo, numOfRows); taosArrayPush(pArray, &colInfo); } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index a03739c90f..fa867543b0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -14,8 +14,8 @@ */ #include -#include "os.h" #include "tsdbDef.h" +#include "os.h" typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T; static const char *tsdbTxnFname[] = {"current.t", "current"}; @@ -203,7 +203,7 @@ STsdbFS *tsdbNewFS(const STsdbCfg *pCfg) { return NULL; } - int code = pthread_rwlock_init(&(pfs->lock), NULL); + int code = taosThreadRwlockInit(&(pfs->lock), NULL); if (code) { terrno = TAOS_SYSTEM_ERROR(code); free(pfs); @@ -241,7 +241,7 @@ void *tsdbFreeFS(STsdbFS *pfs) { taosHashCleanup(pfs->metaCache); pfs->metaCache = NULL; pfs->cstatus = tsdbFreeFSStatus(pfs->cstatus); - pthread_rwlock_destroy(&(pfs->lock)); + taosThreadRwlockDestroy(&(pfs->lock)); free(pfs); } @@ -1289,7 +1289,7 @@ static int tsdbRestoreCurrent(STsdb *pRepo) { } if (tsdbSaveFSStatus(pRepo, pRepo->fs->cstatus) < 0) { - tsdbError("vgId:%d failed to restore corrent since %s", REPO_ID(pRepo), tstrerror(terrno)); + tsdbError("vgId:%d failed to restore current since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c index afa8921c00..131e92cab6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMain.c +++ b/source/dnode/vnode/src/tsdb/tsdbMain.c @@ -110,7 +110,7 @@ static void tsdbCloseImpl(STsdb *pTsdb) { } int tsdbLockRepo(STsdb *pTsdb) { - int code = pthread_mutex_lock(&pTsdb->mutex); + int code = taosThreadMutexLock(&pTsdb->mutex); if (code != 0) { tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pTsdb), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); @@ -123,7 +123,7 @@ int tsdbLockRepo(STsdb *pTsdb) { int tsdbUnlockRepo(STsdb *pTsdb) { ASSERT(IS_REPO_LOCKED(pTsdb)); pTsdb->repoLocked = false; - int code = pthread_mutex_unlock(&pTsdb->mutex); + int code = taosThreadMutexUnlock(&pTsdb->mutex); if (code != 0) { tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pTsdb), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); @@ -298,7 +298,7 @@ STsdbCfg *tsdbGetCfg(const STsdbRepo *repo) { } int tsdbLockRepo(STsdbRepo *pRepo) { - int code = pthread_mutex_lock(&pRepo->mutex); + int code = taosThreadMutexLock(&pRepo->mutex); if (code != 0) { tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pRepo), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); @@ -311,7 +311,7 @@ int tsdbLockRepo(STsdbRepo *pRepo) { int tsdbUnlockRepo(STsdbRepo *pRepo) { ASSERT(IS_REPO_LOCKED(pRepo)); pRepo->repoLocked = false; - int code = pthread_mutex_unlock(&pRepo->mutex); + int code = taosThreadMutexUnlock(&pRepo->mutex); if (code != 0) { tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pRepo), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); @@ -388,7 +388,7 @@ int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { tsdbError("vgId:%d no config changed", REPO_ID(repo)); } - int code = pthread_mutex_lock(&repo->save_mutex); + int code = taosThreadMutexLock(&repo->save_mutex); if (code != 0) { tsdbError("vgId:%d failed to lock tsdb save config mutex since %s", REPO_ID(repo), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); @@ -416,7 +416,7 @@ int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { repo->config_changed = true; - pthread_mutex_unlock(&repo->save_mutex); + taosThreadMutexUnlock(&repo->save_mutex); // schedule a commit msg and wait for the new config applied tsdbSyncCommitConfig(repo); @@ -690,14 +690,14 @@ static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { pRepo->repoLocked = false; pRepo->pthread = NULL; - int code = pthread_mutex_init(&(pRepo->mutex), NULL); + int code = taosThreadMutexInit(&(pRepo->mutex), NULL); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); tsdbFreeRepo(pRepo); return NULL; } - code = pthread_mutex_init(&(pRepo->save_mutex), NULL); + code = taosThreadMutexInit(&(pRepo->save_mutex), NULL); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); tsdbFreeRepo(pRepo); @@ -747,7 +747,7 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) { // tsdbFreeMemTable(pRepo->mem); // tsdbFreeMemTable(pRepo->imem); tsem_destroy(&(pRepo->readyToCommit)); - pthread_mutex_destroy(&pRepo->mutex); + taosThreadMutexDestroy(&pRepo->mutex); free(pRepo); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 01813af556..e46d1a0ed4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -541,7 +541,7 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { } } if (addNew) { - int code = pthread_cond_signal(&pBufPool->poolNotEmpty); + int code = taosThreadCondSignal(&pBufPool->poolNotEmpty); if (code != 0) { if (tsdbUnlockRepo(pRepo) < 0) return -1; tsdbError("vgId:%d failed to signal pool not empty since %s", REPO_ID(pRepo), strerror(code)); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 6b2857c411..b7719240f9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "tsdbDef.h" #include #include "os.h" #include "talgo.h" @@ -20,7 +21,6 @@ #include "tdataformat.h" #include "texception.h" #include "tsdb.h" -#include "tsdbDef.h" #include "tsdbFS.h" #include "tsdbLog.h" #include "tsdbReadImpl.h" diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 02a0b587d5..0eb2d525b3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -67,7 +67,7 @@ typedef struct { */ int8_t state; // ETsdbSmaStat SHashObj *expiredWindows; // key: skey of time window, value: N/A - STSma * pSma; + STSma * pSma; // cache schema } SSmaStatItem; struct SSmaStat { @@ -81,8 +81,8 @@ struct SSmaStat { static int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg); static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat); static int32_t tsdbDestroySmaState(SSmaStat *pSmaStat); -static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path); -static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SSmaEnv **pEnv); +static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did); +static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SDiskID did, SSmaEnv **pEnv); static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t indexUid, TSKEY skey); static int32_t tsdbRefSmaStat(STsdb *pTsdb, SSmaStat *pStat); static int32_t tsdbUnRefSmaStat(STsdb *pTsdb, SSmaStat *pStat); @@ -102,8 +102,8 @@ static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *p static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen); static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision); static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLevel); -static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t storageLevel, int32_t fid); -static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, TSKEY skey); +static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int64_t indexUid, int32_t fid); +static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey); static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey); static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]); static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg); @@ -111,10 +111,11 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg); // implementation static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/%s", vgId, TSDB_SMA_DNAME[smaType]); + snprintf(dirName, TSDB_FILENAME_LEN, "vnode%svnode%d%stsdb%s%s", TD_DIRSEP, vgId, TD_DIRSEP, TD_DIRSEP, + TSDB_SMA_DNAME[smaType]); } -static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) { +static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did) { SSmaEnv *pEnv = NULL; pEnv = (SSmaEnv *)calloc(1, sizeof(SSmaEnv)); @@ -123,7 +124,7 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) { return NULL; } - int code = pthread_rwlock_init(&(pEnv->lock), NULL); + int code = taosThreadRwlockInit(&(pEnv->lock), NULL); if (code) { terrno = TAOS_SYSTEM_ERROR(code); free(pEnv); @@ -137,12 +138,16 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) { return NULL; } + pEnv->did = did; + if (tsdbInitSmaStat(&pEnv->pStat) != TSDB_CODE_SUCCESS) { tsdbFreeSmaEnv(pEnv); return NULL; } - if (tsdbOpenBDBEnv(&pEnv->dbEnv, pEnv->path) != TSDB_CODE_SUCCESS) { + char aname[TSDB_FILENAME_LEN] = {0}; + tfsAbsoluteName(pTsdb->pTfs, did, path, aname); + if (tsdbOpenBDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) { tsdbFreeSmaEnv(pEnv); return NULL; } @@ -150,14 +155,14 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) { return pEnv; } -static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SSmaEnv **pEnv) { +static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SDiskID did, SSmaEnv **pEnv) { if (!pEnv) { terrno = TSDB_CODE_INVALID_PTR; return TSDB_CODE_FAILED; } if (*pEnv == NULL) { - if ((*pEnv = tsdbNewSmaEnv(pTsdb, path)) == NULL) { + if ((*pEnv = tsdbNewSmaEnv(pTsdb, path, did)) == NULL) { return TSDB_CODE_FAILED; } } @@ -176,7 +181,7 @@ void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv) { tsdbDestroySmaState(pSmaEnv->pStat); tfree(pSmaEnv->pStat); tfree(pSmaEnv->path); - pthread_rwlock_destroy(&(pSmaEnv->lock)); + taosThreadRwlockDestroy(&(pSmaEnv->lock)); tsdbCloseBDBEnv(pSmaEnv->dbEnv); } } @@ -296,7 +301,6 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) { pEnv = (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_load_ptr(&pTsdb->pTSmaEnv) : atomic_load_ptr(&pTsdb->pRSmaEnv); if (pEnv == NULL) { char rname[TSDB_FILENAME_LEN] = {0}; - char aname[TSDB_FILENAME_LEN] = {0}; // use TSDB_FILENAME_LEN currently SDiskID did = {0}; tfsAllocDisk(pTsdb->pTfs, TFS_PRIMARY_LEVEL, &did); @@ -305,14 +309,13 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) { return TSDB_CODE_FAILED; } tsdbGetSmaDir(REPO_ID(pTsdb), smaType, rname); - tfsAbsoluteName(pTsdb->pTfs, did, rname, aname); if (tfsMkdirRecurAt(pTsdb->pTfs, rname, did) != TSDB_CODE_SUCCESS) { tsdbUnlockRepo(pTsdb); return TSDB_CODE_FAILED; } - if (tsdbInitSmaEnv(pTsdb, aname, &pEnv) != TSDB_CODE_SUCCESS) { + if (tsdbInitSmaEnv(pTsdb, rname, did, &pEnv) != TSDB_CODE_SUCCESS) { tsdbUnlockRepo(pTsdb); return TSDB_CODE_FAILED; } @@ -339,11 +342,6 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg) { return TSDB_CODE_FAILED; } - if (tsdbCheckAndInitSmaEnv(pTsdb, smaType) != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TDB_INIT_FAILED; - return TSDB_CODE_FAILED; - } - // TODO: decode the msg from Stream Computing module => start int64_t indexUid = SMA_TEST_INDEX_UID; const int32_t SMA_TEST_EXPIRED_WINDOW_SIZE = 10; @@ -354,6 +352,11 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg) { } // TODO: decode the msg <= end + if (tsdbCheckAndInitSmaEnv(pTsdb, smaType) != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TDB_INIT_FAILED; + return TSDB_CODE_FAILED; + } + SSmaEnv * pEnv = REPO_SMA_ENV(pTsdb, smaType); SSmaStat *pStat = SMA_ENV_STAT(pEnv); SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv); @@ -660,14 +663,13 @@ static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH) { } } -static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t storageLevel, int32_t fid) { +static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int64_t indexUid, int32_t fid) { STsdb *pTsdb = pSmaH->pTsdb; ASSERT(pSmaH->dFile.path == NULL && pSmaH->dFile.pDB == NULL); pSmaH->dFile.fid = fid; - char tSmaFile[TSDB_FILENAME_LEN] = {0}; - snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.tsma", REPO_ID(pTsdb), fid); + snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, REPO_ID(pTsdb), fid); pSmaH->dFile.path = strdup(tSmaFile); return TSDB_CODE_SUCCESS; @@ -708,8 +710,9 @@ static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLe static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { STsdbCfg * pCfg = REPO_CFG(pTsdb); STSmaDataWrapper *pData = (STSmaDataWrapper *)msg; + SSmaEnv * pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); - if (!atomic_load_ptr(&pTsdb->pTSmaEnv)) { + if (pEnv == NULL) { terrno = TSDB_CODE_INVALID_PTR; tsdbWarn("vgId:%d insert tSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); return terrno; @@ -727,6 +730,17 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { return TSDB_CODE_FAILED; } + int64_t indexUid = SMA_TEST_INDEX_UID; + char rPath[TSDB_FILENAME_LEN] = {0}; + char aPath[TSDB_FILENAME_LEN] = {0}; + snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); + tfsAbsoluteName(REPO_TFS(pTsdb), SMA_ENV_DID(pEnv), rPath, aPath); + if (!taosCheckExistFile(aPath)) { + if (tfsMkdirRecurAt(REPO_TFS(pTsdb), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_FAILED; + } + } + // Step 1: Judge the storage level and days int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit); int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); @@ -735,7 +749,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file // - Set and open the DFile or the B+Tree file // TODO: tsdbStartTSmaCommit(); - tsdbSetTSmaDataFile(&tSmaH, pData, storageLevel, fid); + tsdbSetTSmaDataFile(&tSmaH, pData, indexUid, fid); if (tsdbOpenDBF(pTsdb->pTSmaEnv->dbEnv, &tSmaH.dFile) != 0) { tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb), tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); @@ -822,13 +836,16 @@ static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interv * @brief Init of tSma FS * * @param pReadH + * @param indexUid * @param skey * @return int32_t */ -static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, TSKEY skey) { - int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, REPO_CFG(pSmaH->pTsdb)->precision)); +static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey) { + STsdb *pTsdb = pSmaH->pTsdb; + + int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, REPO_CFG(pTsdb)->precision)); char tSmaFile[TSDB_FILENAME_LEN] = {0}; - snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.tsma", REPO_ID(pSmaH->pTsdb), fid); + snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, REPO_ID(pTsdb), fid); pSmaH->dFile.path = strdup(tSmaFile); pSmaH->smaFsIter.iter = 0; pSmaH->smaFsIter.fid = fid; @@ -887,14 +904,16 @@ static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) { static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySKey, int32_t nMaxResult) { - if (!atomic_load_ptr(&pTsdb->pTSmaEnv)) { + SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); + + if (!pEnv) { terrno = TSDB_CODE_INVALID_PTR; tsdbWarn("vgId:%d getTSmaDataImpl failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); return TSDB_CODE_FAILED; } - tsdbRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv)); - SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pTsdb->pTSmaEnv), &indexUid, sizeof(indexUid)); + tsdbRefSmaStat(pTsdb, SMA_ENV_STAT(pEnv)); + SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); if ((pItem == NULL) || ((pItem = *(SSmaStatItem **)pItem) == NULL)) { // Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if // it's NULL. @@ -926,11 +945,12 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_ tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv)); #endif + STSmaReadH tReadH = {0}; tsdbInitTSmaReadH(&tReadH, pTsdb, interval, intervalUnit); tsdbCloseDBF(&tReadH.dFile); - tsdbInitTSmaFile(&tReadH, querySKey); + tsdbInitTSmaFile(&tReadH, indexUid, querySKey); if (tsdbOpenDBF(SMA_ENV_ENV(pTsdb->pTSmaEnv), &tReadH.dFile) != 0) { tsdbWarn("vgId:%d open DBF %s failed since %s", REPO_ID(pTsdb), tReadH.dFile.path, tstrerror(terrno)); return TSDB_CODE_FAILED; diff --git a/source/dnode/vnode/src/vnd/vnodeBufferPool.c b/source/dnode/vnode/src/vnd/vnodeBufferPool.c index 9d1877bdb7..3b43fd82f6 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufferPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufferPool.c @@ -19,8 +19,8 @@ #define VNODE_BUF_POOL_SHARDS 3 struct SVBufPool { - pthread_mutex_t mutex; - pthread_cond_t hasFree; + TdThreadMutex mutex; + TdThreadCond hasFree; TD_DLIST(SVMemAllocator) free; TD_DLIST(SVMemAllocator) incycle; SVMemAllocator *inuse; diff --git a/source/dnode/vnode/src/vnd/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c index 2a3862c7cb..c7405fdcea 100644 --- a/source/dnode/vnode/src/vnd/vnodeMain.c +++ b/source/dnode/vnode/src/vnd/vnodeMain.c @@ -27,7 +27,7 @@ SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) { SVnodeCfg cfg = defaultVnodeOptions; if (pVnodeCfg != NULL) { cfg.vgId = pVnodeCfg->vgId; - cfg.pDnode = pVnodeCfg->pDnode; + cfg.pWrapper = pVnodeCfg->pWrapper; cfg.pTfs = pVnodeCfg->pTfs; cfg.dbId = pVnodeCfg->dbId; cfg.hashBegin = pVnodeCfg->hashBegin; @@ -79,7 +79,7 @@ static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg) { } pVnode->vgId = pVnodeCfg->vgId; - pVnode->pDnode = pVnodeCfg->pDnode; + pVnode->pWrapper = pVnodeCfg->pWrapper; pVnode->pTfs = pVnodeCfg->pTfs; pVnode->path = strdup(path); vnodeOptionsCopy(&(pVnode->config), pVnodeCfg); diff --git a/source/dnode/vnode/src/vnd/vnodeMgr.c b/source/dnode/vnode/src/vnd/vnodeMgr.c index 477deed8c8..51aaf9e68f 100644 --- a/source/dnode/vnode/src/vnd/vnodeMgr.c +++ b/source/dnode/vnode/src/vnd/vnodeMgr.c @@ -25,23 +25,26 @@ int vnodeInit(const SVnodeOpt *pOption) { } vnodeMgr.stop = false; - vnodeMgr.putReqToVQueryQFp = pOption->putReqToVQueryQFp; - vnodeMgr.sendReqToDnodeFp = pOption->sendReqToDnodeFp; + vnodeMgr.putToQueryQFp = pOption->putToQueryQFp; + vnodeMgr.putToFetchQFp = pOption->putToFetchQFp; + vnodeMgr.sendReqFp = pOption->sendReqFp; + vnodeMgr.sendMnodeReqFp = pOption->sendMnodeReqFp; + vnodeMgr.sendRspFp = pOption->sendRspFp; // Start commit handers if (pOption->nthreads > 0) { vnodeMgr.nthreads = pOption->nthreads; - vnodeMgr.threads = (pthread_t*)calloc(pOption->nthreads, sizeof(pthread_t)); + vnodeMgr.threads = (TdThread*)calloc(pOption->nthreads, sizeof(TdThread)); if (vnodeMgr.threads == NULL) { return -1; } - pthread_mutex_init(&(vnodeMgr.mutex), NULL); - pthread_cond_init(&(vnodeMgr.hasTask), NULL); + taosThreadMutexInit(&(vnodeMgr.mutex), NULL); + taosThreadCondInit(&(vnodeMgr.hasTask), NULL); TD_DLIST_INIT(&(vnodeMgr.queue)); for (uint16_t i = 0; i < pOption->nthreads; i++) { - pthread_create(&(vnodeMgr.threads[i]), NULL, loop, NULL); + taosThreadCreate(&(vnodeMgr.threads[i]), NULL, loop, NULL); // pthread_setname_np(vnodeMgr.threads[i], "VND Commit Thread"); } } else { @@ -63,42 +66,50 @@ void vnodeCleanup() { } // Stop commit handler - pthread_mutex_lock(&(vnodeMgr.mutex)); + taosThreadMutexLock(&(vnodeMgr.mutex)); vnodeMgr.stop = true; - pthread_cond_broadcast(&(vnodeMgr.hasTask)); - pthread_mutex_unlock(&(vnodeMgr.mutex)); + taosThreadCondBroadcast(&(vnodeMgr.hasTask)); + taosThreadMutexUnlock(&(vnodeMgr.mutex)); for (uint16_t i = 0; i < vnodeMgr.nthreads; i++) { - pthread_join(vnodeMgr.threads[i], NULL); + taosThreadJoin(vnodeMgr.threads[i], NULL); } tfree(vnodeMgr.threads); - pthread_cond_destroy(&(vnodeMgr.hasTask)); - pthread_mutex_destroy(&(vnodeMgr.mutex)); + taosThreadCondDestroy(&(vnodeMgr.hasTask)); + taosThreadMutexDestroy(&(vnodeMgr.mutex)); } int vnodeScheduleTask(SVnodeTask* pTask) { - pthread_mutex_lock(&(vnodeMgr.mutex)); + taosThreadMutexLock(&(vnodeMgr.mutex)); TD_DLIST_APPEND(&(vnodeMgr.queue), pTask); - pthread_cond_signal(&(vnodeMgr.hasTask)); + taosThreadCondSignal(&(vnodeMgr.hasTask)); - pthread_mutex_unlock(&(vnodeMgr.mutex)); + taosThreadMutexUnlock(&(vnodeMgr.mutex)); return 0; } -int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq) { - if (pVnode == NULL || pVnode->pDnode == NULL || vnodeMgr.putReqToVQueryQFp == NULL) { - terrno = TSDB_CODE_VND_APP_ERROR; - return -1; - } - return (*vnodeMgr.putReqToVQueryQFp)(pVnode->pDnode, pReq); +int32_t vnodePutToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq) { + return (*vnodeMgr.putToQueryQFp)(pVnode->pWrapper, pReq); } -void vnodeSendReqToDnode(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq) { - (*vnodeMgr.sendReqToDnodeFp)(pVnode->pDnode, epSet, pReq); +int32_t vnodePutToVFetchQ(SVnode* pVnode, struct SRpcMsg* pReq) { + return (*vnodeMgr.putToFetchQFp)(pVnode->pWrapper, pReq); +} + +int32_t vnodeSendReq(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq) { + return (*vnodeMgr.sendReqFp)(pVnode->pWrapper, epSet, pReq); +} + +int32_t vnodeSendMnodeReq(SVnode* pVnode, struct SRpcMsg* pReq) { + return (*vnodeMgr.sendMnodeReqFp)(pVnode->pWrapper, pReq); +} + +void vnodeSendRsp(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pRsp) { + (*vnodeMgr.sendRspFp)(pVnode->pWrapper, pRsp); } /* ------------------------ STATIC METHODS ------------------------ */ @@ -107,15 +118,15 @@ static void* loop(void* arg) { SVnodeTask* pTask; for (;;) { - pthread_mutex_lock(&(vnodeMgr.mutex)); + taosThreadMutexLock(&(vnodeMgr.mutex)); for (;;) { pTask = TD_DLIST_HEAD(&(vnodeMgr.queue)); if (pTask == NULL) { if (vnodeMgr.stop) { - pthread_mutex_unlock(&(vnodeMgr.mutex)); + taosThreadMutexUnlock(&(vnodeMgr.mutex)); return NULL; } else { - pthread_cond_wait(&(vnodeMgr.hasTask), &(vnodeMgr.mutex)); + taosThreadCondWait(&(vnodeMgr.hasTask), &(vnodeMgr.mutex)); } } else { TD_DLIST_POP(&(vnodeMgr.queue), pTask); @@ -123,7 +134,7 @@ static void* loop(void* arg) { } } - pthread_mutex_unlock(&(vnodeMgr.mutex)); + taosThreadMutexUnlock(&(vnodeMgr.mutex)); (*(pTask->execute))(pTask->arg); free(pTask); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index e8bc6873ab..3132c675e9 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -21,12 +21,10 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg); int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, (void **)&pVnode->pQuery, pVnode, - (putReqToQueryQFp)vnodePutReqToVQueryQ, (sendReqToDnodeFp)vnodeSendReqToDnode); + (putReqToQueryQFp)vnodePutToVQueryQ, (sendReqFp)vnodeSendReq); } -void vnodeQueryClose(SVnode *pVnode) { - qWorkerDestroy((void **)&pVnode->pQuery); -} +void vnodeQueryClose(SVnode *pVnode) { qWorkerDestroy((void **)&pVnode->pQuery); } int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { vTrace("message in query queue is processing"); @@ -68,6 +66,8 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg) { return vnodeGetTableMeta(pVnode, pMsg); case TDMT_VND_CONSUME: return tqProcessPollReq(pVnode->pTq, pMsg); + case TDMT_VND_TASK_EXEC: + return tqProcessTaskExec(pVnode->pTq, pMsg); case TDMT_VND_QUERY_HEARTBEAT: return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg); default: diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 8285020e14..d3769b8a30 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -16,18 +16,20 @@ #include "tq.h" #include "vnd.h" -int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { - SRpcMsg *pMsg; +void vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { + SNodeMsg *pMsg; + SRpcMsg *pRpc; for (int i = 0; i < taosArrayGetSize(pMsgs); i++) { - pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); + pMsg = *(SNodeMsg **)taosArrayGet(pMsgs, i); + pRpc = &pMsg->rpcMsg; // set request version - void *pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + void *pBuf = POINTER_SHIFT(pRpc->pCont, sizeof(SMsgHead)); int64_t ver = pVnode->state.processed++; taosEncodeFixedI64(&pBuf, ver); - if (walWrite(pVnode->pWal, ver, pMsg->msgType, pMsg->pCont, pMsg->contLen) < 0) { + if (walWrite(pVnode->pWal, ver, pRpc->msgType, pRpc->pCont, pRpc->contLen) < 0) { // TODO: handle error /*ASSERT(false);*/ vError("vnode:%d write wal error since %s", pVnode->vgId, terrstr()); @@ -38,7 +40,8 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { // TODO: Integrate RAFT module here - return 0; + // No results are returned because error handling is difficult + // return 0; } int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 5a87c180b6..8140113e67 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -14,13 +14,13 @@ */ #include +#include #include #include #include #include #include -#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -33,6 +33,7 @@ int main(int argc, char **argv) { return RUN_ALL_TESTS(); } +#if 1 TEST(testCase, tSma_Meta_Encode_Decode_Test) { // encode STSma tSma = {0}; @@ -88,6 +89,7 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) { tdDestroyTSma(&tSma); tdDestroyTSmaWrapper(&dstTSmaWrapper); } +#endif #if 1 TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index d766f3cbf1..ea8195bfd1 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -1,6 +1,6 @@ add_subdirectory(transport) add_subdirectory(sync) -add_subdirectory(tdb) +# add_subdirectory(tdb) add_subdirectory(index) add_subdirectory(wal) add_subdirectory(parser) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index c4f1a117fe..60c5886250 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -176,7 +176,7 @@ typedef struct SCatalogMgmt { SCtgQNode *tail; tsem_t sem; uint64_t qRemainNum; - pthread_t updateThread; + TdThread updateThread; SHashObj *pCluster; //key: clusterId, value: SCatalog* SCatalogStat stat; SCatalogCfg cfg; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 08e0ac55d7..fbb2903a8e 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -2070,16 +2070,16 @@ void* ctgUpdateThreadFunc(void* param) { int32_t ctgStartUpdateThread() { - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&gCtgMgmt.updateThread, &thAttr, ctgUpdateThreadFunc, NULL) != 0) { + if (taosThreadCreate(&gCtgMgmt.updateThread, &thAttr, ctgUpdateThreadFunc, NULL) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); CTG_ERR_RET(terrno); } - pthread_attr_destroy(&thAttr); + taosThreadAttrDestroy(&thAttr); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index cc0e5bb1a9..4aa18c3bab 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -2094,14 +2094,14 @@ TEST(multiThread, getSetRmSameDbVgroup) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t thread1, thread2; - pthread_create(&(thread1), &thattr, ctgTestSetSameDbVgroupThread, pCtg); + TdThread thread1, thread2; + taosThreadCreate(&(thread1), &thattr, ctgTestSetSameDbVgroupThread, pCtg); taosSsleep(1); - pthread_create(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg); + taosThreadCreate(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg); while (true) { if (ctgTestDeadLoop) { @@ -2146,14 +2146,14 @@ TEST(multiThread, getSetRmDiffDbVgroup) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t thread1, thread2; - pthread_create(&(thread1), &thattr, ctgTestSetDiffDbVgroupThread, pCtg); + TdThread thread1, thread2; + taosThreadCreate(&(thread1), &thattr, ctgTestSetDiffDbVgroupThread, pCtg); taosSsleep(1); - pthread_create(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg); + taosThreadCreate(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg); while (true) { if (ctgTestDeadLoop) { @@ -2198,13 +2198,13 @@ TEST(multiThread, ctableMeta) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t thread1, thread2; - pthread_create(&(thread1), &thattr, ctgTestSetCtableMetaThread, pCtg); + TdThread thread1, thread2; + taosThreadCreate(&(thread1), &thattr, ctgTestSetCtableMetaThread, pCtg); taosSsleep(1); - pthread_create(&(thread1), &thattr, ctgTestGetCtableMetaThread, pCtg); + taosThreadCreate(&(thread1), &thattr, ctgTestGetCtableMetaThread, pCtg); while (true) { if (ctgTestDeadLoop) { diff --git a/source/libs/executor/inc/dataSinkInt.h b/source/libs/executor/inc/dataSinkInt.h index 8acb6f7e8d..85356a862c 100644 --- a/source/libs/executor/inc/dataSinkInt.h +++ b/source/libs/executor/inc/dataSinkInt.h @@ -29,7 +29,7 @@ struct SDataSinkHandle; typedef struct SDataSinkManager { SDataSinkMgtCfg cfg; - pthread_mutex_t mutex; + TdThreadMutex mutex; } SDataSinkManager; typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 833ac13226..0399a78b09 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -356,7 +356,7 @@ typedef struct SQInfo { STaskAttr query; void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; - pthread_mutex_t lock; // used to synchronize the rsp/query threads + TdThreadMutex lock; // used to synchronize the rsp/query threads tsem_t ready; int32_t dataReady; // denote if query result is ready or not void* rspContext; // response context diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index a0ee048d82..6ad457a91b 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -43,7 +43,7 @@ typedef struct SDataDispatchHandle { int32_t status; bool queryEnd; uint64_t useconds; - pthread_mutex_t mutex; + TdThreadMutex mutex; } SDataDispatchHandle; static bool needCompress(const SSDataBlock* pData, const SDataBlockDescNode* pSchema) { @@ -126,19 +126,19 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, } static int32_t updateStatus(SDataDispatchHandle* pDispatcher) { - pthread_mutex_lock(&pDispatcher->mutex); + taosThreadMutexLock(&pDispatcher->mutex); int32_t blockNums = taosQueueSize(pDispatcher->pDataBlocks); int32_t status = (0 == blockNums ? DS_BUF_EMPTY : (blockNums < pDispatcher->pManager->cfg.maxDataBlockNumPerQuery ? DS_BUF_LOW : DS_BUF_FULL)); pDispatcher->status = status; - pthread_mutex_unlock(&pDispatcher->mutex); + taosThreadMutexUnlock(&pDispatcher->mutex); return status; } static int32_t getStatus(SDataDispatchHandle* pDispatcher) { - pthread_mutex_lock(&pDispatcher->mutex); + taosThreadMutexLock(&pDispatcher->mutex); int32_t status = pDispatcher->status; - pthread_mutex_unlock(&pDispatcher->mutex); + taosThreadMutexUnlock(&pDispatcher->mutex); return status; } @@ -156,10 +156,10 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; - pthread_mutex_lock(&pDispatcher->mutex); + taosThreadMutexLock(&pDispatcher->mutex); pDispatcher->queryEnd = true; pDispatcher->useconds = useconds; - pthread_mutex_unlock(&pDispatcher->mutex); + taosThreadMutexUnlock(&pDispatcher->mutex); } static void getDataLength(SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryEnd) { @@ -191,11 +191,11 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->compressed = pEntry->compressed; tfree(pDispatcher->nextOutput.pData); // todo persistent pOutput->bufStatus = updateStatus(pDispatcher); - pthread_mutex_lock(&pDispatcher->mutex); + taosThreadMutexLock(&pDispatcher->mutex); pOutput->queryEnd = pDispatcher->queryEnd; pOutput->useconds = pDispatcher->useconds; pOutput->precision = pDispatcher->pSchema->precision; - pthread_mutex_unlock(&pDispatcher->mutex); + taosThreadMutexUnlock(&pDispatcher->mutex); return TSDB_CODE_SUCCESS; } @@ -209,7 +209,7 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { taosFreeQitem(pBuf); } taosCloseQueue(pDispatcher->pDataBlocks); - pthread_mutex_destroy(&pDispatcher->mutex); + taosThreadMutexDestroy(&pDispatcher->mutex); } int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle) { @@ -228,7 +228,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD dispatcher->status = DS_BUF_EMPTY; dispatcher->queryEnd = false; dispatcher->pDataBlocks = taosOpenQueue(); - pthread_mutex_init(&dispatcher->mutex, NULL); + taosThreadMutexInit(&dispatcher->mutex, NULL); if (NULL == dispatcher->pDataBlocks) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return TSDB_CODE_QRY_OUT_OF_MEMORY; diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index 4e8583eb2a..997397314f 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -22,7 +22,7 @@ static SDataSinkManager gDataSinkManager = {0}; int32_t dsDataSinkMgtInit(SDataSinkMgtCfg *cfg) { gDataSinkManager.cfg = *cfg; - pthread_mutex_init(&gDataSinkManager.mutex, NULL); + taosThreadMutexInit(&gDataSinkManager.mutex, NULL); return 0; // to avoid compiler eror } diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 7e55a4b3e1..cc9921ce73 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -30,7 +30,7 @@ #include "query.h" typedef struct STaskMgmt { - pthread_mutex_t lock; + TdThreadMutex lock; SCacheObj *qinfoPool; // query handle pool int32_t vgId; bool closed; diff --git a/source/libs/function/inc/tscript.h b/source/libs/function/inc/tscript.h index 281fe6f679..823e2d61f2 100644 --- a/source/libs/function/inc/tscript.h +++ b/source/libs/function/inc/tscript.h @@ -70,7 +70,7 @@ typedef struct { SList *scriptEnvs; // int32_t mSize; // pool limit int32_t cSize; // current available size - pthread_mutex_t mutex; + TdThreadMutex mutex; } ScriptEnvPool; ScriptCtx* createScriptCtx(char *str, int8_t resType, int16_t resBytes); diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 41b0126a07..f9f2e90770 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -26,7 +26,7 @@ typedef struct SFuncMgtService { } SFuncMgtService; static SFuncMgtService gFunMgtService; -static pthread_once_t functionHashTableInit = PTHREAD_ONCE_INIT; +static TdThreadOnce functionHashTableInit = PTHREAD_ONCE_INIT; static int32_t initFunctionCode = 0; static void doInitFunctionHashTable() { @@ -45,7 +45,7 @@ static void doInitFunctionHashTable() { } int32_t fmFuncMgtInit() { - pthread_once(&functionHashTableInit, doInitFunctionHashTable); + taosThreadOnce(&functionHashTableInit, doInitFunctionHashTable); return initFunctionCode; } diff --git a/source/libs/function/src/tscript.c b/source/libs/function/src/tscript.c index 7c07b0b783..25cf3cfbb1 100644 --- a/source/libs/function/src/tscript.c +++ b/source/libs/function/src/tscript.c @@ -333,7 +333,7 @@ void destroyLuaEnv(lua_State *lua) { int32_t scriptEnvPoolInit() { const int size = 10; // configure or not pool = malloc(sizeof(ScriptEnvPool)); - pthread_mutex_init(&pool->mutex, NULL); + taosThreadMutexInit(&pool->mutex, NULL); pool->scriptEnvs = tdListNew(sizeof(ScriptEnv *)); for (int i = 0; i < size; i++) { @@ -359,7 +359,7 @@ void scriptEnvPoolCleanup() { listNodeFree(pNode); } tdListFree(pool->scriptEnvs); - pthread_mutex_destroy(&pool->mutex); + taosThreadMutexDestroy(&pool->mutex); free(pool); } @@ -372,9 +372,9 @@ void destroyScriptEnv(ScriptEnv *pEnv) { ScriptEnv* getScriptEnvFromPool() { ScriptEnv *pEnv = NULL; - pthread_mutex_lock(&pool->mutex); + taosThreadMutexLock(&pool->mutex); if (pool->cSize <= 0) { - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); return NULL; } SListNode *pNode = tdListPopHead(pool->scriptEnvs); @@ -384,7 +384,7 @@ ScriptEnv* getScriptEnvFromPool() { } pool->cSize--; - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); return pEnv; } @@ -392,11 +392,11 @@ void addScriptEnvToPool(ScriptEnv *pEnv) { if (pEnv == NULL) { return; } - pthread_mutex_lock(&pool->mutex); + taosThreadMutexLock(&pool->mutex); lua_settop(pEnv->lua_state, 0); tdListAppend(pool->scriptEnvs, (void *)(&pEnv)); pool->cSize++; - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); } bool hasBaseFuncDefinedInScript(lua_State *lua, const char *funcPrefix, int32_t len) { diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 3d1d5356c2..37318767c7 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -58,7 +58,7 @@ struct SIndex { char* path; SIndexStat stat; - pthread_mutex_t mtx; + TdThreadMutex mtx; }; struct SIndexOpts { diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h index a6ebcd6d6f..086e75d99f 100644 --- a/source/libs/index/inc/index_cache.h +++ b/source/libs/index/inc/index_cache.h @@ -43,8 +43,8 @@ typedef struct IndexCache { int8_t type; uint64_t suid; - pthread_mutex_t mtx; - pthread_cond_t finished; + TdThreadMutex mtx; + TdThreadCond finished; } IndexCache; #define CACHE_VERSION(cache) atomic_load_32(&cache->version) diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index 5da0dc537b..cf5c3f306b 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -260,7 +260,7 @@ typedef struct Fst { FstMeta* meta; FstSlice* data; // FstNode* root; // - pthread_mutex_t mtx; + TdThreadMutex mtx; } Fst; // refactor simple function diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index d25fed6816..83410306eb 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -45,7 +45,7 @@ typedef struct SIdxColInfo { int cVersion; } SIdxColInfo; -static pthread_once_t isInit = PTHREAD_ONCE_INIT; +static TdThreadOnce isInit = PTHREAD_ONCE_INIT; // static void indexInit(); static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result); @@ -61,7 +61,7 @@ static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, Iterat // int32_t indexSerialKey(ICacheKey* key, char* buf); int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { - pthread_once(&isInit, indexInit); + taosThreadOnce(&isInit, indexInit); SIndex* sIdx = calloc(1, sizeof(SIndex)); if (sIdx == NULL) { return -1; @@ -82,7 +82,7 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); sIdx->cVersion = 1; sIdx->path = tstrdup(path); - pthread_mutex_init(&sIdx->mtx, NULL); + taosThreadMutexInit(&sIdx->mtx, NULL); *index = sIdx; return 0; #endif @@ -112,7 +112,7 @@ void indexClose(SIndex* sIdx) { iter = taosHashIterate(sIdx->colObj, iter); } taosHashCleanup(sIdx->colObj); - pthread_mutex_destroy(&sIdx->mtx); + taosThreadMutexDestroy(&sIdx->mtx); indexTFileDestroy(sIdx->tindex); #endif free(sIdx->path); @@ -140,7 +140,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { #ifdef USE_INVERTED_INDEX // TODO(yihao): reduce the lock range - pthread_mutex_lock(&index->mtx); + taosThreadMutexLock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); @@ -154,7 +154,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { taosHashPut(index->colObj, buf, sz, &pCache, sizeof(void*)); } } - pthread_mutex_unlock(&index->mtx); + taosThreadMutexUnlock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); @@ -333,10 +333,10 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result .suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName), .colType = term->colType}; int32_t sz = indexSerialCacheKey(&key, buf); - pthread_mutex_lock(&sIdx->mtx); + taosThreadMutexLock(&sIdx->mtx); IndexCache** pCache = taosHashGet(sIdx->colObj, buf, sz); cache = (pCache == NULL) ? NULL : *pCache; - pthread_mutex_unlock(&sIdx->mtx); + taosThreadMutexUnlock(&sIdx->mtx); *result = taosArrayInit(4, sizeof(uint64_t)); // TODO: iterator mem and tidex @@ -564,10 +564,10 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { TFileHeader* header = &reader->header; ICacheKey key = {.suid = cache->suid, .colName = header->colName, .nColName = strlen(header->colName)}; - pthread_mutex_lock(&sIdx->mtx); + taosThreadMutexLock(&sIdx->mtx); IndexTFile* ifile = (IndexTFile*)sIdx->tindex; tfileCachePut(ifile->cache, &key, reader); - pthread_mutex_unlock(&sIdx->mtx); + taosThreadMutexUnlock(&sIdx->mtx); return ret; END: if (tw != NULL) { diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 34f009dd7e..2b4327b091 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -54,8 +54,8 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in cache->suid = suid; cache->occupiedMem = 0; - pthread_mutex_init(&cache->mtx, NULL); - pthread_cond_init(&cache->finished, NULL); + taosThreadMutexInit(&cache->mtx, NULL); + taosThreadCondInit(&cache->finished, NULL); indexCacheRef(cache); return cache; @@ -63,10 +63,10 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in void indexCacheDebug(IndexCache* cache) { MemTable* tbl = NULL; - pthread_mutex_lock(&cache->mtx); + taosThreadMutexLock(&cache->mtx); tbl = cache->mem; indexMemRef(tbl); - pthread_mutex_unlock(&cache->mtx); + taosThreadMutexUnlock(&cache->mtx); { SSkipList* slt = tbl->mem; @@ -85,10 +85,10 @@ void indexCacheDebug(IndexCache* cache) { } { - pthread_mutex_lock(&cache->mtx); + taosThreadMutexLock(&cache->mtx); tbl = cache->imm; indexMemRef(tbl); - pthread_mutex_unlock(&cache->mtx); + taosThreadMutexUnlock(&cache->mtx); if (tbl != NULL) { SSkipList* slt = tbl->mem; SSkipListIterator* iter = tSkipListCreateIter(slt); @@ -126,13 +126,13 @@ void indexCacheDestroyImm(IndexCache* cache) { } MemTable* tbl = NULL; - pthread_mutex_lock(&cache->mtx); + taosThreadMutexLock(&cache->mtx); tbl = cache->imm; cache->imm = NULL; // or throw int bg thread - pthread_cond_broadcast(&cache->finished); + taosThreadCondBroadcast(&cache->finished); - pthread_mutex_unlock(&cache->mtx); + taosThreadMutexUnlock(&cache->mtx); indexMemUnRef(tbl); indexMemUnRef(tbl); @@ -146,8 +146,8 @@ void indexCacheDestroy(void* cache) { indexMemUnRef(pCache->imm); free(pCache->colName); - pthread_mutex_destroy(&pCache->mtx); - pthread_cond_destroy(&pCache->finished); + taosThreadMutexDestroy(&pCache->mtx); + taosThreadCondDestroy(&pCache->finished); free(pCache); } @@ -158,7 +158,7 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) { return NULL; } - pthread_mutex_lock(&cache->mtx); + taosThreadMutexLock(&cache->mtx); indexMemRef(cache->imm); @@ -169,7 +169,7 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) { iiter->next = indexCacheIteratorNext; iiter->getValue = indexCacheIteratorGetValue; - pthread_mutex_unlock(&cache->mtx); + taosThreadMutexUnlock(&cache->mtx); return iiter; } @@ -200,7 +200,7 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { break; } else if (cache->imm != NULL) { // TODO: wake up by condition variable - pthread_cond_wait(&cache->finished, &cache->mtx); + taosThreadCondWait(&cache->finished, &cache->mtx); } else { indexCacheRef(cache); cache->imm = cache->mem; @@ -240,7 +240,7 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { // ugly code, refactor later int64_t estimate = sizeof(ct) + strlen(ct->colVal); - pthread_mutex_lock(&pCache->mtx); + taosThreadMutexLock(&pCache->mtx); pCache->occupiedMem += estimate; indexCacheMakeRoomForWrite(pCache); MemTable* tbl = pCache->mem; @@ -248,7 +248,7 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { tSkipListPut(tbl->mem, (char*)ct); indexMemUnRef(tbl); - pthread_mutex_unlock(&pCache->mtx); + taosThreadMutexUnlock(&pCache->mtx); indexCacheUnRef(pCache); return 0; @@ -299,12 +299,12 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result IndexCache* pCache = cache; MemTable *mem = NULL, *imm = NULL; - pthread_mutex_lock(&pCache->mtx); + taosThreadMutexLock(&pCache->mtx); mem = pCache->mem; imm = pCache->imm; indexMemRef(mem); indexMemRef(imm); - pthread_mutex_unlock(&pCache->mtx); + taosThreadMutexUnlock(&pCache->mtx); SIndexTerm* term = query->term; EIndexQueryType qtype = query->qType; diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 18cde151d2..58d5a871cd 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -1023,7 +1023,7 @@ Fst* fstCreate(FstSlice* slice) { *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice) - 1); fst->data = s; - pthread_mutex_init(&fst->mtx, NULL); + taosThreadMutexInit(&fst->mtx, NULL); return fst; FST_CREAT_FAILED: @@ -1037,14 +1037,14 @@ void fstDestroy(Fst* fst) { free(fst->meta); fstSliceDestroy(fst->data); free(fst->data); - pthread_mutex_destroy(&fst->mtx); + taosThreadMutexDestroy(&fst->mtx); } free(fst); } bool fstGet(Fst* fst, FstSlice* b, Output* out) { // dec lock range - // pthread_mutex_lock(&fst->mtx); + // taosThreadMutexLock(&fst->mtx); FstNode* root = fstGetRoot(fst); Output tOut = 0; int32_t len; @@ -1057,7 +1057,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { uint8_t inp = data[i]; Output res = 0; if (false == fstNodeFindInput(root, inp, &res)) { - // pthread_mutex_unlock(&fst->mtx); + // taosThreadMutexUnlock(&fst->mtx); return false; } @@ -1068,7 +1068,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { taosArrayPush(nodes, &root); } if (!FST_NODE_IS_FINAL(root)) { - // pthread_mutex_unlock(&fst->mtx); + // taosThreadMutexUnlock(&fst->mtx); return false; } else { tOut = tOut + FST_NODE_FINAL_OUTPUT(root); @@ -1080,7 +1080,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { } taosArrayDestroy(nodes); // fst->root = NULL; - // pthread_mutex_unlock(&fst->mtx); + // taosThreadMutexUnlock(&fst->mtx); *out = tOut; return true; } diff --git a/source/libs/monitor/inc/monInt.h b/source/libs/monitor/inc/monInt.h index 6ef901410b..452c38f66b 100644 --- a/source/libs/monitor/inc/monInt.h +++ b/source/libs/monitor/inc/monInt.h @@ -48,7 +48,7 @@ typedef struct SMonInfo { } SMonInfo; typedef struct { - pthread_mutex_t lock; + TdThreadMutex lock; SArray *logs; // array of SMonLogItem int32_t maxLogs; const char *server; diff --git a/source/libs/monitor/src/monitor.c b/source/libs/monitor/src/monitor.c index 352b59e931..aa5eafd8b2 100644 --- a/source/libs/monitor/src/monitor.c +++ b/source/libs/monitor/src/monitor.c @@ -23,7 +23,7 @@ static SMonitor tsMonitor = {0}; void monRecordLog(int64_t ts, ELogLevel level, const char *content) { - pthread_mutex_lock(&tsMonitor.lock); + taosThreadMutexLock(&tsMonitor.lock); int32_t size = taosArrayGetSize(tsMonitor.logs); if (size < tsMonitor.maxLogs) { SMonLogItem item = {.ts = ts, .level = level}; @@ -32,7 +32,7 @@ void monRecordLog(int64_t ts, ELogLevel level, const char *content) { tstrncpy(pItem->content, content, MON_LOG_LEN); } } - pthread_mutex_unlock(&tsMonitor.lock); + taosThreadMutexUnlock(&tsMonitor.lock); } int32_t monInit(const SMonCfg *pCfg) { @@ -48,7 +48,7 @@ int32_t monInit(const SMonCfg *pCfg) { tsMonitor.comp = pCfg->comp; tsLogFp = monRecordLog; tsMonitor.state.time = taosGetTimestampMs(); - pthread_mutex_init(&tsMonitor.lock, NULL); + taosThreadMutexInit(&tsMonitor.lock, NULL); return 0; } @@ -56,7 +56,7 @@ void monCleanup() { tsLogFp = NULL; taosArrayDestroy(tsMonitor.logs); tsMonitor.logs = NULL; - pthread_mutex_destroy(&tsMonitor.lock); + taosThreadMutexDestroy(&tsMonitor.lock); } SMonInfo *monCreateMonitorInfo() { @@ -66,10 +66,10 @@ SMonInfo *monCreateMonitorInfo() { return NULL; } - pthread_mutex_lock(&tsMonitor.lock); + taosThreadMutexLock(&tsMonitor.lock); pMonitor->logs = taosArrayDup(tsMonitor.logs); taosArrayClear(tsMonitor.logs); - pthread_mutex_unlock(&tsMonitor.lock); + taosThreadMutexUnlock(&tsMonitor.lock); pMonitor->pJson = tjsonCreateObject(); if (pMonitor->pJson == NULL || pMonitor->logs == NULL) { diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 34c211561b..d66203eb40 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -190,7 +190,6 @@ static SNode* fillNodeCopy(const SFillNode* pSrc, SFillNode* pDst) { } static SNode* logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) { - COPY_SCALAR_FIELD(id); CLONE_NODE_LIST_FIELD(pTargets); CLONE_NODE_FIELD(pConditions); CLONE_NODE_LIST_FIELD(pChildren); @@ -198,7 +197,7 @@ static SNode* logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) { } static STableMeta* tableMetaClone(const STableMeta* pSrc) { - int32_t len = sizeof(STableMeta) + (pSrc->tableInfo.numOfTags + pSrc->tableInfo.numOfColumns) * sizeof(SSchema); + int32_t len = TABLE_META_SIZE(pSrc); STableMeta* pDst = malloc(len); if (NULL == pDst) { return NULL; @@ -208,7 +207,7 @@ static STableMeta* tableMetaClone(const STableMeta* pSrc) { } static SVgroupsInfo* vgroupsInfoClone(const SVgroupsInfo* pSrc) { - int32_t len = sizeof(SVgroupsInfo) + pSrc->numOfVgroups * sizeof(SVgroupInfo); + int32_t len = VGROUPS_INFO_SIZE(pSrc); SVgroupsInfo* pDst = malloc(len); if (NULL == pDst) { return NULL; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index a2fd32d238..9b9eaaa6a9 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "cmdnodes.h" #include "nodesUtil.h" #include "plannodes.h" #include "querynodes.h" @@ -85,6 +86,8 @@ const char* nodesNodeName(ENodeType type) { return "ShowDatabaseStmt"; case QUERY_NODE_SHOW_TABLES_STMT: return "ShowTablesStmt"; + case QUERY_NODE_CREATE_TOPIC_STMT: + return "CreateTopicStmt"; case QUERY_NODE_LOGIC_PLAN_SCAN: return "LogicScan"; case QUERY_NODE_LOGIC_PLAN_JOIN: @@ -179,21 +182,151 @@ static int32_t jsonToNodeList(const SJson* pJson, const char* pName, SNodeList** return jsonToNodeListImpl(tjsonGetObjectItem(pJson, pName), pList); } -static const char* jkTableMetaUid = "TableMetaUid"; -static const char* jkTableMetaSuid = "TableMetaSuid"; +static const char* jkTableComInfoNumOfTags = "NumOfTags"; +static const char* jkTableComInfoPrecision = "Precision"; +static const char* jkTableComInfoNumOfColumns = "NumOfColumns"; +static const char* jkTableComInfoRowSize = "RowSize"; + +static int32_t tableComInfoToJson(const void* pObj, SJson* pJson) { + const STableComInfo* pNode = (const STableComInfo*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkTableComInfoNumOfTags, pNode->numOfTags); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableComInfoPrecision, pNode->precision); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableComInfoNumOfColumns, pNode->numOfColumns); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableComInfoRowSize, pNode->rowSize); + } + + return code; +} + +static int32_t jsonToTableComInfo(const SJson* pJson, void* pObj) { + STableComInfo* pNode = (STableComInfo*)pObj; + + int32_t code = tjsonGetNumberValue(pJson, jkTableComInfoNumOfTags, pNode->numOfTags); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableComInfoPrecision, pNode->precision); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableComInfoNumOfColumns, pNode->numOfColumns); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableComInfoRowSize, pNode->rowSize); + } + + return code; +} + +static const char* jkSchemaType = "Type"; +static const char* jkSchemaColId = "ColId"; +static const char* jkSchemaBytes = "bytes"; +static const char* jkSchemaName = "Name"; + +static int32_t schemaToJson(const void* pObj, SJson* pJson) { + const SSchema* pNode = (const SSchema*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkSchemaType, pNode->type); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkSchemaColId, pNode->colId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkSchemaBytes, pNode->bytes); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkSchemaName, pNode->name); + } + + return code; +} + +static int32_t jsonToSchema(const SJson* pJson, void* pObj) { + SSchema* pNode = (SSchema*)pObj; + + int32_t code = tjsonGetNumberValue(pJson, jkSchemaType, pNode->type); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkSchemaColId, pNode->colId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkSchemaBytes, pNode->bytes); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkSchemaName, pNode->name); + } + + return code; +} + +static const char* jkTableMetaVgId = "VgId"; +static const char* jkTableMetaTableType = "TableType"; +static const char* jkTableMetaUid = "Uid"; +static const char* jkTableMetaSuid = "Suid"; +static const char* jkTableMetaSversion = "Sversion"; +static const char* jkTableMetaTversion = "Tversion"; +static const char* jkTableMetaComInfo = "ComInfo"; +static const char* jkTableMetaColSchemas = "ColSchemas"; static int32_t tableMetaToJson(const void* pObj, SJson* pJson) { const STableMeta* pNode = (const STableMeta*)pObj; - int32_t code = tjsonAddIntegerToObject(pJson, jkTableMetaUid, pNode->uid); + int32_t code = tjsonAddIntegerToObject(pJson, jkTableMetaVgId, pNode->vgId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableMetaTableType, pNode->tableType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableMetaUid, pNode->uid); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkTableMetaSuid, pNode->suid); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableMetaSversion, pNode->sversion); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableMetaTversion, pNode->tversion); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkTableMetaComInfo, tableComInfoToJson, &pNode->tableInfo); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddArray(pJson, jkTableMetaColSchemas, schemaToJson, pNode->schema, sizeof(SSchema), TABLE_TOTAL_COL_NUM(pNode)); + } + + return code; +} + +static int32_t jsonToTableMeta(const SJson* pJson, void* pObj) { + STableMeta* pNode = (STableMeta*)pObj; + + int32_t code = tjsonGetNumberValue(pJson, jkTableMetaVgId, pNode->vgId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableMetaTableType, pNode->tableType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableMetaUid, pNode->uid); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableMetaSuid, pNode->suid); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableMetaSversion, pNode->sversion); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableMetaTversion, pNode->tversion); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToObject(pJson, jkTableMetaComInfo, jsonToTableComInfo, &pNode->tableInfo); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToArray(pJson, jkTableMetaColSchemas, jsonToSchema, pNode->schema, sizeof(SSchema)); + } return code; } -static const char* jkLogicPlanId = "Id"; static const char* jkLogicPlanTargets = "Targets"; static const char* jkLogicPlanConditions = "Conditions"; static const char* jkLogicPlanChildren = "Children"; @@ -201,10 +334,7 @@ static const char* jkLogicPlanChildren = "Children"; static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) { const SLogicNode* pNode = (const SLogicNode*)pObj; - int32_t code = tjsonAddIntegerToObject(pJson, jkLogicPlanId, pNode->id); - if (TSDB_CODE_SUCCESS == code) { - code = nodeListToJson(pJson, jkLogicPlanTargets, pNode->pTargets); - } + int32_t code = nodeListToJson(pJson, jkLogicPlanTargets, pNode->pTargets); if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkLogicPlanConditions, nodeToJson, pNode->pConditions); } @@ -215,7 +345,22 @@ static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) { return code; } +static int32_t jsonToLogicPlanNode(const SJson* pJson, void* pObj) { + SLogicNode* pNode = (SLogicNode*)pObj; + + int32_t code = jsonToNodeList(pJson, jkLogicPlanTargets, &pNode->pTargets); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkLogicPlanConditions, &pNode->pConditions); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkLogicPlanChildren, &pNode->pChildren); + } + + return code; +} + static const char* jkScanLogicPlanScanCols = "ScanCols"; +static const char* jkScanLogicPlanTableMetaSize = "TableMetaSize"; static const char* jkScanLogicPlanTableMeta = "TableMeta"; static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { @@ -225,6 +370,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkScanLogicPlanScanCols, pNode->pScanCols); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableMetaSize, TABLE_META_SIZE(pNode->pMeta)); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkScanLogicPlanTableMeta, tableMetaToJson, pNode->pMeta); } @@ -232,6 +380,24 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { return code; } +static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { + SScanLogicNode* pNode = (SScanLogicNode*)pObj; + + int32_t objSize = 0; + int32_t code = jsonToLogicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkScanLogicPlanScanCols, &pNode->pScanCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkScanLogicPlanTableMetaSize, &objSize); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonMakeObject(pJson, jkScanLogicPlanTableMeta, jsonToTableMeta, (void**)&pNode->pMeta, objSize); + } + + return code; +} + static const char* jkProjectLogicPlanProjections = "Projections"; static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) { @@ -245,6 +411,17 @@ static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) { return code; } +static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) { + SProjectLogicNode* pNode = (SProjectLogicNode*)pObj; + + int32_t code = jsonToLogicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkProjectLogicPlanProjections, &pNode->pProjections); + } + + return code; +} + static const char* jkJoinLogicPlanJoinType = "JoinType"; static const char* jkJoinLogicPlanOnConditions = "OnConditions"; @@ -445,6 +622,14 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { return code; } +static int32_t physiStreamScanNodeToJson(const void* pObj, SJson* pJson) { + return physiScanNodeToJson(pObj, pJson); +} + +static int32_t jsonToPhysiStreamScanNode(const SJson* pJson, void* pObj) { + return jsonToPhysiScanNode(pJson, pObj); +} + static const char* jkProjectPhysiPlanProjections = "Projections"; static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) { @@ -1273,6 +1458,193 @@ static int32_t jsonToFunctionNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkTableDbName = "DbName"; +static const char* jkTableTableName = "tableName"; +static const char* jkTableTableAlias = "tableAlias"; + +static int32_t tableNodeToJson(const void* pObj, SJson* pJson) { + const STableNode* pNode = (const STableNode*)pObj; + + int32_t code = exprNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkTableDbName, pNode->dbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkTableTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkTableTableAlias, pNode->tableAlias); + } + + return code; +} + +static int32_t jsonToTableNode(const SJson* pJson, void* pObj) { + STableNode* pNode = (STableNode*)pObj; + + int32_t code = jsonToExprNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkTableDbName, pNode->dbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkTableTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkTableTableAlias, pNode->tableAlias); + } + + return code; +} + +static const char* jkEpSetInUse = "InUse"; +static const char* jkEpSetNumOfEps = "NumOfEps"; +static const char* jkEpSetEps = "Eps"; + +static int32_t epSetToJson(const void* pObj, SJson* pJson) { + const SEpSet* pNode = (const SEpSet*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkEpSetInUse, pNode->inUse); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkEpSetNumOfEps, pNode->numOfEps); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddArray(pJson, jkEpSetEps, epToJson, pNode->eps, sizeof(SEp), pNode->numOfEps); + } + + return code; +} + +static int32_t jsonToEpSet(const SJson* pJson, void* pObj) { + SEpSet* pNode = (SEpSet*)pObj; + + int32_t code = tjsonGetTinyIntValue(pJson, jkEpSetInUse, &pNode->inUse); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkEpSetNumOfEps, &pNode->numOfEps); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToArray(pJson, jkEpSetEps, jsonToEp, pNode->eps, sizeof(SEp)); + } + + return code; +} + +static const char* jkVgroupInfoVgId = "VgId"; +static const char* jkVgroupInfoHashBegin = "HashBegin"; +static const char* jkVgroupInfoHashEnd = "HashEnd"; +static const char* jkVgroupInfoEpSet = "EpSet"; +static const char* jkVgroupInfoNumOfTable = "NumOfTable"; + +static int32_t vgroupInfoToJson(const void* pObj, SJson* pJson) { + const SVgroupInfo* pNode = (const SVgroupInfo*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkVgroupInfoVgId, pNode->vgId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkVgroupInfoHashBegin, pNode->hashBegin); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkVgroupInfoHashEnd, pNode->hashEnd); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkVgroupInfoEpSet, epSetToJson, &pNode->epSet); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkVgroupInfoNumOfTable, pNode->numOfTable); + } + + return code; +} + +static int32_t jsonToVgroupInfo(const SJson* pJson, void* pObj) { + SVgroupInfo* pNode = (SVgroupInfo*)pObj; + + int32_t code = tjsonGetIntValue(pJson, jkVgroupInfoVgId, &pNode->vgId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUIntValue(pJson, jkVgroupInfoHashBegin, &pNode->hashBegin); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUIntValue(pJson, jkVgroupInfoHashEnd, &pNode->hashEnd); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToObject(pJson, jkVgroupInfoEpSet, jsonToEpSet, &pNode->epSet); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkVgroupInfoNumOfTable, &pNode->numOfTable); + } + + return code; +} + +static const char* jkVgroupsInfoNum = "Num"; +static const char* jkVgroupsInfoVgroups = "Vgroups"; + +static int32_t vgroupsInfoToJson(const void* pObj, SJson* pJson) { + const SVgroupsInfo* pNode = (const SVgroupsInfo*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkVgroupsInfoNum, pNode->numOfVgroups); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddArray(pJson, jkVgroupsInfoVgroups, vgroupInfoToJson, pNode->vgroups, sizeof(SVgroupInfo), pNode->numOfVgroups); + } + + return code; +} + +static int32_t jsonToVgroupsInfo(const SJson* pJson, void* pObj) { + SVgroupsInfo* pNode = (SVgroupsInfo*)pObj; + + int32_t code = tjsonGetIntValue(pJson, jkVgroupsInfoNum, &pNode->numOfVgroups); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToArray(pJson, jkVgroupsInfoVgroups, jsonToVgroupInfo, pNode->vgroups, sizeof(SVgroupInfo)); + } + + return code; +} + +static const char* jkRealTableMetaSize = "MetaSize"; +static const char* jkRealTableMeta = "Meta"; +static const char* jkRealTableVgroupsInfoSize = "VgroupsInfoSize"; +static const char* jkRealTableVgroupsInfo = "VgroupsInfo"; + +static int32_t realTableNodeToJson(const void* pObj, SJson* pJson) { + const SRealTableNode* pNode = (const SRealTableNode*)pObj; + + int32_t code = tableNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkRealTableMetaSize, TABLE_META_SIZE(pNode->pMeta)); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkRealTableMeta, tableMetaToJson, pNode->pMeta); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkRealTableVgroupsInfoSize, VGROUPS_INFO_SIZE(pNode->pVgroupList)); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkRealTableVgroupsInfo, vgroupsInfoToJson, pNode->pVgroupList); + } + + return code; +} + +static int32_t jsonToRealTableNode(const SJson* pJson, void* pObj) { + SRealTableNode* pNode = (SRealTableNode*)pObj; + + int32_t objSize = 0; + int32_t code = jsonToTableNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkRealTableMetaSize, &objSize); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonMakeObject(pJson, jkRealTableMeta, jsonToTableMeta, (void**)&pNode->pMeta, objSize); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkRealTableVgroupsInfoSize, &objSize); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonMakeObject(pJson, jkRealTableVgroupsInfo, jsonToVgroupsInfo, (void**)&pNode->pVgroupList, objSize); + } + + return code; +} + static const char* jkGroupingSetType = "GroupingSetType"; static const char* jkGroupingSetParameter = "Parameters"; @@ -1464,7 +1836,7 @@ static const char* jkSelectStmtSlimit = "Slimit"; static int32_t selectStmtTojson(const void* pObj, SJson* pJson) { const SSelectStmt* pNode = (const SSelectStmt*)pObj; - int32_t code = tjsonAddIntegerToObject(pJson, jkSelectStmtDistinct, pNode->isDistinct); + int32_t code = tjsonAddBoolToObject(pJson, jkSelectStmtDistinct, pNode->isDistinct); if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkSelectStmtProjections, pNode->pProjectionList); } @@ -1499,6 +1871,83 @@ static int32_t selectStmtTojson(const void* pObj, SJson* pJson) { return code; } +static int32_t jsonToSelectStmt(const SJson* pJson, void* pObj) { + SSelectStmt* pNode = (SSelectStmt*)pObj; + + int32_t code = tjsonGetBoolValue(pJson, jkSelectStmtDistinct, &pNode->isDistinct); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkSelectStmtProjections, &pNode->pProjectionList); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkSelectStmtFrom, &pNode->pFromTable); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkSelectStmtWhere, &pNode->pWhere); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkSelectStmtPartitionBy, &pNode->pPartitionByList); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkSelectStmtWindow, &pNode->pWindow); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkSelectStmtGroupBy, &pNode->pGroupByList); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkSelectStmtHaving, &pNode->pHaving); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkSelectStmtOrderBy, &pNode->pOrderByList); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkSelectStmtLimit, &pNode->pLimit); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkSelectStmtSlimit, &pNode->pSlimit); + } + + return code; +} + +static const char* jkCreateTopicStmtTopicName = "TopicName"; +static const char* jkCreateTopicStmtSubscribeDbName = "SubscribeDbName"; +static const char* jkCreateTopicStmtIgnoreExists = "IgnoreExists"; +static const char* jkCreateTopicStmtQuery = "Query"; + +static int32_t createTopicStmtToJson(const void* pObj, SJson* pJson) { + const SCreateTopicStmt* pNode = (const SCreateTopicStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkCreateTopicStmtTopicName, pNode->topicName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateTopicStmtSubscribeDbName, pNode->subscribeDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkCreateTopicStmtIgnoreExists, pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkCreateTopicStmtQuery, nodeToJson, pNode->pQuery); + } + + return code; +} + +static int32_t jsonToCreateTopicStmt(const SJson* pJson, void* pObj) { + SCreateTopicStmt* pNode = (SCreateTopicStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkCreateTopicStmtTopicName, pNode->topicName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateTopicStmtSubscribeDbName, pNode->subscribeDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkCreateTopicStmtIgnoreExists, &pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkCreateTopicStmtQuery, &pNode->pQuery); + } + + return code; +} + static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { switch (nodeType(pObj)) { case QUERY_NODE_COLUMN: @@ -1512,6 +1961,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_FUNCTION: return functionNodeToJson(pObj, pJson); case QUERY_NODE_REAL_TABLE: + return realTableNodeToJson(pObj, pJson); case QUERY_NODE_TEMP_TABLE: case QUERY_NODE_JOIN_TABLE: break; @@ -1549,6 +1999,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_SHOW_DATABASES_STMT: case QUERY_NODE_SHOW_TABLES_STMT: break; + case QUERY_NODE_CREATE_TOPIC_STMT: + return createTopicStmtToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_SCAN: return logicScanNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_JOIN: @@ -1565,9 +2017,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return physiTagScanNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: return physiTableScanNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: - break; + return physiStreamScanNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return physiProjectNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_JOIN: @@ -1589,6 +2040,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN: return planToJson(pObj, pJson); default: + assert(0); break; } nodesWarn("specificNodeToJson unknown node = %s", nodesNodeName(nodeType(pObj))); @@ -1607,7 +2059,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToLogicConditionNode(pJson, pObj); case QUERY_NODE_FUNCTION: return jsonToFunctionNode(pJson, pObj); - // case QUERY_NODE_REAL_TABLE: + case QUERY_NODE_REAL_TABLE: + return jsonToRealTableNode(pJson, pObj); // case QUERY_NODE_TEMP_TABLE: // case QUERY_NODE_JOIN_TABLE: // break; @@ -1633,20 +2086,24 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToDownstreamSourceNode(pJson, pObj); // case QUERY_NODE_SET_OPERATOR: // break; - // case QUERY_NODE_SELECT_STMT: - // return jsonToSelectStmt(pJson, pObj); - // case QUERY_NODE_LOGIC_PLAN_SCAN: - // return jsonToLogicScanNode(pJson, pObj); + case QUERY_NODE_SELECT_STMT: + return jsonToSelectStmt(pJson, pObj); + case QUERY_NODE_CREATE_TOPIC_STMT: + return jsonToCreateTopicStmt(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN_SCAN: + return jsonToLogicScanNode(pJson, pObj); // case QUERY_NODE_LOGIC_PLAN_JOIN: // return jsonToLogicJoinNode(pJson, pObj); // case QUERY_NODE_LOGIC_PLAN_AGG: // return jsonToLogicAggNode(pJson, pObj); - // case QUERY_NODE_LOGIC_PLAN_PROJECT: - // return jsonToLogicProjectNode(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN_PROJECT: + return jsonToLogicProjectNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: return jsonToPhysiTagScanNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: return jsonToPhysiTableScanNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: + return jsonToPhysiStreamScanNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return jsonToPhysiProjectNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_JOIN: @@ -1768,6 +2225,7 @@ int32_t nodesStringToNode(const char* pStr, SNode** pNode) { int32_t code = makeNodeByJson(pJson, pNode); if (TSDB_CODE_SUCCESS != code) { nodesDestroyNode(*pNode); + *pNode = NULL; terrno = code; return code; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 75568019cb..0d48202f73 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -125,6 +125,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SCreateDnodeStmt)); case QUERY_NODE_DROP_DNODE_STMT: return makeNode(type, sizeof(SDropDnodeStmt)); + case QUERY_NODE_ALTER_DNODE_STMT: + return makeNode(type, sizeof(SAlterDnodeStmt)); case QUERY_NODE_SHOW_DNODES_STMT: return makeNode(type, sizeof(SShowStmt)); case QUERY_NODE_SHOW_VGROUPS_STMT: @@ -168,7 +170,7 @@ SNodeptr nodesMakeNode(ENodeType type) { case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: return makeNode(type, sizeof(STableSeqScanPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: - return makeNode(type, sizeof(SNode)); + return makeNode(type, sizeof(SStreamScanPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return makeNode(type, sizeof(SProjectPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_JOIN: @@ -357,6 +359,17 @@ int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) { return TSDB_CODE_SUCCESS; } +int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc) { + if (NULL == pSrc) { + return TSDB_CODE_OUT_OF_MEMORY; + } + int32_t code = nodesListAppendList(pTarget, pSrc); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyList(pSrc); + } + return code; +} + SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) { if (NULL == pCell->pPrev) { pList->pHead = pCell->pNext; @@ -571,7 +584,7 @@ typedef struct SCollectFuncsCxt { static EDealRes collectFuncs(SNode* pNode, void* pContext) { SCollectFuncsCxt* pCxt = (SCollectFuncsCxt*)pContext; if (QUERY_NODE_FUNCTION == nodeType(pNode) && pCxt->classifier(((SFunctionNode*)pNode)->funcId)) { - pCxt->errCode = nodesListAppend(pCxt->pFuncs, pNode); + pCxt->errCode = nodesListStrictAppend(pCxt->pFuncs, nodesCloneNode(pNode)); return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); } return DEAL_RES_CONTINUE; diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 31d7331d0e..46bed1d61e 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -50,8 +50,10 @@ typedef enum EDatabaseOptionType { DB_OPTION_TTL, DB_OPTION_WAL, DB_OPTION_VGROUPS, - DB_OPTION_SINGLESTABLE, - DB_OPTION_STREAMMODE, + DB_OPTION_SINGLE_STABLE, + DB_OPTION_STREAM_MODE, + DB_OPTION_RETENTIONS, + DB_OPTION_FILE_FACTOR, DB_OPTION_MAX } EDatabaseOptionType; @@ -65,6 +67,11 @@ typedef enum ETableOptionType { TABLE_OPTION_MAX } ETableOptionType; +typedef struct SAlterOption { + int32_t type; + SToken val; +} SAlterOption; + extern SToken nil_token; void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt); @@ -110,13 +117,16 @@ SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pPr SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight); SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt); +SNode* createDefaultAlterDatabaseOptions(SAstCreateContext* pCxt); SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, const SToken* pVal); SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pDbName, SNode* pOptions); SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pDbName); SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName, SNode* pOptions); SNode* createDefaultTableOptions(SAstCreateContext* pCxt); +SNode* createDefaultAlterTableOptions(SAstCreateContext* pCxt); SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, const SToken* pVal); SNode* setTableSmaOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pSma); +SNode* setTableRollupOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pFuncs); SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment); SDataType createDataType(uint8_t type); SDataType createVarLenDataType(uint8_t type, const SToken* pLen); @@ -126,6 +136,11 @@ SNode* createCreateMultiTableStmt(SAstCreateContext* pCxt, SNodeList* pSubTables SNode* createDropTableClause(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable); SNode* createDropTableStmt(SAstCreateContext* pCxt, SNodeList* pTables); SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable); +SNode* createAlterTableOption(SAstCreateContext* pCxt, SNode* pRealTable, SNode* pOptions); +SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName, SDataType dataType); +SNode* createAlterTableDropCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName); +SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pOldColName, const SToken* pNewColName); +SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, const SToken* pTagName, SNode* pVal); SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName); SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDbName); SNode* createCreateUserStmt(SAstCreateContext* pCxt, const SToken* pUserName, const SToken* pPassword); @@ -133,6 +148,7 @@ SNode* createAlterUserStmt(SAstCreateContext* pCxt, const SToken* pUserName, int SNode* createDropUserStmt(SAstCreateContext* pCxt, const SToken* pUserName); SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort); SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode); +SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue); SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, const SToken* pIndexName, const SToken* pTableName, SNodeList* pCols, SNode* pOptions); SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInterval, SNode* pOffset, SNode* pSliding); SNode* createDropIndexStmt(SAstCreateContext* pCxt, const SToken* pIndexName, const SToken* pTableName); @@ -140,6 +156,7 @@ SNode* createCreateQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId); SNode* createDropQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId); SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery, const SToken* pSubscribeDbName); SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pTopicName); +SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue); #ifdef __cplusplus } diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 31c26e1e40..2298ae0761 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -21,13 +21,15 @@ #include "parAst.h" } -%syntax_error { - if(TOKEN.z) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); - } else { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL); +%syntax_error { + if (pCxt->valid) { + if(TOKEN.z) { + generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); + } else { + generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL); + } + pCxt->valid = false; } - pCxt->valid = false; } %left OR. @@ -41,6 +43,41 @@ %left NK_CONCAT. //%right NK_BITNOT. +/************************************************ create/alter account *****************************************/ +cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options. { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } +cmd ::= ALTER ACCOUNT NK_ID alter_account_options. { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } + +%type account_options { int32_t } +%destructor account_options { } +account_options ::= . { } +account_options ::= account_options PPS literal. { } +account_options ::= account_options TSERIES literal. { } +account_options ::= account_options STORAGE literal. { } +account_options ::= account_options STREAMS literal. { } +account_options ::= account_options QTIME literal. { } +account_options ::= account_options DBS literal. { } +account_options ::= account_options USERS literal. { } +account_options ::= account_options CONNS literal. { } +account_options ::= account_options STATE literal. { } + +%type alter_account_options { int32_t } +%destructor alter_account_options { } +alter_account_options ::= alter_account_option. { } +alter_account_options ::= alter_account_options alter_account_option. { } + +%type alter_account_option { int32_t } +%destructor alter_account_option { } +alter_account_option ::= PASS literal. { } +alter_account_option ::= PPS literal. { } +alter_account_option ::= TSERIES literal. { } +alter_account_option ::= STORAGE literal. { } +alter_account_option ::= STREAMS literal. { } +alter_account_option ::= QTIME literal. { } +alter_account_option ::= DBS literal. { } +alter_account_option ::= USERS literal. { } +alter_account_option ::= CONNS literal. { } +alter_account_option ::= STATE literal. { } + /************************************************ create/alter/drop/show user *****************************************/ cmd ::= CREATE USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createCreateUserStmt(pCxt, &A, &B); } cmd ::= ALTER USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PASSWD, &B); } @@ -48,12 +85,16 @@ cmd ::= ALTER USER user_name(A) PRIVILEGE NK_STRING(B). cmd ::= DROP USER user_name(A). { pCxt->pRootNode = createDropUserStmt(pCxt, &A); } cmd ::= SHOW USERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT, NULL); } -/************************************************ create/drop/show dnode **********************************************/ +/************************************************ create/drop/alter/show dnode ****************************************/ cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL); } cmd ::= CREATE DNODE dnode_host_name(A) PORT NK_INTEGER(B). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, &B); } cmd ::= DROP DNODE NK_INTEGER(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A); } cmd ::= DROP DNODE dnode_endpoint(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A); } cmd ::= SHOW DNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT, NULL); } +cmd ::= ALTER DNODE NK_INTEGER(A) NK_STRING(B). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &A, &B, NULL); } +cmd ::= ALTER DNODE NK_INTEGER(A) NK_STRING(B) NK_STRING(C). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &A, &B, &C); } +cmd ::= ALTER ALL DNODES NK_STRING(A). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &A, NULL); } +cmd ::= ALTER ALL DNODES NK_STRING(A) NK_STRING(B). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &A, &B); } %type dnode_endpoint { SToken } %destructor dnode_endpoint { } @@ -64,9 +105,13 @@ dnode_endpoint(A) ::= NK_STRING(B). dnode_host_name(A) ::= NK_ID(B). { A = B; } dnode_host_name(A) ::= NK_IPTOKEN(B). { A = B; } +/************************************************ alter local *********************************************************/ +cmd ::= ALTER LOCAL NK_STRING(A). { pCxt->pRootNode = createAlterLocalStmt(pCxt, &A, NULL); } +cmd ::= ALTER LOCAL NK_STRING(A) NK_STRING(B). { pCxt->pRootNode = createAlterLocalStmt(pCxt, &A, &B); } + /************************************************ create/drop qnode ***************************************************/ cmd ::= CREATE QNODE ON DNODE NK_INTEGER(A). { pCxt->pRootNode = createCreateQnodeStmt(pCxt, &A); } -cmd ::= DROP QNODE ON DNODE NK_INTEGER(A). { pCxt->pRootNode = createDropQnodeStmt(pCxt, &A); } +cmd ::= DROP QNODE ON DNODE NK_INTEGER(A). { pCxt->pRootNode = createDropQnodeStmt(pCxt, &A); } cmd ::= SHOW QNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT, NULL); } /************************************************ create/drop/show/use database ***************************************/ @@ -74,7 +119,7 @@ cmd ::= CREATE DATABASE not_exists_opt(A) db_name(B) db_options(C). cmd ::= DROP DATABASE exists_opt(A) db_name(B). { pCxt->pRootNode = createDropDatabaseStmt(pCxt, A, &B); } cmd ::= SHOW DATABASES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT, NULL); } cmd ::= USE db_name(A). { pCxt->pRootNode = createUseDatabaseStmt(pCxt, &A); } -cmd ::= ALTER DATABASE db_name(A) db_options(B). { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &A, B); } +cmd ::= ALTER DATABASE db_name(A) alter_db_options(B). { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &A, B); } %type not_exists_opt { bool } %destructor not_exists_opt { } @@ -102,20 +147,55 @@ db_options(A) ::= db_options(B) REPLICA NK_INTEGER(C). db_options(A) ::= db_options(B) TTL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TTL, &C); } db_options(A) ::= db_options(B) WAL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL, &C); } db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_VGROUPS, &C); } -db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLESTABLE, &C); } -db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STREAMMODE, &C); } +db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLE_STABLE, &C); } +db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STREAM_MODE, &C); } +db_options(A) ::= db_options(B) RETENTIONS NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_RETENTIONS, &C); } +db_options(A) ::= db_options(B) FILE_FACTOR NK_FLOAT(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_FILE_FACTOR, &C); } + +alter_db_options(A) ::= alter_db_option(B). { A = createDefaultAlterDatabaseOptions(pCxt); A = setDatabaseOption(pCxt, A, B.type, &B.val); } +alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setDatabaseOption(pCxt, B, C.type, &C.val); } + +%type alter_db_option { SAlterOption } +%destructor alter_db_option { } +alter_db_option(A) ::= BLOCKS NK_INTEGER(B). { A.type = DB_OPTION_BLOCKS; A.val = B; } +alter_db_option(A) ::= FSYNC NK_INTEGER(B). { A.type = DB_OPTION_FSYNC; A.val = B; } +alter_db_option(A) ::= KEEP NK_INTEGER(B). { A.type = DB_OPTION_KEEP; A.val = B; } +alter_db_option(A) ::= WAL NK_INTEGER(B). { A.type = DB_OPTION_WAL; A.val = B; } +alter_db_option(A) ::= QUORUM NK_INTEGER(B). { A.type = DB_OPTION_QUORUM; A.val = B; } +alter_db_option(A) ::= CACHELAST NK_INTEGER(B). { A.type = DB_OPTION_CACHELAST; A.val = B; } /************************************************ create/drop/show table/stable ***************************************/ cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B) - NK_LP column_def_list(C) NK_RP tags_def_opt(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E);} -cmd ::= CREATE TABLE multi_create_clause(A). { pCxt->pRootNode = createCreateMultiTableStmt(pCxt, A);} + NK_LP column_def_list(C) NK_RP tags_def_opt(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E); } +cmd ::= CREATE TABLE multi_create_clause(A). { pCxt->pRootNode = createCreateMultiTableStmt(pCxt, A); } cmd ::= CREATE STABLE not_exists_opt(A) full_table_name(B) - NK_LP column_def_list(C) NK_RP tags_def(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E);} + NK_LP column_def_list(C) NK_RP tags_def(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E); } cmd ::= DROP TABLE multi_drop_clause(A). { pCxt->pRootNode = createDropTableStmt(pCxt, A); } cmd ::= DROP STABLE exists_opt(A) full_table_name(B). { pCxt->pRootNode = createDropSuperTableStmt(pCxt, A, B); } cmd ::= SHOW TABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, NULL); } cmd ::= SHOW STABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, NULL); } +cmd ::= ALTER TABLE alter_table_clause(A). { pCxt->pRootNode = A; } +cmd ::= ALTER STABLE alter_table_clause(A). { pCxt->pRootNode = A; } + +alter_table_clause(A) ::= full_table_name(B) alter_table_options(C). { A = createAlterTableOption(pCxt, B, C); } +alter_table_clause(A) ::= + full_table_name(B) ADD COLUMN column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_ADD_COLUMN, &C, D); } +alter_table_clause(A) ::= full_table_name(B) DROP COLUMN column_name(C). { A = createAlterTableDropCol(pCxt, B, TSDB_ALTER_TABLE_DROP_COLUMN, &C); } +alter_table_clause(A) ::= + full_table_name(B) MODIFY COLUMN column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &C, D); } +alter_table_clause(A) ::= + full_table_name(B) RENAME COLUMN column_name(C) column_name(D). { A = createAlterTableRenameCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &C, &D); } +alter_table_clause(A) ::= + full_table_name(B) ADD TAG column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_ADD_TAG, &C, D); } +alter_table_clause(A) ::= full_table_name(B) DROP TAG column_name(C). { A = createAlterTableDropCol(pCxt, B, TSDB_ALTER_TABLE_DROP_TAG, &C); } +alter_table_clause(A) ::= + full_table_name(B) MODIFY TAG column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &C, D); } +alter_table_clause(A) ::= + full_table_name(B) RENAME TAG column_name(C) column_name(D). { A = createAlterTableRenameCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &C, &D); } +alter_table_clause(A) ::= + full_table_name(B) SET TAG column_name(C) NK_EQ literal(D). { A = createAlterTableSetTag(pCxt, B, &C, D); } + %type multi_create_clause { SNodeList* } %destructor multi_create_clause { nodesDestroyList($$); } multi_create_clause(A) ::= create_subtable_clause(B). { A = createNodeList(pCxt, B); } @@ -183,11 +263,21 @@ tags_def_opt(A) ::= tags_def(B). %destructor tags_def { nodesDestroyList($$); } tags_def(A) ::= TAGS NK_LP column_def_list(B) NK_RP. { A = B; } -table_options(A) ::= . { A = createDefaultTableOptions(pCxt);} +table_options(A) ::= . { A = createDefaultTableOptions(pCxt); } table_options(A) ::= table_options(B) COMMENT NK_STRING(C). { A = setTableOption(pCxt, B, TABLE_OPTION_COMMENT, &C); } table_options(A) ::= table_options(B) KEEP NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_KEEP, &C); } table_options(A) ::= table_options(B) TTL NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_TTL, &C); } table_options(A) ::= table_options(B) SMA NK_LP col_name_list(C) NK_RP. { A = setTableSmaOption(pCxt, B, C); } +table_options(A) ::= table_options(B) ROLLUP NK_LP func_name_list(C) NK_RP. { A = setTableRollupOption(pCxt, B, C); } + +alter_table_options(A) ::= alter_table_option(B). { A = createDefaultAlterTableOptions(pCxt); A = setTableOption(pCxt, A, B.type, &B.val); } +alter_table_options(A) ::= alter_table_options(B) alter_table_option(C). { A = setTableOption(pCxt, B, C.type, &C.val); } + +%type alter_table_option { SAlterOption } +%destructor alter_table_option { } +alter_table_option(A) ::= COMMENT NK_STRING(B). { A.type = TABLE_OPTION_COMMENT; A.val = B; } +alter_table_option(A) ::= KEEP NK_INTEGER(B). { A.type = TABLE_OPTION_KEEP; A.val = B; } +alter_table_option(A) ::= TTL NK_INTEGER(B). { A.type = TABLE_OPTION_TTL; A.val = B; } %type col_name_list { SNodeList* } %destructor col_name_list { nodesDestroyList($$); } @@ -196,6 +286,13 @@ col_name_list(A) ::= col_name_list(B) NK_COMMA col_name(C). col_name(A) ::= column_name(B). { A = createColumnNode(pCxt, NULL, &B); } +%type func_name_list { SNodeList* } +%destructor func_name_list { nodesDestroyList($$); } +func_name_list(A) ::= func_name(B). { A = createNodeList(pCxt, B); } +func_name_list(A) ::= func_name_list(B) NK_COMMA col_name(C). { A = addNodeToList(pCxt, B, C); } + +func_name(A) ::= function_name(B). { A = createFunctionNode(pCxt, &B, NULL); } + /************************************************ create index ********************************************************/ cmd ::= CREATE SMA INDEX index_name(A) ON table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, &A, &B, NULL, C); } cmd ::= CREATE FULLTEXT INDEX @@ -546,7 +643,7 @@ query_expression_body(A) ::= query_primary(A) ::= query_specification(B). { A = B; } //query_primary(A) ::= // NK_LP query_expression_body(B) -// order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP. { A = B;} +// order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP. { A = B; } %type order_by_clause_opt { SNodeList* } %destructor order_by_clause_opt { nodesDestroyList($$); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 034346362c..eed6391d5c 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -246,6 +246,16 @@ static SDatabaseOptions* setDbStreamMode(SAstCreateContext* pCxt, SDatabaseOptio return pOptions; } +static SDatabaseOptions* setDbRetentions(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) { + // todo + return pOptions; +} + +static SDatabaseOptions* setDbFileFactor(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) { + // todo + return pOptions; +} + static void initSetDatabaseOptionFp() { setDbOptionFuncs[DB_OPTION_BLOCKS] = setDbBlocks; setDbOptionFuncs[DB_OPTION_CACHE] = setDbCache; @@ -262,8 +272,10 @@ static void initSetDatabaseOptionFp() { setDbOptionFuncs[DB_OPTION_TTL] = setDbTtl; setDbOptionFuncs[DB_OPTION_WAL] = setDbWal; setDbOptionFuncs[DB_OPTION_VGROUPS] = setDbVgroups; - setDbOptionFuncs[DB_OPTION_SINGLESTABLE] = setDbSingleStable; - setDbOptionFuncs[DB_OPTION_STREAMMODE] = setDbStreamMode; + setDbOptionFuncs[DB_OPTION_SINGLE_STABLE] = setDbSingleStable; + setDbOptionFuncs[DB_OPTION_STREAM_MODE] = setDbStreamMode; + setDbOptionFuncs[DB_OPTION_RETENTIONS] = setDbRetentions; + setDbOptionFuncs[DB_OPTION_FILE_FACTOR] = setDbFileFactor; } static STableOptions* setTableKeep(SAstCreateContext* pCxt, STableOptions* pOptions, const SToken* pVal) { @@ -772,6 +784,29 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) { return (SNode*)pOptions; } +SNode* createDefaultAlterDatabaseOptions(SAstCreateContext* pCxt) { + SDatabaseOptions* pOptions = nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS); + CHECK_OUT_OF_MEM(pOptions); + pOptions->numOfBlocks = -1; + pOptions->cacheBlockSize = -1; + pOptions->cachelast = -1; + pOptions->compressionLevel = -1; + pOptions->daysPerFile = -1; + pOptions->fsyncPeriod = -1; + pOptions->maxRowsPerBlock = -1; + pOptions->minRowsPerBlock = -1; + pOptions->keep = -1; + pOptions->precision = -1; + pOptions->quorum = -1; + pOptions->replica = -1; + pOptions->ttl = -1; + pOptions->walLevel = -1; + pOptions->numOfVgroups = -1; + pOptions->singleStable = -1; + pOptions->streamMode = -1; + return (SNode*)pOptions; +} + SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, const SToken* pVal) { return (SNode*)setDbOptionFuncs[type](pCxt, (SDatabaseOptions*)pOptions, pVal); } @@ -818,6 +853,14 @@ SNode* createDefaultTableOptions(SAstCreateContext* pCxt) { return (SNode*)pOptions; } +SNode* createDefaultAlterTableOptions(SAstCreateContext* pCxt) { + STableOptions* pOptions = nodesMakeNode(QUERY_NODE_TABLE_OPTIONS); + CHECK_OUT_OF_MEM(pOptions); + pOptions->keep = -1; + pOptions->ttl = -1; + return (SNode*)pOptions; +} + SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, const SToken* pVal) { return (SNode*)setTableOptionFuncs[type](pCxt, (STableOptions*)pOptions, pVal); } @@ -827,6 +870,11 @@ SNode* setTableSmaOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pS return pOptions; } +SNode* setTableRollupOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pFuncs) { + // todo + return pOptions; +} + SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment) { SColumnDefNode* pCol = (SColumnDefNode*)nodesMakeNode(QUERY_NODE_COLUMN_DEF); CHECK_OUT_OF_MEM(pCol); @@ -912,6 +960,49 @@ SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool ignoreNotExists, S return (SNode*)pStmt; } +SNode* createAlterTableOption(SAstCreateContext* pCxt, SNode* pRealTable, SNode* pOptions) { + SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT); + CHECK_OUT_OF_MEM(pStmt); + pStmt->alterType = TSDB_ALTER_TABLE_UPDATE_OPTIONS; + pStmt->pOptions = (STableOptions*)pOptions; + return (SNode*)pStmt; +} + +SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName, SDataType dataType) { + SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT); + CHECK_OUT_OF_MEM(pStmt); + pStmt->alterType = alterType; + strncpy(pStmt->colName, pColName->z, pColName->n); + pStmt->dataType = dataType; + return (SNode*)pStmt; +} + +SNode* createAlterTableDropCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName) { + SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT); + CHECK_OUT_OF_MEM(pStmt); + pStmt->alterType = alterType; + strncpy(pStmt->colName, pColName->z, pColName->n); + return (SNode*)pStmt; +} + +SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pOldColName, const SToken* pNewColName) { + SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT); + CHECK_OUT_OF_MEM(pStmt); + pStmt->alterType = alterType; + strncpy(pStmt->colName, pOldColName->z, pOldColName->n); + strncpy(pStmt->newColName, pNewColName->z, pNewColName->n); + return (SNode*)pStmt; +} + +SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, const SToken* pTagName, SNode* pVal) { + SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT); + CHECK_OUT_OF_MEM(pStmt); + pStmt->alterType = TSDB_ALTER_TABLE_UPDATE_TAG_VAL; + strncpy(pStmt->colName, pTagName->z, pTagName->n); + pStmt->pVal = (SValueNode*)pVal; + return (SNode*)pStmt; +} + SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName) { SUseDatabaseStmt* pStmt = (SUseDatabaseStmt*)nodesMakeNode(QUERY_NODE_USE_DATABASE_STMT); CHECK_OUT_OF_MEM(pStmt); @@ -1009,6 +1100,17 @@ SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode) { return (SNode*)pStmt; } +SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue) { + SAlterDnodeStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_DNODE_STMT); + CHECK_OUT_OF_MEM(pStmt); + pStmt->dnodeId = strtol(pDnode->z, NULL, 10); + trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config)); + if (NULL != pValue) { + trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value)); + } + return (SNode*)pStmt; +} + SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, const SToken* pIndexName, const SToken* pTableName, SNodeList* pCols, SNode* pOptions) { if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName)) { return NULL; @@ -1077,3 +1179,13 @@ SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const pStmt->ignoreNotExists = ignoreNotExists; return (SNode*)pStmt; } + +SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue) { + SAlterLocalStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_LOCAL_STMT); + CHECK_OUT_OF_MEM(pStmt); + trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config)); + if (NULL != pValue) { + trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value)); + } + return (SNode*)pStmt; +} diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index a37820634f..803d8e00d8 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -391,7 +391,7 @@ static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, cha } // Remove quotation marks - if (TSDB_DATA_TYPE_BINARY == type) { + if (TK_NK_STRING == pToken->type) { if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) { return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z); } diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 2d58d4e1e3..3cd4549ad0 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -28,6 +28,7 @@ typedef struct SKeyword { // keywords in sql string static SKeyword keywordTable[] = { + {"ACCOUNT", TK_ACCOUNT}, {"ALL", TK_ALL}, {"ALTER", TK_ALTER}, {"AND", TK_AND}, @@ -168,7 +169,6 @@ static SKeyword keywordTable[] = { // {"SCORES", TK_SCORES}, // {"GRANTS", TK_GRANTS}, // {"DOT", TK_DOT}, - // {"ACCOUNT", TK_ACCOUNT}, // {"DESCRIBE", TK_DESCRIBE}, // {"SYNCDB", TK_SYNCDB}, // {"LOCAL", TK_LOCAL}, @@ -266,10 +266,10 @@ static void doInitKeywordsTable(void) { } } -static pthread_once_t keywordsHashTableInit = PTHREAD_ONCE_INIT; +static TdThreadOnce keywordsHashTableInit = PTHREAD_ONCE_INIT; static int32_t tKeywordCode(const char* z, int n) { - pthread_once(&keywordsHashTableInit, doInitKeywordsTable); + taosThreadOnce(&keywordsHashTableInit, doInitKeywordsTable); char key[512] = {0}; if (n > tListLen(key)) { // too long token, can not be any other token type diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index af7a38b702..70652901ef 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -126,7 +126,7 @@ static void setColumnInfoByExpr(const STableNode* pTable, SExprNode* pExpr, SCol static int32_t createColumnNodeByTable(STranslateContext* pCxt, const STableNode* pTable, SNodeList* pList) { if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; - int32_t nums = pMeta->tableInfo.numOfColumns + ((TSDB_SUPER_TABLE == pMeta->tableType)? pMeta->tableInfo.numOfTags:0); + int32_t nums = pMeta->tableInfo.numOfColumns + ((TSDB_SUPER_TABLE == pMeta->tableType) ? pMeta->tableInfo.numOfTags : 0); for (int32_t i = 0; i < nums; ++i) { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { @@ -499,6 +499,10 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) } static int32_t setTableVgroupList(SParseContext* pCxt, SName* name, SRealTableNode* pRealTable) { + if (pCxt->topicQuery) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_SUPER_TABLE == pRealTable->pMeta->tableType) { SArray* vgroupList = NULL; int32_t code = catalogGetTableDistVgInfo(pCxt->pCatalog, pCxt->pTransporter, &pCxt->mgmtEpSet, name, &vgroupList); @@ -962,6 +966,73 @@ static int32_t translateDropSuperTable(STranslateContext* pCxt, SDropSuperTableS return doTranslateDropSuperTable(pCxt, &tableName, pStmt->ignoreNotExists); } +static int32_t setAlterTableField(SAlterTableStmt* pStmt, SMAltertbReq* pAlterReq) { + pAlterReq->pFields = taosArrayInit(2, sizeof(TAOS_FIELD)); + if (NULL == pAlterReq->pFields) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + switch (pStmt->alterType) { + case TSDB_ALTER_TABLE_ADD_TAG: + case TSDB_ALTER_TABLE_DROP_TAG: + case TSDB_ALTER_TABLE_ADD_COLUMN: + case TSDB_ALTER_TABLE_DROP_COLUMN: + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: + case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: { + TAOS_FIELD field = { .type = pStmt->dataType.type, .bytes = pStmt->dataType.bytes }; + strcpy(field.name, pStmt->colName); + taosArrayPush(pAlterReq->pFields, &field); + break; + } + case TSDB_ALTER_TABLE_UPDATE_TAG_NAME: + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: { + TAOS_FIELD oldField = {0}; + strcpy(oldField.name, pStmt->colName); + taosArrayPush(pAlterReq->pFields, &oldField); + TAOS_FIELD newField = {0}; + strcpy(oldField.name, pStmt->newColName); + taosArrayPush(pAlterReq->pFields, &newField); + break; + } + default: + break; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t translateAlterTable(STranslateContext* pCxt, SAlterTableStmt* pStmt) { + SMAltertbReq alterReq = {0}; + SName tableName = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId }; + strcpy(tableName.dbname, pStmt->dbName); + strcpy(tableName.tname, pStmt->tableName); + tNameExtractFullName(&tableName, alterReq.name); + alterReq.alterType = pStmt->alterType; + alterReq.numOfFields = 1; + if (TSDB_ALTER_TABLE_UPDATE_OPTIONS == pStmt->alterType) { + // todo + } else { + if (TSDB_CODE_SUCCESS != setAlterTableField(pStmt, &alterReq)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + if (NULL == pCxt->pCmdMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; + pCxt->pCmdMsg->msgType = TDMT_MND_ALTER_STB; + pCxt->pCmdMsg->msgLen = tSerializeSMAlterStbReq(NULL, 0, &alterReq); + pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + if (NULL == pCxt->pCmdMsg->pMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tSerializeSMAlterStbReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &alterReq); + + return TSDB_CODE_SUCCESS; +} + static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* pStmt) { SName name = {0}; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); @@ -1099,6 +1170,28 @@ static int32_t translateDropDnode(STranslateContext* pCxt, SDropDnodeStmt* pStmt return TSDB_CODE_SUCCESS; } +static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pStmt) { + SMCfgDnodeReq cfgReq = {0}; + cfgReq.dnodeId = pStmt->dnodeId; + strcpy(cfgReq.config, pStmt->config); + strcpy(cfgReq.value, pStmt->value); + + pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + if (NULL == pCxt->pCmdMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; + pCxt->pCmdMsg->msgType = TDMT_MND_CONFIG_DNODE; + pCxt->pCmdMsg->msgLen = tSerializeSMCfgDnodeReq(NULL, 0, &cfgReq); + pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + if (NULL == pCxt->pCmdMsg->pMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tSerializeSMCfgDnodeReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &cfgReq); + + return TSDB_CODE_SUCCESS; +} + static int32_t nodeTypeToShowType(ENodeType nt) { switch (nt) { case QUERY_NODE_SHOW_DATABASES_STMT: @@ -1300,6 +1393,7 @@ static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* p SCMCreateTopicReq createReq = {0}; if (NULL != pStmt->pQuery) { + pCxt->pParseCxt->topicQuery = true; int32_t code = translateQuery(pCxt, pStmt->pQuery); if (TSDB_CODE_SUCCESS == code) { code = nodesNodeToString(pStmt->pQuery, false, &createReq.ast, NULL); @@ -1364,6 +1458,11 @@ static int32_t translateDropTopic(STranslateContext* pCxt, SDropTopicStmt* pStmt return TSDB_CODE_SUCCESS; } +static int32_t translateAlterLocal(STranslateContext* pCxt, SAlterLocalStmt* pStmt) { + // todo + return TSDB_CODE_SUCCESS; +} + static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pNode)) { @@ -1388,6 +1487,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { case QUERY_NODE_DROP_SUPER_TABLE_STMT: code = translateDropSuperTable(pCxt, (SDropSuperTableStmt*)pNode); break; + case QUERY_NODE_ALTER_TABLE_STMT: + code = translateAlterTable(pCxt, (SAlterTableStmt*)pNode); + break; case QUERY_NODE_CREATE_USER_STMT: code = translateCreateUser(pCxt, (SCreateUserStmt*)pNode); break; @@ -1406,6 +1508,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { case QUERY_NODE_DROP_DNODE_STMT: code = translateDropDnode(pCxt, (SDropDnodeStmt*)pNode); break; + case QUERY_NODE_ALTER_DNODE_STMT: + code = translateAlterDnode(pCxt, (SAlterDnodeStmt*)pNode); + break; case QUERY_NODE_SHOW_DATABASES_STMT: case QUERY_NODE_SHOW_STABLES_STMT: case QUERY_NODE_SHOW_USERS_STMT: @@ -1436,6 +1541,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { case QUERY_NODE_DROP_TOPIC_STMT: code = translateDropTopic(pCxt, (SDropTopicStmt*)pNode); break; + case QUERY_NODE_ALTER_LOCAL_STMT: + code = translateAlterLocal(pCxt, (SAlterLocalStmt*)pNode); + break; default: break; } @@ -1855,6 +1963,11 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) return rewriteToVnodeModifOpStmt(pQuery, pBufArray); } +static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) { + // todo + return TSDB_CODE_SUCCESS; +} + static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pQuery->pRoot)) { @@ -1866,6 +1979,11 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { case QUERY_NODE_CREATE_MULTI_TABLE_STMT: code = rewriteCreateMultiTable(pCxt, pQuery); break; + case QUERY_NODE_ALTER_TABLE_STMT: + if (TSDB_ALTER_TABLE_UPDATE_TAG_VAL == ((SAlterTableStmt*)pQuery->pRoot)->alterType) { + code = rewriteAlterTable(pCxt, pQuery); + } + break; default: break; } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index aa2516e2b9..69c2380b45 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -46,17 +46,19 @@ static char* getSyntaxErrFormat(int32_t errCode) { case TSDB_CODE_PAR_NOT_SINGLE_GROUP: return "Not a single-group group function"; case TSDB_CODE_PAR_TAGS_NOT_MATCHED: - return "tags number not matched"; + return "Tags number not matched"; case TSDB_CODE_PAR_INVALID_TAG_NAME: - return "invalid tag name : %s"; + return "Invalid tag name : %s"; case TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG: - return "name or password too long"; + return "Name or password too long"; case TSDB_CODE_PAR_PASSWD_EMPTY: - return "password can not be empty"; + return "Password can not be empty"; case TSDB_CODE_PAR_INVALID_PORT: - return "port should be an integer that is less than 65535 and greater than 0"; + return "Port should be an integer that is less than 65535 and greater than 0"; case TSDB_CODE_PAR_INVALID_ENDPOINT: - return "endpoint should be in the format of 'fqdn:port'"; + return "Endpoint should be in the format of 'fqdn:port'"; + case TSDB_CODE_PAR_EXPRIE_STATEMENT: + return "This statement is no longer supported"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 18b507c1b0..3d88eb5336 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -99,22 +99,24 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned char -#define YYNOCODE 220 +#define YYNOCODE 249 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SToken typedef union { int yyinit; ParseTOKENTYPE yy0; - SNodeList* yy24; - bool yy97; - SToken yy129; - SDataType yy224; - ENullOrder yy257; - EOperatorType yy260; - EFillMode yy294; - EJoinType yy332; - EOrder yy378; - SNode* yy432; + SNode* yy26; + EOrder yy32; + SNodeList* yy64; + EOperatorType yy80; + bool yy107; + EFillMode yy192; + SToken yy353; + SDataType yy370; + EJoinType yy372; + ENullOrder yy391; + SAlterOption yy443; + int32_t yy448; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -129,17 +131,17 @@ typedef union { #define ParseCTX_PARAM #define ParseCTX_FETCH #define ParseCTX_STORE -#define YYNSTATE 333 -#define YYNRULE 255 -#define YYNTOKEN 141 -#define YY_MAX_SHIFT 332 -#define YY_MIN_SHIFTREDUCE 501 -#define YY_MAX_SHIFTREDUCE 755 -#define YY_ERROR_ACTION 756 -#define YY_ACCEPT_ACTION 757 -#define YY_NO_ACTION 758 -#define YY_MIN_REDUCE 759 -#define YY_MAX_REDUCE 1013 +#define YYNSTATE 414 +#define YYNRULE 316 +#define YYNTOKEN 160 +#define YY_MAX_SHIFT 413 +#define YY_MIN_SHIFTREDUCE 631 +#define YY_MAX_SHIFTREDUCE 946 +#define YY_ERROR_ACTION 947 +#define YY_ACCEPT_ACTION 948 +#define YY_NO_ACTION 949 +#define YY_MIN_REDUCE 950 +#define YY_MAX_REDUCE 1265 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -206,322 +208,379 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (1012) +#define YY_ACTTAB_COUNT (1182) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 163, 856, 240, 855, 272, 162, 272, 200, 856, 814, - /* 10 */ 854, 878, 31, 29, 27, 26, 25, 65, 880, 817, - /* 20 */ 178, 817, 256, 176, 906, 31, 29, 27, 26, 25, - /* 30 */ 779, 878, 40, 705, 55, 819, 71, 63, 880, 62, - /* 40 */ 155, 229, 66, 812, 243, 809, 906, 79, 938, 939, - /* 50 */ 626, 943, 878, 155, 175, 258, 511, 863, 259, 891, - /* 60 */ 511, 638, 58, 892, 895, 931, 512, 513, 208, 165, - /* 70 */ 927, 77, 321, 320, 319, 318, 317, 316, 315, 314, - /* 80 */ 313, 312, 311, 310, 309, 308, 307, 306, 305, 775, - /* 90 */ 959, 27, 26, 25, 31, 29, 27, 26, 25, 23, - /* 100 */ 170, 271, 656, 657, 658, 659, 660, 661, 662, 664, - /* 110 */ 665, 666, 23, 170, 271, 656, 657, 658, 659, 660, - /* 120 */ 661, 662, 664, 665, 666, 31, 29, 27, 26, 25, - /* 130 */ 10, 321, 320, 319, 318, 317, 316, 315, 314, 313, - /* 140 */ 312, 311, 310, 309, 308, 307, 306, 305, 565, 295, - /* 150 */ 294, 293, 569, 292, 571, 572, 291, 574, 288, 804, - /* 160 */ 580, 285, 582, 583, 282, 279, 24, 117, 256, 196, - /* 170 */ 906, 992, 626, 195, 624, 256, 177, 906, 194, 856, - /* 180 */ 193, 854, 83, 878, 991, 83, 258, 233, 990, 244, - /* 190 */ 891, 180, 271, 57, 892, 895, 931, 54, 182, 190, - /* 200 */ 154, 927, 51, 819, 71, 272, 192, 191, 269, 301, - /* 210 */ 819, 71, 992, 300, 805, 243, 624, 906, 751, 752, - /* 220 */ 817, 10, 234, 878, 298, 82, 258, 945, 302, 990, - /* 230 */ 891, 757, 627, 58, 892, 895, 931, 256, 186, 906, - /* 240 */ 165, 927, 77, 228, 942, 878, 201, 299, 258, 628, - /* 250 */ 30, 28, 891, 201, 113, 58, 892, 895, 931, 617, - /* 260 */ 221, 958, 165, 927, 1004, 184, 256, 615, 906, 172, - /* 270 */ 629, 235, 230, 965, 878, 304, 878, 258, 12, 272, - /* 280 */ 297, 891, 270, 881, 58, 892, 895, 931, 992, 9, - /* 290 */ 8, 165, 927, 1004, 817, 256, 272, 906, 1, 130, - /* 300 */ 83, 82, 988, 878, 181, 990, 258, 856, 40, 854, - /* 310 */ 891, 817, 112, 58, 892, 895, 931, 273, 21, 813, - /* 320 */ 165, 927, 1004, 30, 28, 698, 272, 663, 189, 183, - /* 330 */ 667, 949, 617, 99, 616, 618, 621, 114, 30, 28, - /* 340 */ 615, 817, 172, 329, 328, 803, 247, 617, 256, 674, - /* 350 */ 906, 12, 722, 83, 140, 615, 878, 172, 161, 258, - /* 360 */ 141, 72, 244, 891, 30, 28, 144, 892, 895, 49, - /* 370 */ 945, 1, 945, 617, 84, 225, 720, 721, 723, 724, - /* 380 */ 810, 615, 226, 172, 304, 992, 7, 941, 6, 940, - /* 390 */ 273, 697, 12, 864, 259, 256, 248, 906, 82, 820, - /* 400 */ 71, 136, 990, 878, 847, 273, 258, 616, 618, 621, - /* 410 */ 891, 240, 1, 59, 892, 895, 931, 256, 107, 906, - /* 420 */ 930, 927, 616, 618, 621, 878, 65, 217, 258, 950, - /* 430 */ 693, 273, 891, 251, 69, 59, 892, 895, 931, 207, - /* 440 */ 244, 83, 254, 927, 213, 961, 63, 211, 616, 618, - /* 450 */ 621, 83, 94, 30, 28, 257, 110, 938, 239, 241, - /* 460 */ 238, 249, 617, 992, 719, 617, 30, 28, 693, 93, - /* 470 */ 615, 60, 172, 615, 246, 617, 82, 104, 907, 256, - /* 480 */ 990, 906, 668, 615, 102, 172, 116, 878, 255, 32, - /* 490 */ 258, 2, 754, 755, 891, 635, 624, 73, 892, 895, - /* 500 */ 252, 7, 32, 696, 607, 9, 8, 30, 28, 185, - /* 510 */ 256, 32, 906, 632, 7, 885, 617, 625, 878, 85, - /* 520 */ 273, 258, 883, 273, 615, 891, 172, 631, 59, 892, - /* 530 */ 895, 931, 214, 273, 245, 1005, 928, 616, 618, 621, - /* 540 */ 616, 618, 621, 197, 198, 256, 199, 906, 39, 653, - /* 550 */ 616, 618, 621, 878, 202, 1, 258, 171, 90, 122, - /* 560 */ 891, 215, 209, 148, 892, 895, 120, 256, 630, 906, - /* 570 */ 264, 134, 160, 127, 273, 878, 558, 67, 258, 222, - /* 580 */ 68, 216, 891, 69, 992, 148, 892, 895, 256, 218, - /* 590 */ 906, 616, 618, 621, 629, 553, 878, 82, 95, 258, - /* 600 */ 962, 990, 60, 891, 586, 133, 147, 892, 895, 590, - /* 610 */ 256, 277, 906, 61, 802, 227, 68, 219, 878, 325, - /* 620 */ 972, 258, 132, 262, 256, 891, 906, 100, 73, 892, - /* 630 */ 895, 621, 878, 224, 971, 258, 169, 236, 256, 891, - /* 640 */ 906, 103, 148, 892, 895, 56, 878, 164, 128, 258, - /* 650 */ 173, 5, 256, 891, 906, 952, 148, 892, 895, 237, - /* 660 */ 878, 75, 106, 258, 301, 223, 1006, 891, 300, 108, - /* 670 */ 146, 892, 895, 240, 268, 4, 220, 64, 256, 96, - /* 680 */ 906, 693, 628, 302, 595, 946, 878, 33, 65, 258, - /* 690 */ 253, 69, 256, 891, 906, 109, 149, 892, 895, 70, - /* 700 */ 878, 989, 299, 258, 1007, 250, 68, 891, 63, 166, - /* 710 */ 142, 892, 895, 256, 17, 906, 115, 242, 78, 938, - /* 720 */ 939, 878, 943, 913, 258, 862, 260, 256, 891, 906, - /* 730 */ 261, 150, 892, 895, 861, 878, 265, 174, 258, 266, - /* 740 */ 124, 256, 891, 906, 267, 143, 892, 895, 48, 878, - /* 750 */ 135, 818, 258, 50, 275, 256, 891, 906, 137, 151, - /* 760 */ 892, 895, 131, 878, 872, 138, 258, 332, 152, 256, - /* 770 */ 891, 906, 153, 903, 892, 895, 139, 878, 778, 871, - /* 780 */ 258, 870, 187, 256, 891, 906, 188, 902, 892, 895, - /* 790 */ 869, 878, 808, 807, 258, 777, 774, 256, 891, 906, - /* 800 */ 768, 901, 892, 895, 763, 878, 868, 859, 258, 87, - /* 810 */ 806, 256, 891, 906, 524, 158, 892, 895, 776, 878, - /* 820 */ 773, 205, 258, 203, 204, 256, 891, 906, 767, 157, - /* 830 */ 892, 895, 766, 878, 762, 761, 258, 210, 760, 867, - /* 840 */ 891, 866, 212, 159, 892, 895, 36, 858, 256, 42, - /* 850 */ 906, 97, 98, 3, 240, 14, 878, 15, 74, 258, - /* 860 */ 101, 105, 256, 891, 906, 43, 156, 892, 895, 65, - /* 870 */ 878, 32, 231, 258, 718, 712, 92, 891, 34, 37, - /* 880 */ 145, 892, 895, 711, 76, 44, 11, 232, 690, 63, - /* 890 */ 206, 883, 689, 91, 19, 45, 20, 111, 745, 80, - /* 900 */ 938, 939, 740, 943, 35, 739, 31, 29, 27, 26, - /* 910 */ 25, 22, 81, 167, 16, 744, 41, 743, 168, 89, - /* 920 */ 8, 31, 29, 27, 26, 25, 118, 636, 13, 31, - /* 930 */ 29, 27, 26, 25, 654, 18, 263, 857, 119, 716, - /* 940 */ 121, 123, 46, 126, 125, 47, 88, 619, 51, 276, - /* 950 */ 86, 179, 38, 280, 882, 579, 587, 129, 278, 274, - /* 960 */ 584, 281, 283, 581, 284, 286, 575, 287, 289, 573, - /* 970 */ 564, 290, 52, 53, 594, 296, 593, 578, 638, 592, - /* 980 */ 303, 577, 522, 543, 576, 542, 541, 536, 540, 539, - /* 990 */ 538, 537, 535, 534, 533, 772, 532, 531, 530, 529, - /* 1000 */ 528, 527, 322, 323, 765, 764, 326, 327, 324, 759, - /* 1010 */ 330, 331, + /* 0 */ 996, 347, 347, 24, 162, 334, 253, 1057, 227, 1049, + /* 10 */ 108, 1115, 1158, 31, 29, 27, 26, 25, 951, 331, + /* 20 */ 347, 1098, 1060, 1060, 103, 64, 962, 31, 29, 27, + /* 30 */ 26, 25, 263, 233, 1051, 973, 1106, 1108, 304, 76, + /* 40 */ 346, 1060, 75, 74, 73, 72, 71, 70, 69, 68, + /* 50 */ 67, 201, 398, 397, 396, 395, 394, 393, 392, 391, + /* 60 */ 390, 389, 388, 387, 386, 385, 384, 383, 382, 381, + /* 70 */ 380, 1130, 1130, 30, 28, 829, 31, 29, 27, 26, + /* 80 */ 25, 224, 346, 808, 76, 852, 42, 75, 74, 73, + /* 90 */ 72, 71, 70, 69, 68, 67, 275, 102, 270, 42, + /* 100 */ 806, 274, 1244, 1056, 273, 1063, 271, 1105, 88, 272, + /* 110 */ 289, 12, 950, 214, 201, 1243, 1055, 815, 1103, 1242, + /* 120 */ 23, 222, 346, 847, 848, 849, 850, 851, 853, 855, + /* 130 */ 856, 857, 807, 254, 1, 10, 86, 85, 84, 83, + /* 140 */ 82, 81, 80, 79, 78, 1132, 290, 303, 852, 747, + /* 150 */ 369, 368, 367, 751, 366, 753, 754, 365, 756, 362, + /* 160 */ 410, 762, 359, 764, 765, 356, 353, 287, 1244, 215, + /* 170 */ 942, 943, 817, 310, 305, 10, 913, 1143, 809, 812, + /* 180 */ 285, 116, 1130, 23, 222, 1242, 847, 848, 849, 850, + /* 190 */ 851, 853, 855, 856, 857, 117, 117, 1158, 300, 911, + /* 200 */ 912, 914, 915, 1132, 331, 31, 29, 27, 26, 25, + /* 210 */ 333, 347, 347, 888, 1130, 1143, 344, 345, 815, 319, + /* 220 */ 105, 1025, 60, 1144, 1147, 1183, 948, 230, 249, 200, + /* 230 */ 1179, 117, 1060, 1060, 347, 1158, 972, 292, 248, 64, + /* 240 */ 1130, 1244, 318, 247, 315, 246, 269, 309, 333, 379, + /* 250 */ 1158, 96, 1130, 1197, 116, 1060, 180, 331, 1242, 1090, + /* 260 */ 61, 1144, 1147, 1183, 243, 91, 1105, 217, 1179, 111, + /* 270 */ 1194, 237, 229, 1130, 245, 244, 308, 1103, 1143, 406, + /* 280 */ 405, 158, 30, 28, 889, 89, 347, 296, 1210, 819, + /* 290 */ 224, 175, 808, 1244, 317, 112, 1190, 1191, 1158, 1195, + /* 300 */ 228, 315, 27, 26, 25, 318, 116, 1060, 102, 806, + /* 310 */ 1242, 333, 51, 1143, 239, 1130, 1062, 58, 1202, 884, + /* 320 */ 12, 818, 91, 61, 1144, 1147, 1183, 92, 1036, 1053, + /* 330 */ 217, 1179, 111, 1158, 1052, 31, 29, 27, 26, 25, + /* 340 */ 331, 807, 89, 1, 887, 267, 333, 232, 1143, 266, + /* 350 */ 1130, 1211, 113, 1190, 1191, 102, 1195, 910, 61, 1144, + /* 360 */ 1147, 1183, 235, 1062, 117, 217, 1179, 1256, 1158, 410, + /* 370 */ 102, 77, 372, 971, 268, 331, 1217, 254, 1062, 1143, + /* 380 */ 1105, 333, 30, 28, 157, 1130, 234, 809, 812, 1035, + /* 390 */ 224, 1103, 808, 61, 1144, 1147, 1183, 829, 347, 1158, + /* 400 */ 217, 1179, 1256, 236, 9, 8, 331, 30, 28, 806, + /* 410 */ 1130, 1240, 333, 30, 28, 224, 1130, 808, 1143, 1060, + /* 420 */ 12, 224, 334, 808, 61, 1144, 1147, 1183, 1116, 1197, + /* 430 */ 808, 217, 1179, 1256, 806, 669, 970, 670, 1158, 669, + /* 440 */ 806, 807, 1201, 1, 1045, 331, 1193, 806, 1047, 376, + /* 450 */ 1105, 333, 261, 375, 969, 1130, 1143, 1034, 671, 21, + /* 460 */ 319, 1107, 322, 190, 1144, 1147, 807, 854, 7, 410, + /* 470 */ 858, 149, 807, 1130, 7, 1105, 1158, 865, 377, 807, + /* 480 */ 57, 6, 1244, 331, 968, 147, 1104, 809, 812, 333, + /* 490 */ 1143, 1130, 53, 1130, 410, 116, 967, 374, 373, 1242, + /* 500 */ 410, 62, 1144, 1147, 1183, 820, 379, 410, 1182, 1179, + /* 510 */ 1158, 315, 809, 812, 30, 28, 332, 331, 809, 812, + /* 520 */ 966, 1130, 224, 333, 808, 809, 812, 1130, 896, 117, + /* 530 */ 965, 964, 91, 1130, 817, 62, 1144, 1147, 1183, 1133, + /* 540 */ 961, 806, 329, 1179, 960, 30, 28, 319, 945, 946, + /* 550 */ 959, 1143, 89, 224, 958, 808, 957, 1130, 131, 9, + /* 560 */ 8, 129, 155, 1190, 314, 413, 313, 1130, 1130, 1244, + /* 570 */ 884, 1158, 806, 807, 956, 7, 1130, 1130, 331, 178, + /* 580 */ 1043, 1130, 116, 87, 333, 955, 1242, 1130, 1130, 402, + /* 590 */ 1143, 1130, 177, 1130, 954, 323, 106, 1144, 1147, 330, + /* 600 */ 144, 410, 953, 126, 807, 326, 1, 110, 1197, 321, + /* 610 */ 1158, 1130, 371, 259, 963, 242, 125, 331, 59, 809, + /* 620 */ 812, 173, 1130, 333, 1026, 1192, 133, 1130, 1143, 132, + /* 630 */ 159, 1130, 410, 320, 1257, 62, 1144, 1147, 1183, 1130, + /* 640 */ 991, 135, 43, 1180, 134, 123, 137, 301, 1158, 136, + /* 650 */ 809, 812, 1143, 859, 343, 331, 986, 295, 152, 324, + /* 660 */ 141, 333, 276, 1143, 185, 1130, 984, 32, 223, 187, + /* 670 */ 844, 327, 1158, 196, 1144, 1147, 1099, 826, 278, 331, + /* 680 */ 122, 186, 104, 1158, 120, 333, 1143, 260, 281, 1130, + /* 690 */ 331, 32, 297, 1137, 118, 1213, 333, 196, 1144, 1147, + /* 700 */ 1130, 1143, 316, 798, 167, 1159, 1158, 1135, 195, 1144, + /* 710 */ 1147, 1033, 339, 331, 161, 172, 1143, 32, 165, 333, + /* 720 */ 2, 1158, 250, 1130, 238, 815, 93, 823, 331, 94, + /* 730 */ 119, 106, 1144, 1147, 333, 816, 1158, 1143, 1130, 311, + /* 740 */ 251, 221, 252, 331, 740, 735, 196, 1144, 1147, 333, + /* 750 */ 822, 41, 1143, 1130, 255, 124, 225, 1158, 96, 77, + /* 760 */ 821, 196, 1144, 1147, 331, 262, 264, 768, 1143, 1258, + /* 770 */ 333, 376, 1158, 772, 1130, 375, 778, 1050, 777, 331, + /* 780 */ 117, 351, 194, 1144, 1147, 333, 66, 94, 1158, 1130, + /* 790 */ 95, 1143, 96, 128, 97, 331, 1143, 197, 1144, 1147, + /* 800 */ 377, 333, 1046, 291, 130, 1130, 98, 1143, 94, 99, + /* 810 */ 1048, 1158, 1143, 188, 1144, 1147, 1158, 1044, 331, 374, + /* 820 */ 373, 140, 100, 331, 333, 101, 213, 1158, 1130, 333, + /* 830 */ 820, 1214, 1158, 1130, 331, 293, 198, 1144, 1147, 331, + /* 840 */ 333, 189, 1144, 1147, 1130, 333, 1224, 1143, 294, 1130, + /* 850 */ 1143, 302, 199, 1144, 1147, 337, 145, 1155, 1144, 1147, + /* 860 */ 812, 148, 299, 216, 298, 5, 1204, 1158, 1223, 312, + /* 870 */ 1158, 4, 884, 90, 331, 819, 151, 331, 109, 1198, + /* 880 */ 333, 33, 153, 333, 1130, 1143, 154, 1130, 218, 328, + /* 890 */ 17, 1241, 1154, 1144, 1147, 1153, 1144, 1147, 1259, 325, + /* 900 */ 1143, 160, 1165, 340, 1114, 1158, 335, 169, 336, 50, + /* 910 */ 341, 1113, 331, 226, 1061, 342, 1143, 179, 333, 52, + /* 920 */ 1158, 1143, 1130, 315, 181, 176, 409, 331, 191, 349, + /* 930 */ 204, 1144, 1147, 333, 183, 184, 1158, 1130, 1124, 192, + /* 940 */ 999, 1158, 1123, 331, 91, 203, 1144, 1147, 331, 333, + /* 950 */ 1122, 240, 1143, 1130, 333, 241, 1121, 208, 1130, 1039, + /* 960 */ 1038, 205, 1144, 1147, 89, 998, 202, 1144, 1147, 1000, + /* 970 */ 995, 983, 1158, 121, 114, 1190, 1191, 280, 1195, 331, + /* 980 */ 978, 1120, 267, 1111, 1037, 333, 266, 684, 997, 1130, + /* 990 */ 994, 256, 288, 258, 257, 982, 981, 193, 1144, 1147, + /* 1000 */ 209, 977, 207, 206, 1041, 265, 139, 65, 127, 781, + /* 1010 */ 283, 268, 1040, 783, 782, 277, 713, 712, 711, 138, + /* 1020 */ 992, 275, 210, 270, 710, 709, 274, 987, 708, 273, + /* 1030 */ 211, 271, 279, 985, 272, 212, 282, 976, 284, 975, + /* 1040 */ 286, 63, 1119, 1118, 1110, 38, 36, 142, 37, 44, + /* 1050 */ 3, 20, 32, 143, 14, 39, 146, 15, 306, 34, + /* 1060 */ 307, 31, 29, 27, 26, 25, 22, 909, 903, 47, + /* 1070 */ 11, 107, 150, 931, 930, 45, 31, 29, 27, 26, + /* 1080 */ 25, 902, 46, 8, 881, 880, 219, 935, 1109, 934, + /* 1090 */ 220, 993, 1135, 980, 19, 949, 936, 170, 827, 164, + /* 1100 */ 350, 156, 13, 35, 115, 18, 231, 354, 357, 360, + /* 1110 */ 907, 16, 845, 163, 166, 363, 53, 746, 168, 48, + /* 1120 */ 774, 704, 776, 775, 348, 49, 400, 1134, 769, 40, + /* 1130 */ 352, 399, 766, 355, 401, 378, 763, 757, 682, 358, + /* 1140 */ 171, 338, 174, 703, 361, 702, 755, 364, 701, 700, + /* 1150 */ 699, 698, 54, 55, 697, 56, 696, 705, 695, 694, + /* 1160 */ 761, 693, 692, 691, 690, 689, 403, 370, 688, 687, + /* 1170 */ 404, 979, 974, 407, 760, 408, 759, 810, 758, 182, + /* 1180 */ 411, 412, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 158, 163, 146, 165, 146, 160, 146, 149, 163, 149, - /* 10 */ 165, 169, 12, 13, 14, 15, 16, 161, 176, 161, - /* 20 */ 158, 161, 161, 150, 163, 12, 13, 14, 15, 16, - /* 30 */ 0, 169, 148, 14, 145, 162, 163, 181, 176, 155, - /* 40 */ 40, 180, 153, 159, 161, 156, 163, 191, 192, 193, - /* 50 */ 31, 195, 169, 40, 168, 172, 21, 171, 172, 176, - /* 60 */ 21, 61, 179, 180, 181, 182, 31, 32, 29, 186, - /* 70 */ 187, 188, 42, 43, 44, 45, 46, 47, 48, 49, - /* 80 */ 50, 51, 52, 53, 54, 55, 56, 57, 58, 0, - /* 90 */ 207, 14, 15, 16, 12, 13, 14, 15, 16, 99, - /* 100 */ 100, 31, 102, 103, 104, 105, 106, 107, 108, 109, - /* 110 */ 110, 111, 99, 100, 31, 102, 103, 104, 105, 106, - /* 120 */ 107, 108, 109, 110, 111, 12, 13, 14, 15, 16, - /* 130 */ 60, 42, 43, 44, 45, 46, 47, 48, 49, 50, - /* 140 */ 51, 52, 53, 54, 55, 56, 57, 58, 70, 71, - /* 150 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 0, - /* 160 */ 82, 83, 84, 85, 86, 87, 183, 184, 161, 26, - /* 170 */ 163, 198, 31, 30, 31, 161, 160, 163, 35, 163, - /* 180 */ 37, 165, 115, 169, 211, 115, 172, 180, 215, 175, - /* 190 */ 176, 150, 31, 179, 180, 181, 182, 60, 150, 56, - /* 200 */ 186, 187, 65, 162, 163, 146, 63, 64, 149, 50, - /* 210 */ 162, 163, 198, 54, 0, 161, 31, 163, 136, 137, - /* 220 */ 161, 60, 31, 169, 66, 211, 172, 177, 69, 215, - /* 230 */ 176, 141, 31, 179, 180, 181, 182, 161, 95, 163, - /* 240 */ 186, 187, 188, 92, 194, 169, 39, 88, 172, 31, - /* 250 */ 12, 13, 176, 39, 200, 179, 180, 181, 182, 21, - /* 260 */ 206, 207, 186, 187, 188, 175, 161, 29, 163, 31, - /* 270 */ 31, 120, 121, 197, 169, 39, 169, 172, 40, 146, - /* 280 */ 164, 176, 149, 176, 179, 180, 181, 182, 198, 1, - /* 290 */ 2, 186, 187, 188, 161, 161, 146, 163, 60, 149, - /* 300 */ 115, 211, 197, 169, 160, 215, 172, 163, 148, 165, - /* 310 */ 176, 161, 94, 179, 180, 181, 182, 79, 99, 159, - /* 320 */ 186, 187, 188, 12, 13, 14, 146, 108, 146, 149, - /* 330 */ 111, 197, 21, 94, 96, 97, 98, 218, 12, 13, - /* 340 */ 29, 161, 31, 143, 144, 0, 3, 21, 161, 61, - /* 350 */ 163, 40, 101, 115, 18, 29, 169, 31, 22, 172, - /* 360 */ 24, 25, 175, 176, 12, 13, 179, 180, 181, 145, - /* 370 */ 177, 60, 177, 21, 38, 124, 125, 126, 127, 128, - /* 380 */ 156, 29, 209, 31, 39, 198, 60, 194, 34, 194, - /* 390 */ 79, 4, 40, 171, 172, 161, 68, 163, 211, 162, - /* 400 */ 163, 151, 215, 169, 154, 79, 172, 96, 97, 98, - /* 410 */ 176, 146, 60, 179, 180, 181, 182, 161, 203, 163, - /* 420 */ 186, 187, 96, 97, 98, 169, 161, 61, 172, 113, - /* 430 */ 114, 79, 176, 68, 68, 179, 180, 181, 182, 143, - /* 440 */ 175, 115, 186, 187, 20, 178, 181, 23, 96, 97, - /* 450 */ 98, 115, 19, 12, 13, 14, 191, 192, 193, 196, - /* 460 */ 195, 133, 21, 198, 61, 21, 12, 13, 114, 36, - /* 470 */ 29, 68, 31, 29, 131, 21, 211, 61, 163, 161, - /* 480 */ 215, 163, 61, 29, 68, 31, 212, 169, 40, 68, - /* 490 */ 172, 199, 139, 140, 176, 61, 31, 179, 180, 181, - /* 500 */ 135, 60, 68, 116, 61, 1, 2, 12, 13, 146, - /* 510 */ 161, 68, 163, 31, 60, 60, 21, 31, 169, 148, - /* 520 */ 79, 172, 67, 79, 29, 176, 31, 31, 179, 180, - /* 530 */ 181, 182, 146, 79, 216, 217, 187, 96, 97, 98, - /* 540 */ 96, 97, 98, 174, 161, 161, 166, 163, 148, 101, - /* 550 */ 96, 97, 98, 169, 146, 60, 172, 173, 148, 61, - /* 560 */ 176, 175, 142, 179, 180, 181, 68, 161, 31, 163, - /* 570 */ 61, 146, 142, 61, 79, 169, 61, 68, 172, 173, - /* 580 */ 68, 174, 176, 68, 198, 179, 180, 181, 161, 161, - /* 590 */ 163, 96, 97, 98, 31, 61, 169, 211, 145, 172, - /* 600 */ 178, 215, 68, 176, 61, 19, 179, 180, 181, 61, - /* 610 */ 161, 68, 163, 27, 0, 123, 68, 166, 169, 33, - /* 620 */ 208, 172, 36, 122, 161, 176, 163, 170, 179, 180, - /* 630 */ 181, 98, 169, 169, 208, 172, 173, 210, 161, 176, - /* 640 */ 163, 170, 179, 180, 181, 59, 169, 169, 62, 172, - /* 650 */ 173, 130, 161, 176, 163, 205, 179, 180, 181, 129, - /* 660 */ 169, 202, 204, 172, 50, 118, 217, 176, 54, 201, - /* 670 */ 179, 180, 181, 146, 88, 117, 90, 161, 161, 93, - /* 680 */ 163, 114, 31, 69, 61, 177, 169, 112, 161, 172, - /* 690 */ 134, 68, 161, 176, 163, 189, 179, 180, 181, 61, - /* 700 */ 169, 214, 88, 172, 219, 132, 68, 176, 181, 138, - /* 710 */ 179, 180, 181, 161, 60, 163, 213, 190, 191, 192, - /* 720 */ 193, 169, 195, 185, 172, 170, 169, 161, 176, 163, - /* 730 */ 169, 179, 180, 181, 170, 169, 91, 169, 172, 167, - /* 740 */ 161, 161, 176, 163, 166, 179, 180, 181, 145, 169, - /* 750 */ 154, 161, 172, 60, 157, 161, 176, 163, 146, 179, - /* 760 */ 180, 181, 145, 169, 0, 147, 172, 142, 152, 161, - /* 770 */ 176, 163, 152, 179, 180, 181, 147, 169, 0, 0, - /* 780 */ 172, 0, 56, 161, 176, 163, 67, 179, 180, 181, - /* 790 */ 0, 169, 0, 0, 172, 0, 0, 161, 176, 163, - /* 800 */ 0, 179, 180, 181, 0, 169, 0, 0, 172, 34, - /* 810 */ 0, 161, 176, 163, 41, 179, 180, 181, 0, 169, - /* 820 */ 0, 34, 172, 29, 27, 161, 176, 163, 0, 179, - /* 830 */ 180, 181, 0, 169, 0, 0, 172, 21, 0, 0, - /* 840 */ 176, 0, 21, 179, 180, 181, 94, 0, 161, 60, - /* 850 */ 163, 34, 89, 68, 146, 119, 169, 119, 60, 172, - /* 860 */ 61, 60, 161, 176, 163, 60, 179, 180, 181, 161, - /* 870 */ 169, 68, 29, 172, 61, 61, 19, 176, 113, 68, - /* 880 */ 179, 180, 181, 61, 27, 60, 119, 68, 61, 181, - /* 890 */ 33, 67, 61, 36, 68, 4, 2, 67, 61, 191, - /* 900 */ 192, 193, 29, 195, 68, 29, 12, 13, 14, 15, - /* 910 */ 16, 2, 67, 29, 68, 29, 59, 29, 29, 62, - /* 920 */ 2, 12, 13, 14, 15, 16, 67, 61, 60, 12, - /* 930 */ 13, 14, 15, 16, 101, 60, 92, 0, 61, 61, - /* 940 */ 60, 60, 60, 89, 34, 60, 89, 21, 65, 29, - /* 950 */ 93, 29, 60, 29, 67, 81, 61, 67, 60, 66, - /* 960 */ 61, 60, 29, 61, 60, 29, 61, 60, 29, 61, - /* 970 */ 21, 60, 60, 60, 29, 69, 29, 81, 61, 21, - /* 980 */ 40, 81, 41, 29, 81, 29, 29, 21, 29, 29, - /* 990 */ 29, 29, 29, 29, 29, 0, 29, 29, 29, 29, - /* 1000 */ 29, 29, 29, 27, 0, 0, 29, 28, 34, 0, - /* 1010 */ 21, 20, 220, 220, 220, 220, 220, 220, 220, 220, - /* 1020 */ 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, - /* 1030 */ 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, - /* 1040 */ 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, - /* 1050 */ 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, - /* 1060 */ 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, - /* 1070 */ 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, - /* 1080 */ 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, - /* 1090 */ 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, - /* 1100 */ 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, - /* 1110 */ 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, - /* 1120 */ 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, - /* 1130 */ 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, - /* 1140 */ 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, - /* 1150 */ 220, 220, 220, + /* 0 */ 0, 169, 169, 212, 213, 196, 174, 174, 199, 184, + /* 10 */ 182, 202, 183, 12, 13, 14, 15, 16, 0, 190, + /* 20 */ 169, 193, 190, 190, 162, 174, 164, 12, 13, 14, + /* 30 */ 15, 16, 181, 192, 163, 163, 195, 196, 209, 21, + /* 40 */ 20, 190, 24, 25, 26, 27, 28, 29, 30, 31, + /* 50 */ 32, 50, 52, 53, 54, 55, 56, 57, 58, 59, + /* 60 */ 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + /* 70 */ 70, 200, 200, 12, 13, 74, 12, 13, 14, 15, + /* 80 */ 16, 20, 20, 22, 21, 84, 171, 24, 25, 26, + /* 90 */ 27, 28, 29, 30, 31, 32, 52, 183, 54, 171, + /* 100 */ 39, 57, 227, 188, 60, 191, 62, 183, 180, 65, + /* 110 */ 169, 50, 0, 189, 50, 240, 188, 20, 194, 244, + /* 120 */ 119, 120, 20, 122, 123, 124, 125, 126, 127, 128, + /* 130 */ 129, 130, 71, 49, 73, 73, 24, 25, 26, 27, + /* 140 */ 28, 29, 30, 31, 32, 163, 205, 113, 84, 90, + /* 150 */ 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, + /* 160 */ 99, 102, 103, 104, 105, 106, 107, 21, 227, 187, + /* 170 */ 155, 156, 20, 139, 140, 73, 121, 163, 117, 118, + /* 180 */ 34, 240, 200, 119, 120, 244, 122, 123, 124, 125, + /* 190 */ 126, 127, 128, 129, 130, 134, 134, 183, 143, 144, + /* 200 */ 145, 146, 147, 163, 190, 12, 13, 14, 15, 16, + /* 210 */ 196, 169, 169, 4, 200, 163, 174, 174, 20, 205, + /* 220 */ 172, 173, 208, 209, 210, 211, 160, 187, 30, 215, + /* 230 */ 216, 134, 190, 190, 169, 183, 163, 74, 40, 174, + /* 240 */ 200, 227, 190, 45, 169, 47, 181, 20, 196, 49, + /* 250 */ 183, 88, 200, 206, 240, 190, 176, 190, 244, 179, + /* 260 */ 208, 209, 210, 211, 66, 190, 183, 215, 216, 217, + /* 270 */ 223, 205, 189, 200, 76, 77, 209, 194, 163, 166, + /* 280 */ 167, 229, 12, 13, 14, 210, 169, 235, 236, 20, + /* 290 */ 20, 174, 22, 227, 219, 220, 221, 222, 183, 224, + /* 300 */ 175, 169, 14, 15, 16, 190, 240, 190, 183, 39, + /* 310 */ 244, 196, 168, 163, 116, 200, 191, 168, 132, 133, + /* 320 */ 50, 20, 190, 208, 209, 210, 211, 178, 0, 185, + /* 330 */ 215, 216, 217, 183, 185, 12, 13, 14, 15, 16, + /* 340 */ 190, 71, 210, 73, 135, 60, 196, 175, 163, 64, + /* 350 */ 200, 236, 220, 221, 222, 183, 224, 74, 208, 209, + /* 360 */ 210, 211, 175, 191, 134, 215, 216, 217, 183, 99, + /* 370 */ 183, 88, 86, 163, 89, 190, 226, 49, 191, 163, + /* 380 */ 183, 196, 12, 13, 115, 200, 189, 117, 118, 0, + /* 390 */ 20, 194, 22, 208, 209, 210, 211, 74, 169, 183, + /* 400 */ 215, 216, 217, 174, 1, 2, 190, 12, 13, 39, + /* 410 */ 200, 226, 196, 12, 13, 20, 200, 22, 163, 190, + /* 420 */ 50, 20, 196, 22, 208, 209, 210, 211, 202, 206, + /* 430 */ 22, 215, 216, 217, 39, 22, 163, 20, 183, 22, + /* 440 */ 39, 71, 226, 73, 184, 190, 223, 39, 184, 60, + /* 450 */ 183, 196, 39, 64, 163, 200, 163, 0, 41, 119, + /* 460 */ 205, 194, 3, 208, 209, 210, 71, 127, 73, 99, + /* 470 */ 130, 74, 71, 200, 73, 183, 183, 74, 89, 71, + /* 480 */ 73, 44, 227, 190, 163, 88, 194, 117, 118, 196, + /* 490 */ 163, 200, 85, 200, 99, 240, 163, 108, 109, 244, + /* 500 */ 99, 208, 209, 210, 211, 20, 49, 99, 215, 216, + /* 510 */ 183, 169, 117, 118, 12, 13, 14, 190, 117, 118, + /* 520 */ 163, 200, 20, 196, 22, 117, 118, 200, 14, 134, + /* 530 */ 163, 163, 190, 200, 20, 208, 209, 210, 211, 163, + /* 540 */ 163, 39, 215, 216, 163, 12, 13, 205, 158, 159, + /* 550 */ 163, 163, 210, 20, 163, 22, 163, 200, 79, 1, + /* 560 */ 2, 82, 220, 221, 222, 19, 224, 200, 200, 227, + /* 570 */ 133, 183, 39, 71, 163, 73, 200, 200, 190, 33, + /* 580 */ 184, 200, 240, 37, 196, 163, 244, 200, 200, 43, + /* 590 */ 163, 200, 46, 200, 163, 88, 208, 209, 210, 50, + /* 600 */ 115, 99, 163, 33, 71, 88, 73, 37, 206, 150, + /* 610 */ 183, 200, 184, 43, 164, 169, 46, 190, 72, 117, + /* 620 */ 118, 75, 200, 196, 173, 223, 79, 200, 163, 82, + /* 630 */ 247, 200, 99, 245, 246, 208, 209, 210, 211, 200, + /* 640 */ 0, 79, 72, 216, 82, 75, 79, 238, 183, 82, + /* 650 */ 117, 118, 163, 74, 108, 190, 0, 111, 232, 152, + /* 660 */ 114, 196, 22, 163, 18, 200, 0, 88, 203, 23, + /* 670 */ 121, 154, 183, 208, 209, 210, 193, 74, 22, 190, + /* 680 */ 110, 35, 36, 183, 114, 196, 163, 166, 22, 200, + /* 690 */ 190, 88, 203, 73, 48, 207, 196, 208, 209, 210, + /* 700 */ 200, 163, 225, 74, 74, 183, 183, 87, 208, 209, + /* 710 */ 210, 0, 74, 190, 241, 74, 163, 88, 88, 196, + /* 720 */ 228, 183, 204, 200, 169, 20, 88, 20, 190, 88, + /* 730 */ 171, 208, 209, 210, 196, 20, 183, 163, 200, 239, + /* 740 */ 190, 203, 197, 190, 74, 74, 208, 209, 210, 196, + /* 750 */ 20, 171, 163, 200, 169, 171, 203, 183, 88, 88, + /* 760 */ 20, 208, 209, 210, 190, 165, 183, 74, 163, 246, + /* 770 */ 196, 60, 183, 74, 200, 64, 74, 183, 74, 190, + /* 780 */ 134, 88, 208, 209, 210, 196, 169, 88, 183, 200, + /* 790 */ 88, 163, 88, 183, 74, 190, 163, 208, 209, 210, + /* 800 */ 89, 196, 183, 204, 183, 200, 183, 163, 88, 183, + /* 810 */ 183, 183, 163, 208, 209, 210, 183, 183, 190, 108, + /* 820 */ 109, 168, 183, 190, 196, 183, 165, 183, 200, 196, + /* 830 */ 20, 207, 183, 200, 190, 190, 208, 209, 210, 190, + /* 840 */ 196, 208, 209, 210, 200, 196, 237, 163, 197, 200, + /* 850 */ 163, 142, 208, 209, 210, 141, 201, 208, 209, 210, + /* 860 */ 118, 201, 200, 200, 137, 149, 234, 183, 237, 148, + /* 870 */ 183, 136, 133, 190, 190, 20, 233, 190, 231, 206, + /* 880 */ 196, 131, 230, 196, 200, 163, 218, 200, 157, 153, + /* 890 */ 73, 243, 208, 209, 210, 208, 209, 210, 248, 151, + /* 900 */ 163, 242, 214, 112, 201, 183, 200, 190, 200, 168, + /* 910 */ 198, 201, 190, 200, 190, 197, 163, 179, 196, 73, + /* 920 */ 183, 163, 200, 169, 169, 168, 165, 190, 177, 186, + /* 930 */ 208, 209, 210, 196, 170, 161, 183, 200, 0, 177, + /* 940 */ 0, 183, 0, 190, 190, 208, 209, 210, 190, 196, + /* 950 */ 0, 66, 163, 200, 196, 87, 0, 35, 200, 0, + /* 960 */ 0, 208, 209, 210, 210, 0, 208, 209, 210, 0, + /* 970 */ 0, 0, 183, 44, 220, 221, 222, 4, 224, 190, + /* 980 */ 0, 0, 60, 0, 0, 196, 64, 51, 0, 200, + /* 990 */ 0, 39, 19, 44, 37, 0, 0, 208, 209, 210, + /* 1000 */ 78, 0, 80, 81, 0, 83, 33, 84, 82, 22, + /* 1010 */ 37, 89, 0, 39, 39, 42, 39, 39, 39, 46, + /* 1020 */ 0, 52, 22, 54, 39, 39, 57, 0, 39, 60, + /* 1030 */ 22, 62, 40, 0, 65, 22, 39, 0, 22, 0, + /* 1040 */ 22, 20, 0, 0, 0, 72, 115, 44, 75, 73, + /* 1050 */ 88, 2, 88, 110, 138, 88, 74, 138, 39, 132, + /* 1060 */ 88, 12, 13, 14, 15, 16, 2, 74, 74, 4, + /* 1070 */ 138, 73, 73, 39, 39, 73, 12, 13, 14, 15, + /* 1080 */ 16, 74, 73, 2, 74, 74, 39, 39, 0, 39, + /* 1090 */ 39, 0, 87, 0, 88, 249, 74, 44, 74, 74, + /* 1100 */ 39, 87, 73, 88, 87, 73, 39, 39, 39, 39, + /* 1110 */ 74, 88, 121, 87, 73, 39, 85, 22, 73, 73, + /* 1120 */ 22, 22, 39, 39, 86, 73, 37, 87, 74, 73, + /* 1130 */ 73, 39, 74, 73, 44, 50, 74, 74, 51, 73, + /* 1140 */ 110, 113, 87, 39, 73, 39, 74, 73, 39, 39, + /* 1150 */ 39, 39, 73, 73, 39, 73, 22, 71, 39, 39, + /* 1160 */ 101, 39, 39, 39, 39, 39, 39, 89, 39, 39, + /* 1170 */ 38, 0, 0, 22, 101, 21, 101, 22, 101, 22, + /* 1180 */ 21, 20, 249, 249, 249, 249, 249, 249, 249, 249, + /* 1190 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, + /* 1200 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, + /* 1210 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, + /* 1220 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, + /* 1230 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, + /* 1240 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, + /* 1250 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, + /* 1260 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, + /* 1270 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, + /* 1280 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, + /* 1290 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, + /* 1300 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, + /* 1310 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, + /* 1320 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, + /* 1330 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, + /* 1340 */ 249, 249, }; -#define YY_SHIFT_COUNT (332) +#define YY_SHIFT_COUNT (413) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (1009) +#define YY_SHIFT_MAX (1172) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 336, 238, 311, 352, 352, 352, 352, 326, 352, 352, - /* 10 */ 70, 454, 495, 441, 454, 454, 454, 454, 454, 454, - /* 20 */ 454, 454, 454, 454, 454, 454, 454, 454, 454, 454, - /* 30 */ 454, 454, 454, 161, 161, 161, 185, 444, 444, 83, - /* 40 */ 83, 207, 141, 191, 191, 67, 201, 141, 83, 83, - /* 50 */ 141, 83, 141, 141, 141, 83, 236, 0, 13, 13, - /* 60 */ 444, 35, 214, 218, 218, 218, 345, 201, 141, 141, - /* 70 */ 158, 78, 143, 82, 251, 151, 39, 239, 316, 354, - /* 80 */ 316, 19, 343, 387, 465, 482, 207, 486, 496, 207, - /* 90 */ 465, 207, 537, 465, 537, 482, 236, 486, 496, 563, - /* 100 */ 492, 501, 533, 492, 501, 533, 521, 530, 547, 558, - /* 110 */ 567, 486, 651, 575, 571, 556, 573, 654, 141, 501, - /* 120 */ 533, 533, 501, 533, 645, 486, 496, 158, 236, 486, - /* 130 */ 693, 465, 236, 537, 1012, 1012, 1012, 1012, 30, 89, - /* 140 */ 586, 857, 894, 909, 917, 113, 113, 113, 113, 113, - /* 150 */ 113, 113, 159, 614, 288, 219, 77, 77, 77, 77, - /* 160 */ 424, 433, 366, 403, 416, 504, 353, 328, 365, 421, - /* 170 */ 448, 434, 455, 443, 498, 509, 512, 515, 534, 543, - /* 180 */ 548, 623, 638, 137, 764, 778, 779, 781, 726, 719, - /* 190 */ 790, 792, 793, 795, 796, 800, 804, 806, 807, 775, - /* 200 */ 810, 773, 818, 820, 794, 797, 787, 828, 832, 834, - /* 210 */ 835, 816, 838, 821, 839, 841, 752, 847, 789, 817, - /* 220 */ 763, 785, 803, 736, 799, 811, 813, 798, 801, 814, - /* 230 */ 805, 822, 843, 819, 824, 825, 826, 738, 827, 831, - /* 240 */ 830, 765, 836, 845, 837, 846, 767, 891, 873, 876, - /* 250 */ 884, 886, 888, 889, 918, 833, 859, 866, 868, 875, - /* 260 */ 877, 878, 880, 881, 844, 882, 937, 910, 854, 885, - /* 270 */ 883, 887, 890, 926, 892, 893, 895, 920, 922, 898, - /* 280 */ 899, 924, 901, 902, 933, 904, 905, 936, 907, 908, - /* 290 */ 939, 911, 874, 896, 900, 903, 949, 906, 912, 913, - /* 300 */ 945, 947, 958, 941, 940, 954, 956, 957, 959, 960, - /* 310 */ 961, 962, 966, 963, 964, 965, 967, 968, 969, 970, - /* 320 */ 971, 972, 995, 973, 976, 974, 1004, 977, 979, 1005, - /* 330 */ 1009, 989, 991, + /* 0 */ 646, 61, 270, 370, 370, 370, 370, 395, 370, 370, + /* 10 */ 62, 401, 533, 502, 401, 401, 401, 401, 401, 401, + /* 20 */ 401, 401, 401, 401, 401, 401, 401, 401, 401, 401, + /* 30 */ 401, 401, 401, 102, 102, 102, 97, 20, 20, 408, + /* 40 */ 408, 20, 20, 84, 152, 227, 227, 230, 301, 152, + /* 50 */ 20, 20, 152, 20, 152, 301, 152, 152, 20, 200, + /* 60 */ 1, 64, 64, 63, 922, 408, 44, 408, 408, 408, + /* 70 */ 408, 408, 408, 408, 408, 408, 408, 408, 408, 408, + /* 80 */ 408, 408, 408, 408, 408, 408, 408, 417, 328, 269, + /* 90 */ 269, 269, 457, 301, 152, 152, 152, 286, 59, 59, + /* 100 */ 59, 59, 59, 18, 198, 969, 15, 55, 285, 34, + /* 110 */ 413, 485, 186, 437, 186, 514, 459, 209, 705, 707, + /* 120 */ 84, 715, 730, 84, 705, 84, 740, 152, 152, 152, + /* 130 */ 152, 152, 152, 152, 152, 152, 152, 152, 705, 740, + /* 140 */ 707, 200, 715, 730, 810, 709, 714, 742, 709, 714, + /* 150 */ 742, 716, 721, 727, 735, 739, 715, 855, 750, 731, + /* 160 */ 736, 748, 817, 152, 714, 742, 742, 714, 742, 791, + /* 170 */ 715, 730, 286, 200, 715, 846, 705, 200, 740, 1182, + /* 180 */ 1182, 1182, 1182, 0, 112, 546, 570, 973, 1049, 1064, + /* 190 */ 323, 389, 711, 193, 193, 193, 193, 193, 193, 193, + /* 200 */ 403, 340, 288, 288, 288, 288, 479, 547, 562, 567, + /* 210 */ 640, 656, 666, 146, 163, 283, 397, 558, 390, 507, + /* 220 */ 517, 579, 549, 603, 620, 629, 630, 638, 641, 670, + /* 230 */ 671, 693, 699, 702, 704, 720, 407, 938, 940, 942, + /* 240 */ 950, 885, 868, 956, 959, 960, 965, 970, 971, 980, + /* 250 */ 981, 983, 929, 984, 936, 988, 990, 952, 957, 949, + /* 260 */ 995, 996, 1001, 1004, 923, 926, 974, 975, 987, 1012, + /* 270 */ 977, 978, 979, 985, 986, 989, 1020, 1000, 1027, 1008, + /* 280 */ 992, 1033, 1013, 997, 1037, 1016, 1039, 1018, 1021, 1042, + /* 290 */ 1043, 931, 1044, 976, 1003, 943, 962, 964, 916, 982, + /* 300 */ 967, 993, 998, 999, 994, 1002, 1007, 1019, 972, 1005, + /* 310 */ 1009, 1006, 919, 1010, 1011, 1014, 927, 1015, 1017, 1022, + /* 320 */ 1023, 932, 1065, 1034, 1035, 1047, 1048, 1050, 1051, 1081, + /* 330 */ 991, 1026, 1024, 1029, 1032, 1025, 1036, 1041, 1045, 1028, + /* 340 */ 1046, 1088, 1053, 1030, 1052, 1031, 1040, 1055, 1056, 1038, + /* 350 */ 1054, 1061, 1067, 1057, 1058, 1068, 1060, 1062, 1069, 1066, + /* 360 */ 1063, 1070, 1071, 1072, 1076, 1074, 1059, 1073, 1075, 1077, + /* 370 */ 1095, 1078, 1079, 1080, 1082, 1083, 1084, 1098, 1087, 1085, + /* 380 */ 1086, 1099, 1104, 1106, 1109, 1110, 1111, 1112, 1115, 1134, + /* 390 */ 1119, 1120, 1122, 1123, 1124, 1125, 1126, 1129, 1130, 1091, + /* 400 */ 1092, 1089, 1090, 1093, 1127, 1132, 1171, 1172, 1151, 1154, + /* 410 */ 1155, 1157, 1159, 1161, }; -#define YY_REDUCE_COUNT (137) -#define YY_REDUCE_MIN (-162) -#define YY_REDUCE_MAX (708) +#define YY_REDUCE_COUNT (182) +#define YY_REDUCE_MIN (-209) +#define YY_REDUCE_MAX (789) static const short yy_reduce_ofst[] = { - /* 0 */ 90, 14, 54, -117, 76, 105, 134, 187, 234, 256, - /* 10 */ 265, 318, 349, 384, 406, 427, 449, 463, 477, 491, - /* 20 */ 517, 531, 552, 566, 580, 594, 608, 622, 636, 650, - /* 30 */ 664, 687, 701, 527, -144, 708, 386, -158, -138, -142, - /* 40 */ -140, -116, -155, -139, 7, -27, -114, -127, 59, 133, - /* 50 */ 16, 150, 41, 144, 48, 180, -111, -17, -17, -17, - /* 60 */ 107, 200, 160, 50, 193, 195, 224, 222, 237, -162, - /* 70 */ 250, 116, 182, 119, 173, 215, 296, 267, 263, 263, - /* 80 */ 263, 315, 274, 292, 363, 369, 371, 383, 380, 400, - /* 90 */ 408, 410, 420, 425, 430, 407, 453, 428, 451, 422, - /* 100 */ 412, 457, 464, 426, 471, 478, 450, 458, 459, 468, - /* 110 */ 263, 516, 508, 506, 485, 487, 503, 538, 315, 555, - /* 120 */ 557, 561, 564, 568, 572, 579, 578, 596, 603, 590, - /* 130 */ 597, 612, 617, 625, 618, 616, 620, 629, + /* 0 */ 66, 14, 52, 115, 150, 185, 216, 255, 293, 327, + /* 10 */ 342, 388, 427, 465, 489, 500, 523, 538, 553, 574, + /* 20 */ 589, 605, 628, 633, 644, 649, 684, 687, 722, 737, + /* 30 */ 753, 758, 789, 75, 132, 754, -59, -149, 65, -18, + /* 40 */ 40, -168, -167, -72, -76, -171, 67, -125, -191, 125, + /* 50 */ 42, 43, 83, 117, 172, -159, 197, 187, 229, 149, + /* 60 */ -209, -209, -209, -138, -172, -129, 48, -128, 73, 210, + /* 70 */ 273, 291, 321, 333, 357, 367, 368, 376, 377, 381, + /* 80 */ 387, 391, 393, 411, 422, 431, 439, 113, -85, 47, + /* 90 */ 223, 402, 144, 226, -86, 267, 292, 80, -175, 260, + /* 100 */ 264, 396, 428, 450, 446, 451, 383, 409, 483, 426, + /* 110 */ 521, 488, 477, 477, 477, 522, 473, 492, 555, 518, + /* 120 */ 559, 550, 545, 580, 585, 584, 600, 583, 594, 610, + /* 130 */ 619, 621, 623, 626, 627, 634, 639, 642, 617, 661, + /* 140 */ 599, 653, 645, 651, 624, 609, 655, 662, 631, 660, + /* 150 */ 663, 632, 643, 647, 652, 477, 683, 673, 668, 650, + /* 160 */ 648, 659, 688, 522, 703, 706, 708, 710, 713, 712, + /* 170 */ 717, 718, 738, 741, 724, 743, 755, 757, 761, 751, + /* 180 */ 762, 764, 774, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 756, 756, 756, 756, 756, 756, 756, 756, 756, 756, - /* 10 */ 756, 756, 756, 756, 756, 756, 756, 756, 756, 756, - /* 20 */ 756, 756, 756, 756, 756, 756, 756, 756, 756, 756, - /* 30 */ 756, 756, 756, 756, 756, 756, 756, 756, 756, 756, - /* 40 */ 756, 783, 756, 756, 756, 756, 756, 756, 756, 756, - /* 50 */ 756, 756, 756, 756, 756, 756, 781, 756, 933, 756, - /* 60 */ 756, 756, 783, 944, 944, 944, 781, 756, 756, 756, - /* 70 */ 846, 756, 756, 1008, 756, 968, 756, 960, 936, 950, - /* 80 */ 937, 756, 993, 953, 756, 756, 783, 756, 756, 783, - /* 90 */ 756, 783, 756, 756, 756, 756, 781, 756, 756, 756, - /* 100 */ 975, 973, 756, 975, 973, 756, 987, 983, 966, 964, - /* 110 */ 950, 756, 756, 756, 1011, 999, 995, 756, 756, 973, - /* 120 */ 756, 756, 973, 756, 860, 756, 756, 756, 781, 756, - /* 130 */ 815, 756, 781, 756, 784, 849, 849, 784, 756, 756, - /* 140 */ 756, 756, 756, 756, 756, 905, 986, 985, 904, 910, - /* 150 */ 909, 908, 756, 756, 756, 756, 899, 900, 898, 897, - /* 160 */ 756, 756, 756, 756, 756, 934, 756, 996, 1000, 756, - /* 170 */ 756, 756, 884, 756, 756, 756, 756, 756, 756, 756, - /* 180 */ 756, 756, 756, 756, 756, 756, 756, 756, 756, 756, - /* 190 */ 756, 756, 756, 756, 756, 756, 756, 756, 756, 756, - /* 200 */ 756, 756, 756, 756, 756, 756, 756, 756, 756, 756, - /* 210 */ 756, 756, 756, 756, 756, 756, 756, 756, 756, 756, - /* 220 */ 756, 957, 967, 756, 756, 756, 756, 756, 756, 756, - /* 230 */ 756, 756, 756, 756, 884, 756, 984, 756, 943, 939, - /* 240 */ 756, 756, 935, 756, 756, 994, 756, 756, 756, 756, - /* 250 */ 756, 756, 756, 756, 929, 756, 756, 756, 756, 756, - /* 260 */ 756, 756, 756, 756, 756, 756, 756, 756, 756, 756, - /* 270 */ 756, 883, 756, 756, 756, 756, 756, 756, 756, 843, - /* 280 */ 756, 756, 756, 756, 756, 756, 756, 756, 756, 756, - /* 290 */ 756, 756, 828, 826, 825, 824, 756, 821, 756, 756, - /* 300 */ 756, 756, 756, 756, 756, 756, 756, 756, 756, 756, - /* 310 */ 756, 756, 756, 756, 756, 756, 756, 756, 756, 756, - /* 320 */ 756, 756, 756, 756, 756, 756, 756, 756, 756, 756, - /* 330 */ 756, 756, 756, + /* 0 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, + /* 10 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, + /* 20 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, + /* 30 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, + /* 40 */ 947, 947, 947, 1004, 947, 947, 947, 947, 947, 947, + /* 50 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 1002, + /* 60 */ 947, 1185, 947, 947, 947, 947, 947, 947, 947, 947, + /* 70 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, + /* 80 */ 947, 947, 947, 947, 947, 947, 947, 947, 1004, 1196, + /* 90 */ 1196, 1196, 1002, 947, 947, 947, 947, 1089, 947, 947, + /* 100 */ 947, 947, 947, 947, 947, 947, 1260, 947, 1042, 1220, + /* 110 */ 947, 1212, 1188, 1202, 1189, 947, 1245, 1205, 947, 947, + /* 120 */ 1004, 947, 947, 1004, 947, 1004, 947, 947, 947, 947, + /* 130 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, + /* 140 */ 947, 1002, 947, 947, 947, 1227, 1225, 947, 1227, 1225, + /* 150 */ 947, 1239, 1235, 1218, 1216, 1202, 947, 947, 947, 1263, + /* 160 */ 1251, 1247, 947, 947, 1225, 947, 947, 1225, 947, 1112, + /* 170 */ 947, 947, 947, 1002, 947, 1058, 947, 1002, 947, 1092, + /* 180 */ 1092, 1005, 952, 947, 947, 947, 947, 947, 947, 947, + /* 190 */ 947, 947, 947, 1157, 1238, 1237, 1156, 1162, 1161, 1160, + /* 200 */ 947, 947, 1151, 1152, 1150, 1149, 947, 947, 947, 947, + /* 210 */ 947, 947, 947, 947, 947, 947, 947, 1186, 947, 1248, + /* 220 */ 1252, 947, 947, 947, 1136, 947, 947, 947, 947, 947, + /* 230 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, + /* 240 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, + /* 250 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, + /* 260 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, + /* 270 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, + /* 280 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, + /* 290 */ 947, 947, 947, 947, 947, 947, 1209, 1219, 947, 947, + /* 300 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 1136, + /* 310 */ 947, 1236, 947, 1195, 1191, 947, 947, 1187, 947, 947, + /* 320 */ 1246, 947, 947, 947, 947, 947, 947, 947, 947, 1181, + /* 330 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, + /* 340 */ 947, 947, 947, 947, 947, 947, 1135, 947, 947, 947, + /* 350 */ 947, 947, 947, 1086, 947, 947, 947, 947, 947, 947, + /* 360 */ 947, 947, 947, 947, 947, 947, 1071, 1069, 1068, 1067, + /* 370 */ 947, 1064, 947, 947, 947, 947, 947, 947, 947, 947, + /* 380 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, + /* 390 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, + /* 400 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, + /* 410 */ 947, 947, 947, 947, }; /********** End of lemon-generated parsing tables *****************************/ @@ -647,207 +706,236 @@ static const char *const yyTokenName[] = { /* 16 */ "NK_REM", /* 17 */ "NK_CONCAT", /* 18 */ "CREATE", - /* 19 */ "USER", - /* 20 */ "PASS", - /* 21 */ "NK_STRING", - /* 22 */ "ALTER", - /* 23 */ "PRIVILEGE", - /* 24 */ "DROP", - /* 25 */ "SHOW", - /* 26 */ "USERS", - /* 27 */ "DNODE", - /* 28 */ "PORT", - /* 29 */ "NK_INTEGER", - /* 30 */ "DNODES", - /* 31 */ "NK_ID", - /* 32 */ "NK_IPTOKEN", - /* 33 */ "QNODE", - /* 34 */ "ON", - /* 35 */ "QNODES", - /* 36 */ "DATABASE", - /* 37 */ "DATABASES", - /* 38 */ "USE", - /* 39 */ "IF", - /* 40 */ "NOT", - /* 41 */ "EXISTS", - /* 42 */ "BLOCKS", - /* 43 */ "CACHE", - /* 44 */ "CACHELAST", - /* 45 */ "COMP", - /* 46 */ "DAYS", - /* 47 */ "FSYNC", - /* 48 */ "MAXROWS", - /* 49 */ "MINROWS", - /* 50 */ "KEEP", - /* 51 */ "PRECISION", - /* 52 */ "QUORUM", - /* 53 */ "REPLICA", - /* 54 */ "TTL", - /* 55 */ "WAL", - /* 56 */ "VGROUPS", - /* 57 */ "SINGLE_STABLE", - /* 58 */ "STREAM_MODE", - /* 59 */ "TABLE", - /* 60 */ "NK_LP", - /* 61 */ "NK_RP", - /* 62 */ "STABLE", - /* 63 */ "TABLES", - /* 64 */ "STABLES", - /* 65 */ "USING", - /* 66 */ "TAGS", - /* 67 */ "NK_DOT", - /* 68 */ "NK_COMMA", - /* 69 */ "COMMENT", - /* 70 */ "BOOL", - /* 71 */ "TINYINT", - /* 72 */ "SMALLINT", - /* 73 */ "INT", - /* 74 */ "INTEGER", - /* 75 */ "BIGINT", - /* 76 */ "FLOAT", - /* 77 */ "DOUBLE", - /* 78 */ "BINARY", - /* 79 */ "TIMESTAMP", - /* 80 */ "NCHAR", - /* 81 */ "UNSIGNED", - /* 82 */ "JSON", - /* 83 */ "VARCHAR", - /* 84 */ "MEDIUMBLOB", - /* 85 */ "BLOB", - /* 86 */ "VARBINARY", - /* 87 */ "DECIMAL", - /* 88 */ "SMA", - /* 89 */ "INDEX", - /* 90 */ "FULLTEXT", - /* 91 */ "FUNCTION", - /* 92 */ "INTERVAL", - /* 93 */ "TOPIC", - /* 94 */ "AS", - /* 95 */ "MNODES", - /* 96 */ "NK_FLOAT", - /* 97 */ "NK_BOOL", - /* 98 */ "NK_VARIABLE", - /* 99 */ "BETWEEN", - /* 100 */ "IS", - /* 101 */ "NULL", - /* 102 */ "NK_LT", - /* 103 */ "NK_GT", - /* 104 */ "NK_LE", - /* 105 */ "NK_GE", - /* 106 */ "NK_NE", - /* 107 */ "NK_EQ", - /* 108 */ "LIKE", - /* 109 */ "MATCH", - /* 110 */ "NMATCH", - /* 111 */ "IN", - /* 112 */ "FROM", - /* 113 */ "JOIN", - /* 114 */ "INNER", - /* 115 */ "SELECT", - /* 116 */ "DISTINCT", - /* 117 */ "WHERE", - /* 118 */ "PARTITION", - /* 119 */ "BY", - /* 120 */ "SESSION", - /* 121 */ "STATE_WINDOW", - /* 122 */ "SLIDING", - /* 123 */ "FILL", - /* 124 */ "VALUE", - /* 125 */ "NONE", - /* 126 */ "PREV", - /* 127 */ "LINEAR", - /* 128 */ "NEXT", - /* 129 */ "GROUP", - /* 130 */ "HAVING", - /* 131 */ "ORDER", - /* 132 */ "SLIMIT", - /* 133 */ "SOFFSET", - /* 134 */ "LIMIT", - /* 135 */ "OFFSET", - /* 136 */ "ASC", - /* 137 */ "DESC", - /* 138 */ "NULLS", - /* 139 */ "FIRST", - /* 140 */ "LAST", - /* 141 */ "cmd", - /* 142 */ "user_name", - /* 143 */ "dnode_endpoint", - /* 144 */ "dnode_host_name", - /* 145 */ "not_exists_opt", - /* 146 */ "db_name", - /* 147 */ "db_options", - /* 148 */ "exists_opt", - /* 149 */ "full_table_name", - /* 150 */ "column_def_list", - /* 151 */ "tags_def_opt", - /* 152 */ "table_options", - /* 153 */ "multi_create_clause", - /* 154 */ "tags_def", - /* 155 */ "multi_drop_clause", - /* 156 */ "create_subtable_clause", - /* 157 */ "specific_tags_opt", - /* 158 */ "literal_list", - /* 159 */ "drop_table_clause", - /* 160 */ "col_name_list", - /* 161 */ "table_name", - /* 162 */ "column_def", - /* 163 */ "column_name", - /* 164 */ "type_name", - /* 165 */ "col_name", - /* 166 */ "index_name", - /* 167 */ "index_options", - /* 168 */ "func_list", - /* 169 */ "duration_literal", - /* 170 */ "sliding_opt", - /* 171 */ "func", - /* 172 */ "function_name", - /* 173 */ "expression_list", - /* 174 */ "topic_name", - /* 175 */ "query_expression", - /* 176 */ "literal", - /* 177 */ "table_alias", - /* 178 */ "column_alias", - /* 179 */ "expression", - /* 180 */ "column_reference", - /* 181 */ "subquery", - /* 182 */ "predicate", - /* 183 */ "compare_op", - /* 184 */ "in_op", - /* 185 */ "in_predicate_value", - /* 186 */ "boolean_value_expression", - /* 187 */ "boolean_primary", - /* 188 */ "common_expression", - /* 189 */ "from_clause", - /* 190 */ "table_reference_list", - /* 191 */ "table_reference", - /* 192 */ "table_primary", - /* 193 */ "joined_table", - /* 194 */ "alias_opt", - /* 195 */ "parenthesized_joined_table", - /* 196 */ "join_type", - /* 197 */ "search_condition", - /* 198 */ "query_specification", - /* 199 */ "set_quantifier_opt", - /* 200 */ "select_list", - /* 201 */ "where_clause_opt", - /* 202 */ "partition_by_clause_opt", - /* 203 */ "twindow_clause_opt", - /* 204 */ "group_by_clause_opt", - /* 205 */ "having_clause_opt", - /* 206 */ "select_sublist", - /* 207 */ "select_item", - /* 208 */ "fill_opt", - /* 209 */ "fill_mode", - /* 210 */ "group_by_list", - /* 211 */ "query_expression_body", - /* 212 */ "order_by_clause_opt", - /* 213 */ "slimit_clause_opt", - /* 214 */ "limit_clause_opt", - /* 215 */ "query_primary", - /* 216 */ "sort_specification_list", - /* 217 */ "sort_specification", - /* 218 */ "ordering_specification_opt", - /* 219 */ "null_ordering_opt", + /* 19 */ "ACCOUNT", + /* 20 */ "NK_ID", + /* 21 */ "PASS", + /* 22 */ "NK_STRING", + /* 23 */ "ALTER", + /* 24 */ "PPS", + /* 25 */ "TSERIES", + /* 26 */ "STORAGE", + /* 27 */ "STREAMS", + /* 28 */ "QTIME", + /* 29 */ "DBS", + /* 30 */ "USERS", + /* 31 */ "CONNS", + /* 32 */ "STATE", + /* 33 */ "USER", + /* 34 */ "PRIVILEGE", + /* 35 */ "DROP", + /* 36 */ "SHOW", + /* 37 */ "DNODE", + /* 38 */ "PORT", + /* 39 */ "NK_INTEGER", + /* 40 */ "DNODES", + /* 41 */ "NK_IPTOKEN", + /* 42 */ "LOCAL", + /* 43 */ "QNODE", + /* 44 */ "ON", + /* 45 */ "QNODES", + /* 46 */ "DATABASE", + /* 47 */ "DATABASES", + /* 48 */ "USE", + /* 49 */ "IF", + /* 50 */ "NOT", + /* 51 */ "EXISTS", + /* 52 */ "BLOCKS", + /* 53 */ "CACHE", + /* 54 */ "CACHELAST", + /* 55 */ "COMP", + /* 56 */ "DAYS", + /* 57 */ "FSYNC", + /* 58 */ "MAXROWS", + /* 59 */ "MINROWS", + /* 60 */ "KEEP", + /* 61 */ "PRECISION", + /* 62 */ "QUORUM", + /* 63 */ "REPLICA", + /* 64 */ "TTL", + /* 65 */ "WAL", + /* 66 */ "VGROUPS", + /* 67 */ "SINGLE_STABLE", + /* 68 */ "STREAM_MODE", + /* 69 */ "RETENTIONS", + /* 70 */ "FILE_FACTOR", + /* 71 */ "NK_FLOAT", + /* 72 */ "TABLE", + /* 73 */ "NK_LP", + /* 74 */ "NK_RP", + /* 75 */ "STABLE", + /* 76 */ "TABLES", + /* 77 */ "STABLES", + /* 78 */ "ADD", + /* 79 */ "COLUMN", + /* 80 */ "MODIFY", + /* 81 */ "RENAME", + /* 82 */ "TAG", + /* 83 */ "SET", + /* 84 */ "NK_EQ", + /* 85 */ "USING", + /* 86 */ "TAGS", + /* 87 */ "NK_DOT", + /* 88 */ "NK_COMMA", + /* 89 */ "COMMENT", + /* 90 */ "BOOL", + /* 91 */ "TINYINT", + /* 92 */ "SMALLINT", + /* 93 */ "INT", + /* 94 */ "INTEGER", + /* 95 */ "BIGINT", + /* 96 */ "FLOAT", + /* 97 */ "DOUBLE", + /* 98 */ "BINARY", + /* 99 */ "TIMESTAMP", + /* 100 */ "NCHAR", + /* 101 */ "UNSIGNED", + /* 102 */ "JSON", + /* 103 */ "VARCHAR", + /* 104 */ "MEDIUMBLOB", + /* 105 */ "BLOB", + /* 106 */ "VARBINARY", + /* 107 */ "DECIMAL", + /* 108 */ "SMA", + /* 109 */ "ROLLUP", + /* 110 */ "INDEX", + /* 111 */ "FULLTEXT", + /* 112 */ "FUNCTION", + /* 113 */ "INTERVAL", + /* 114 */ "TOPIC", + /* 115 */ "AS", + /* 116 */ "MNODES", + /* 117 */ "NK_BOOL", + /* 118 */ "NK_VARIABLE", + /* 119 */ "BETWEEN", + /* 120 */ "IS", + /* 121 */ "NULL", + /* 122 */ "NK_LT", + /* 123 */ "NK_GT", + /* 124 */ "NK_LE", + /* 125 */ "NK_GE", + /* 126 */ "NK_NE", + /* 127 */ "LIKE", + /* 128 */ "MATCH", + /* 129 */ "NMATCH", + /* 130 */ "IN", + /* 131 */ "FROM", + /* 132 */ "JOIN", + /* 133 */ "INNER", + /* 134 */ "SELECT", + /* 135 */ "DISTINCT", + /* 136 */ "WHERE", + /* 137 */ "PARTITION", + /* 138 */ "BY", + /* 139 */ "SESSION", + /* 140 */ "STATE_WINDOW", + /* 141 */ "SLIDING", + /* 142 */ "FILL", + /* 143 */ "VALUE", + /* 144 */ "NONE", + /* 145 */ "PREV", + /* 146 */ "LINEAR", + /* 147 */ "NEXT", + /* 148 */ "GROUP", + /* 149 */ "HAVING", + /* 150 */ "ORDER", + /* 151 */ "SLIMIT", + /* 152 */ "SOFFSET", + /* 153 */ "LIMIT", + /* 154 */ "OFFSET", + /* 155 */ "ASC", + /* 156 */ "DESC", + /* 157 */ "NULLS", + /* 158 */ "FIRST", + /* 159 */ "LAST", + /* 160 */ "cmd", + /* 161 */ "account_options", + /* 162 */ "alter_account_options", + /* 163 */ "literal", + /* 164 */ "alter_account_option", + /* 165 */ "user_name", + /* 166 */ "dnode_endpoint", + /* 167 */ "dnode_host_name", + /* 168 */ "not_exists_opt", + /* 169 */ "db_name", + /* 170 */ "db_options", + /* 171 */ "exists_opt", + /* 172 */ "alter_db_options", + /* 173 */ "alter_db_option", + /* 174 */ "full_table_name", + /* 175 */ "column_def_list", + /* 176 */ "tags_def_opt", + /* 177 */ "table_options", + /* 178 */ "multi_create_clause", + /* 179 */ "tags_def", + /* 180 */ "multi_drop_clause", + /* 181 */ "alter_table_clause", + /* 182 */ "alter_table_options", + /* 183 */ "column_name", + /* 184 */ "type_name", + /* 185 */ "create_subtable_clause", + /* 186 */ "specific_tags_opt", + /* 187 */ "literal_list", + /* 188 */ "drop_table_clause", + /* 189 */ "col_name_list", + /* 190 */ "table_name", + /* 191 */ "column_def", + /* 192 */ "func_name_list", + /* 193 */ "alter_table_option", + /* 194 */ "col_name", + /* 195 */ "func_name", + /* 196 */ "function_name", + /* 197 */ "index_name", + /* 198 */ "index_options", + /* 199 */ "func_list", + /* 200 */ "duration_literal", + /* 201 */ "sliding_opt", + /* 202 */ "func", + /* 203 */ "expression_list", + /* 204 */ "topic_name", + /* 205 */ "query_expression", + /* 206 */ "table_alias", + /* 207 */ "column_alias", + /* 208 */ "expression", + /* 209 */ "column_reference", + /* 210 */ "subquery", + /* 211 */ "predicate", + /* 212 */ "compare_op", + /* 213 */ "in_op", + /* 214 */ "in_predicate_value", + /* 215 */ "boolean_value_expression", + /* 216 */ "boolean_primary", + /* 217 */ "common_expression", + /* 218 */ "from_clause", + /* 219 */ "table_reference_list", + /* 220 */ "table_reference", + /* 221 */ "table_primary", + /* 222 */ "joined_table", + /* 223 */ "alias_opt", + /* 224 */ "parenthesized_joined_table", + /* 225 */ "join_type", + /* 226 */ "search_condition", + /* 227 */ "query_specification", + /* 228 */ "set_quantifier_opt", + /* 229 */ "select_list", + /* 230 */ "where_clause_opt", + /* 231 */ "partition_by_clause_opt", + /* 232 */ "twindow_clause_opt", + /* 233 */ "group_by_clause_opt", + /* 234 */ "having_clause_opt", + /* 235 */ "select_sublist", + /* 236 */ "select_item", + /* 237 */ "fill_opt", + /* 238 */ "fill_mode", + /* 239 */ "group_by_list", + /* 240 */ "query_expression_body", + /* 241 */ "order_by_clause_opt", + /* 242 */ "slimit_clause_opt", + /* 243 */ "limit_clause_opt", + /* 244 */ "query_primary", + /* 245 */ "sort_specification_list", + /* 246 */ "sort_specification", + /* 247 */ "ordering_specification_opt", + /* 248 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -855,261 +943,322 @@ static const char *const yyTokenName[] = { /* For tracing reduce actions, the names of all rules are required. */ static const char *const yyRuleName[] = { - /* 0 */ "cmd ::= CREATE USER user_name PASS NK_STRING", - /* 1 */ "cmd ::= ALTER USER user_name PASS NK_STRING", - /* 2 */ "cmd ::= ALTER USER user_name PRIVILEGE NK_STRING", - /* 3 */ "cmd ::= DROP USER user_name", - /* 4 */ "cmd ::= SHOW USERS", - /* 5 */ "cmd ::= CREATE DNODE dnode_endpoint", - /* 6 */ "cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER", - /* 7 */ "cmd ::= DROP DNODE NK_INTEGER", - /* 8 */ "cmd ::= DROP DNODE dnode_endpoint", - /* 9 */ "cmd ::= SHOW DNODES", - /* 10 */ "dnode_endpoint ::= NK_STRING", - /* 11 */ "dnode_host_name ::= NK_ID", - /* 12 */ "dnode_host_name ::= NK_IPTOKEN", - /* 13 */ "cmd ::= CREATE QNODE ON DNODE NK_INTEGER", - /* 14 */ "cmd ::= DROP QNODE ON DNODE NK_INTEGER", - /* 15 */ "cmd ::= SHOW QNODES", - /* 16 */ "cmd ::= CREATE DATABASE not_exists_opt db_name db_options", - /* 17 */ "cmd ::= DROP DATABASE exists_opt db_name", - /* 18 */ "cmd ::= SHOW DATABASES", - /* 19 */ "cmd ::= USE db_name", - /* 20 */ "cmd ::= ALTER DATABASE db_name db_options", - /* 21 */ "not_exists_opt ::= IF NOT EXISTS", - /* 22 */ "not_exists_opt ::=", - /* 23 */ "exists_opt ::= IF EXISTS", - /* 24 */ "exists_opt ::=", - /* 25 */ "db_options ::=", - /* 26 */ "db_options ::= db_options BLOCKS NK_INTEGER", - /* 27 */ "db_options ::= db_options CACHE NK_INTEGER", - /* 28 */ "db_options ::= db_options CACHELAST NK_INTEGER", - /* 29 */ "db_options ::= db_options COMP NK_INTEGER", - /* 30 */ "db_options ::= db_options DAYS NK_INTEGER", - /* 31 */ "db_options ::= db_options FSYNC NK_INTEGER", - /* 32 */ "db_options ::= db_options MAXROWS NK_INTEGER", - /* 33 */ "db_options ::= db_options MINROWS NK_INTEGER", - /* 34 */ "db_options ::= db_options KEEP NK_INTEGER", - /* 35 */ "db_options ::= db_options PRECISION NK_STRING", - /* 36 */ "db_options ::= db_options QUORUM NK_INTEGER", - /* 37 */ "db_options ::= db_options REPLICA NK_INTEGER", - /* 38 */ "db_options ::= db_options TTL NK_INTEGER", - /* 39 */ "db_options ::= db_options WAL NK_INTEGER", - /* 40 */ "db_options ::= db_options VGROUPS NK_INTEGER", - /* 41 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", - /* 42 */ "db_options ::= db_options STREAM_MODE NK_INTEGER", - /* 43 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", - /* 44 */ "cmd ::= CREATE TABLE multi_create_clause", - /* 45 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", - /* 46 */ "cmd ::= DROP TABLE multi_drop_clause", - /* 47 */ "cmd ::= DROP STABLE exists_opt full_table_name", - /* 48 */ "cmd ::= SHOW TABLES", - /* 49 */ "cmd ::= SHOW STABLES", - /* 50 */ "multi_create_clause ::= create_subtable_clause", - /* 51 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", - /* 52 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP", - /* 53 */ "multi_drop_clause ::= drop_table_clause", - /* 54 */ "multi_drop_clause ::= multi_drop_clause drop_table_clause", - /* 55 */ "drop_table_clause ::= exists_opt full_table_name", - /* 56 */ "specific_tags_opt ::=", - /* 57 */ "specific_tags_opt ::= NK_LP col_name_list NK_RP", - /* 58 */ "full_table_name ::= table_name", - /* 59 */ "full_table_name ::= db_name NK_DOT table_name", - /* 60 */ "column_def_list ::= column_def", - /* 61 */ "column_def_list ::= column_def_list NK_COMMA column_def", - /* 62 */ "column_def ::= column_name type_name", - /* 63 */ "column_def ::= column_name type_name COMMENT NK_STRING", - /* 64 */ "type_name ::= BOOL", - /* 65 */ "type_name ::= TINYINT", - /* 66 */ "type_name ::= SMALLINT", - /* 67 */ "type_name ::= INT", - /* 68 */ "type_name ::= INTEGER", - /* 69 */ "type_name ::= BIGINT", - /* 70 */ "type_name ::= FLOAT", - /* 71 */ "type_name ::= DOUBLE", - /* 72 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", - /* 73 */ "type_name ::= TIMESTAMP", - /* 74 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", - /* 75 */ "type_name ::= TINYINT UNSIGNED", - /* 76 */ "type_name ::= SMALLINT UNSIGNED", - /* 77 */ "type_name ::= INT UNSIGNED", - /* 78 */ "type_name ::= BIGINT UNSIGNED", - /* 79 */ "type_name ::= JSON", - /* 80 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", - /* 81 */ "type_name ::= MEDIUMBLOB", - /* 82 */ "type_name ::= BLOB", - /* 83 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", - /* 84 */ "type_name ::= DECIMAL", - /* 85 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", - /* 86 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", - /* 87 */ "tags_def_opt ::=", - /* 88 */ "tags_def_opt ::= tags_def", - /* 89 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", - /* 90 */ "table_options ::=", - /* 91 */ "table_options ::= table_options COMMENT NK_STRING", - /* 92 */ "table_options ::= table_options KEEP NK_INTEGER", - /* 93 */ "table_options ::= table_options TTL NK_INTEGER", - /* 94 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", - /* 95 */ "col_name_list ::= col_name", - /* 96 */ "col_name_list ::= col_name_list NK_COMMA col_name", - /* 97 */ "col_name ::= column_name", - /* 98 */ "cmd ::= CREATE SMA INDEX index_name ON table_name index_options", - /* 99 */ "cmd ::= CREATE FULLTEXT INDEX index_name ON table_name NK_LP col_name_list NK_RP", - /* 100 */ "cmd ::= DROP INDEX index_name ON table_name", - /* 101 */ "index_options ::=", - /* 102 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt", - /* 103 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt", - /* 104 */ "func_list ::= func", - /* 105 */ "func_list ::= func_list NK_COMMA func", - /* 106 */ "func ::= function_name NK_LP expression_list NK_RP", - /* 107 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression", - /* 108 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name", - /* 109 */ "cmd ::= DROP TOPIC exists_opt topic_name", - /* 110 */ "cmd ::= SHOW VGROUPS", - /* 111 */ "cmd ::= SHOW db_name NK_DOT VGROUPS", - /* 112 */ "cmd ::= SHOW MNODES", - /* 113 */ "cmd ::= query_expression", - /* 114 */ "literal ::= NK_INTEGER", - /* 115 */ "literal ::= NK_FLOAT", - /* 116 */ "literal ::= NK_STRING", - /* 117 */ "literal ::= NK_BOOL", - /* 118 */ "literal ::= TIMESTAMP NK_STRING", - /* 119 */ "literal ::= duration_literal", - /* 120 */ "duration_literal ::= NK_VARIABLE", - /* 121 */ "literal_list ::= literal", - /* 122 */ "literal_list ::= literal_list NK_COMMA literal", - /* 123 */ "db_name ::= NK_ID", - /* 124 */ "table_name ::= NK_ID", - /* 125 */ "column_name ::= NK_ID", - /* 126 */ "function_name ::= NK_ID", - /* 127 */ "table_alias ::= NK_ID", - /* 128 */ "column_alias ::= NK_ID", - /* 129 */ "user_name ::= NK_ID", - /* 130 */ "index_name ::= NK_ID", - /* 131 */ "topic_name ::= NK_ID", - /* 132 */ "expression ::= literal", - /* 133 */ "expression ::= column_reference", - /* 134 */ "expression ::= function_name NK_LP expression_list NK_RP", - /* 135 */ "expression ::= function_name NK_LP NK_STAR NK_RP", - /* 136 */ "expression ::= subquery", - /* 137 */ "expression ::= NK_LP expression NK_RP", - /* 138 */ "expression ::= NK_PLUS expression", - /* 139 */ "expression ::= NK_MINUS expression", - /* 140 */ "expression ::= expression NK_PLUS expression", - /* 141 */ "expression ::= expression NK_MINUS expression", - /* 142 */ "expression ::= expression NK_STAR expression", - /* 143 */ "expression ::= expression NK_SLASH expression", - /* 144 */ "expression ::= expression NK_REM expression", - /* 145 */ "expression_list ::= expression", - /* 146 */ "expression_list ::= expression_list NK_COMMA expression", - /* 147 */ "column_reference ::= column_name", - /* 148 */ "column_reference ::= table_name NK_DOT column_name", - /* 149 */ "predicate ::= expression compare_op expression", - /* 150 */ "predicate ::= expression BETWEEN expression AND expression", - /* 151 */ "predicate ::= expression NOT BETWEEN expression AND expression", - /* 152 */ "predicate ::= expression IS NULL", - /* 153 */ "predicate ::= expression IS NOT NULL", - /* 154 */ "predicate ::= expression in_op in_predicate_value", - /* 155 */ "compare_op ::= NK_LT", - /* 156 */ "compare_op ::= NK_GT", - /* 157 */ "compare_op ::= NK_LE", - /* 158 */ "compare_op ::= NK_GE", - /* 159 */ "compare_op ::= NK_NE", - /* 160 */ "compare_op ::= NK_EQ", - /* 161 */ "compare_op ::= LIKE", - /* 162 */ "compare_op ::= NOT LIKE", - /* 163 */ "compare_op ::= MATCH", - /* 164 */ "compare_op ::= NMATCH", - /* 165 */ "in_op ::= IN", - /* 166 */ "in_op ::= NOT IN", - /* 167 */ "in_predicate_value ::= NK_LP expression_list NK_RP", - /* 168 */ "boolean_value_expression ::= boolean_primary", - /* 169 */ "boolean_value_expression ::= NOT boolean_primary", - /* 170 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", - /* 171 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", - /* 172 */ "boolean_primary ::= predicate", - /* 173 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 174 */ "common_expression ::= expression", - /* 175 */ "common_expression ::= boolean_value_expression", - /* 176 */ "from_clause ::= FROM table_reference_list", - /* 177 */ "table_reference_list ::= table_reference", - /* 178 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 179 */ "table_reference ::= table_primary", - /* 180 */ "table_reference ::= joined_table", - /* 181 */ "table_primary ::= table_name alias_opt", - /* 182 */ "table_primary ::= db_name NK_DOT table_name alias_opt", - /* 183 */ "table_primary ::= subquery alias_opt", - /* 184 */ "table_primary ::= parenthesized_joined_table", - /* 185 */ "alias_opt ::=", - /* 186 */ "alias_opt ::= table_alias", - /* 187 */ "alias_opt ::= AS table_alias", - /* 188 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 189 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 190 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 191 */ "join_type ::=", - /* 192 */ "join_type ::= INNER", - /* 193 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 194 */ "set_quantifier_opt ::=", - /* 195 */ "set_quantifier_opt ::= DISTINCT", - /* 196 */ "set_quantifier_opt ::= ALL", - /* 197 */ "select_list ::= NK_STAR", - /* 198 */ "select_list ::= select_sublist", - /* 199 */ "select_sublist ::= select_item", - /* 200 */ "select_sublist ::= select_sublist NK_COMMA select_item", - /* 201 */ "select_item ::= common_expression", - /* 202 */ "select_item ::= common_expression column_alias", - /* 203 */ "select_item ::= common_expression AS column_alias", - /* 204 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 205 */ "where_clause_opt ::=", - /* 206 */ "where_clause_opt ::= WHERE search_condition", - /* 207 */ "partition_by_clause_opt ::=", - /* 208 */ "partition_by_clause_opt ::= PARTITION BY expression_list", - /* 209 */ "twindow_clause_opt ::=", - /* 210 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP", - /* 211 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP", - /* 212 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 213 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 214 */ "sliding_opt ::=", - /* 215 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 216 */ "fill_opt ::=", - /* 217 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 218 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 219 */ "fill_mode ::= NONE", - /* 220 */ "fill_mode ::= PREV", - /* 221 */ "fill_mode ::= NULL", - /* 222 */ "fill_mode ::= LINEAR", - /* 223 */ "fill_mode ::= NEXT", - /* 224 */ "group_by_clause_opt ::=", - /* 225 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 226 */ "group_by_list ::= expression", - /* 227 */ "group_by_list ::= group_by_list NK_COMMA expression", - /* 228 */ "having_clause_opt ::=", - /* 229 */ "having_clause_opt ::= HAVING search_condition", - /* 230 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 231 */ "query_expression_body ::= query_primary", - /* 232 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", - /* 233 */ "query_primary ::= query_specification", - /* 234 */ "order_by_clause_opt ::=", - /* 235 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 236 */ "slimit_clause_opt ::=", - /* 237 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 238 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 239 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 240 */ "limit_clause_opt ::=", - /* 241 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 242 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 243 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 244 */ "subquery ::= NK_LP query_expression NK_RP", - /* 245 */ "search_condition ::= common_expression", - /* 246 */ "sort_specification_list ::= sort_specification", - /* 247 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 248 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", - /* 249 */ "ordering_specification_opt ::=", - /* 250 */ "ordering_specification_opt ::= ASC", - /* 251 */ "ordering_specification_opt ::= DESC", - /* 252 */ "null_ordering_opt ::=", - /* 253 */ "null_ordering_opt ::= NULLS FIRST", - /* 254 */ "null_ordering_opt ::= NULLS LAST", + /* 0 */ "cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options", + /* 1 */ "cmd ::= ALTER ACCOUNT NK_ID alter_account_options", + /* 2 */ "account_options ::=", + /* 3 */ "account_options ::= account_options PPS literal", + /* 4 */ "account_options ::= account_options TSERIES literal", + /* 5 */ "account_options ::= account_options STORAGE literal", + /* 6 */ "account_options ::= account_options STREAMS literal", + /* 7 */ "account_options ::= account_options QTIME literal", + /* 8 */ "account_options ::= account_options DBS literal", + /* 9 */ "account_options ::= account_options USERS literal", + /* 10 */ "account_options ::= account_options CONNS literal", + /* 11 */ "account_options ::= account_options STATE literal", + /* 12 */ "alter_account_options ::= alter_account_option", + /* 13 */ "alter_account_options ::= alter_account_options alter_account_option", + /* 14 */ "alter_account_option ::= PASS literal", + /* 15 */ "alter_account_option ::= PPS literal", + /* 16 */ "alter_account_option ::= TSERIES literal", + /* 17 */ "alter_account_option ::= STORAGE literal", + /* 18 */ "alter_account_option ::= STREAMS literal", + /* 19 */ "alter_account_option ::= QTIME literal", + /* 20 */ "alter_account_option ::= DBS literal", + /* 21 */ "alter_account_option ::= USERS literal", + /* 22 */ "alter_account_option ::= CONNS literal", + /* 23 */ "alter_account_option ::= STATE literal", + /* 24 */ "cmd ::= CREATE USER user_name PASS NK_STRING", + /* 25 */ "cmd ::= ALTER USER user_name PASS NK_STRING", + /* 26 */ "cmd ::= ALTER USER user_name PRIVILEGE NK_STRING", + /* 27 */ "cmd ::= DROP USER user_name", + /* 28 */ "cmd ::= SHOW USERS", + /* 29 */ "cmd ::= CREATE DNODE dnode_endpoint", + /* 30 */ "cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER", + /* 31 */ "cmd ::= DROP DNODE NK_INTEGER", + /* 32 */ "cmd ::= DROP DNODE dnode_endpoint", + /* 33 */ "cmd ::= SHOW DNODES", + /* 34 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING", + /* 35 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING", + /* 36 */ "cmd ::= ALTER ALL DNODES NK_STRING", + /* 37 */ "cmd ::= ALTER ALL DNODES NK_STRING NK_STRING", + /* 38 */ "dnode_endpoint ::= NK_STRING", + /* 39 */ "dnode_host_name ::= NK_ID", + /* 40 */ "dnode_host_name ::= NK_IPTOKEN", + /* 41 */ "cmd ::= ALTER LOCAL NK_STRING", + /* 42 */ "cmd ::= ALTER LOCAL NK_STRING NK_STRING", + /* 43 */ "cmd ::= CREATE QNODE ON DNODE NK_INTEGER", + /* 44 */ "cmd ::= DROP QNODE ON DNODE NK_INTEGER", + /* 45 */ "cmd ::= SHOW QNODES", + /* 46 */ "cmd ::= CREATE DATABASE not_exists_opt db_name db_options", + /* 47 */ "cmd ::= DROP DATABASE exists_opt db_name", + /* 48 */ "cmd ::= SHOW DATABASES", + /* 49 */ "cmd ::= USE db_name", + /* 50 */ "cmd ::= ALTER DATABASE db_name alter_db_options", + /* 51 */ "not_exists_opt ::= IF NOT EXISTS", + /* 52 */ "not_exists_opt ::=", + /* 53 */ "exists_opt ::= IF EXISTS", + /* 54 */ "exists_opt ::=", + /* 55 */ "db_options ::=", + /* 56 */ "db_options ::= db_options BLOCKS NK_INTEGER", + /* 57 */ "db_options ::= db_options CACHE NK_INTEGER", + /* 58 */ "db_options ::= db_options CACHELAST NK_INTEGER", + /* 59 */ "db_options ::= db_options COMP NK_INTEGER", + /* 60 */ "db_options ::= db_options DAYS NK_INTEGER", + /* 61 */ "db_options ::= db_options FSYNC NK_INTEGER", + /* 62 */ "db_options ::= db_options MAXROWS NK_INTEGER", + /* 63 */ "db_options ::= db_options MINROWS NK_INTEGER", + /* 64 */ "db_options ::= db_options KEEP NK_INTEGER", + /* 65 */ "db_options ::= db_options PRECISION NK_STRING", + /* 66 */ "db_options ::= db_options QUORUM NK_INTEGER", + /* 67 */ "db_options ::= db_options REPLICA NK_INTEGER", + /* 68 */ "db_options ::= db_options TTL NK_INTEGER", + /* 69 */ "db_options ::= db_options WAL NK_INTEGER", + /* 70 */ "db_options ::= db_options VGROUPS NK_INTEGER", + /* 71 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", + /* 72 */ "db_options ::= db_options STREAM_MODE NK_INTEGER", + /* 73 */ "db_options ::= db_options RETENTIONS NK_STRING", + /* 74 */ "db_options ::= db_options FILE_FACTOR NK_FLOAT", + /* 75 */ "alter_db_options ::= alter_db_option", + /* 76 */ "alter_db_options ::= alter_db_options alter_db_option", + /* 77 */ "alter_db_option ::= BLOCKS NK_INTEGER", + /* 78 */ "alter_db_option ::= FSYNC NK_INTEGER", + /* 79 */ "alter_db_option ::= KEEP NK_INTEGER", + /* 80 */ "alter_db_option ::= WAL NK_INTEGER", + /* 81 */ "alter_db_option ::= QUORUM NK_INTEGER", + /* 82 */ "alter_db_option ::= CACHELAST NK_INTEGER", + /* 83 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", + /* 84 */ "cmd ::= CREATE TABLE multi_create_clause", + /* 85 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", + /* 86 */ "cmd ::= DROP TABLE multi_drop_clause", + /* 87 */ "cmd ::= DROP STABLE exists_opt full_table_name", + /* 88 */ "cmd ::= SHOW TABLES", + /* 89 */ "cmd ::= SHOW STABLES", + /* 90 */ "cmd ::= ALTER TABLE alter_table_clause", + /* 91 */ "cmd ::= ALTER STABLE alter_table_clause", + /* 92 */ "alter_table_clause ::= full_table_name alter_table_options", + /* 93 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name", + /* 94 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name", + /* 95 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name", + /* 96 */ "alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name", + /* 97 */ "alter_table_clause ::= full_table_name ADD TAG column_name type_name", + /* 98 */ "alter_table_clause ::= full_table_name DROP TAG column_name", + /* 99 */ "alter_table_clause ::= full_table_name MODIFY TAG column_name type_name", + /* 100 */ "alter_table_clause ::= full_table_name RENAME TAG column_name column_name", + /* 101 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal", + /* 102 */ "multi_create_clause ::= create_subtable_clause", + /* 103 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", + /* 104 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP", + /* 105 */ "multi_drop_clause ::= drop_table_clause", + /* 106 */ "multi_drop_clause ::= multi_drop_clause drop_table_clause", + /* 107 */ "drop_table_clause ::= exists_opt full_table_name", + /* 108 */ "specific_tags_opt ::=", + /* 109 */ "specific_tags_opt ::= NK_LP col_name_list NK_RP", + /* 110 */ "full_table_name ::= table_name", + /* 111 */ "full_table_name ::= db_name NK_DOT table_name", + /* 112 */ "column_def_list ::= column_def", + /* 113 */ "column_def_list ::= column_def_list NK_COMMA column_def", + /* 114 */ "column_def ::= column_name type_name", + /* 115 */ "column_def ::= column_name type_name COMMENT NK_STRING", + /* 116 */ "type_name ::= BOOL", + /* 117 */ "type_name ::= TINYINT", + /* 118 */ "type_name ::= SMALLINT", + /* 119 */ "type_name ::= INT", + /* 120 */ "type_name ::= INTEGER", + /* 121 */ "type_name ::= BIGINT", + /* 122 */ "type_name ::= FLOAT", + /* 123 */ "type_name ::= DOUBLE", + /* 124 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", + /* 125 */ "type_name ::= TIMESTAMP", + /* 126 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", + /* 127 */ "type_name ::= TINYINT UNSIGNED", + /* 128 */ "type_name ::= SMALLINT UNSIGNED", + /* 129 */ "type_name ::= INT UNSIGNED", + /* 130 */ "type_name ::= BIGINT UNSIGNED", + /* 131 */ "type_name ::= JSON", + /* 132 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", + /* 133 */ "type_name ::= MEDIUMBLOB", + /* 134 */ "type_name ::= BLOB", + /* 135 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", + /* 136 */ "type_name ::= DECIMAL", + /* 137 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", + /* 138 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", + /* 139 */ "tags_def_opt ::=", + /* 140 */ "tags_def_opt ::= tags_def", + /* 141 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", + /* 142 */ "table_options ::=", + /* 143 */ "table_options ::= table_options COMMENT NK_STRING", + /* 144 */ "table_options ::= table_options KEEP NK_INTEGER", + /* 145 */ "table_options ::= table_options TTL NK_INTEGER", + /* 146 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", + /* 147 */ "table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP", + /* 148 */ "alter_table_options ::= alter_table_option", + /* 149 */ "alter_table_options ::= alter_table_options alter_table_option", + /* 150 */ "alter_table_option ::= COMMENT NK_STRING", + /* 151 */ "alter_table_option ::= KEEP NK_INTEGER", + /* 152 */ "alter_table_option ::= TTL NK_INTEGER", + /* 153 */ "col_name_list ::= col_name", + /* 154 */ "col_name_list ::= col_name_list NK_COMMA col_name", + /* 155 */ "col_name ::= column_name", + /* 156 */ "func_name_list ::= func_name", + /* 157 */ "func_name_list ::= func_name_list NK_COMMA col_name", + /* 158 */ "func_name ::= function_name", + /* 159 */ "cmd ::= CREATE SMA INDEX index_name ON table_name index_options", + /* 160 */ "cmd ::= CREATE FULLTEXT INDEX index_name ON table_name NK_LP col_name_list NK_RP", + /* 161 */ "cmd ::= DROP INDEX index_name ON table_name", + /* 162 */ "index_options ::=", + /* 163 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt", + /* 164 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt", + /* 165 */ "func_list ::= func", + /* 166 */ "func_list ::= func_list NK_COMMA func", + /* 167 */ "func ::= function_name NK_LP expression_list NK_RP", + /* 168 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression", + /* 169 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name", + /* 170 */ "cmd ::= DROP TOPIC exists_opt topic_name", + /* 171 */ "cmd ::= SHOW VGROUPS", + /* 172 */ "cmd ::= SHOW db_name NK_DOT VGROUPS", + /* 173 */ "cmd ::= SHOW MNODES", + /* 174 */ "cmd ::= query_expression", + /* 175 */ "literal ::= NK_INTEGER", + /* 176 */ "literal ::= NK_FLOAT", + /* 177 */ "literal ::= NK_STRING", + /* 178 */ "literal ::= NK_BOOL", + /* 179 */ "literal ::= TIMESTAMP NK_STRING", + /* 180 */ "literal ::= duration_literal", + /* 181 */ "duration_literal ::= NK_VARIABLE", + /* 182 */ "literal_list ::= literal", + /* 183 */ "literal_list ::= literal_list NK_COMMA literal", + /* 184 */ "db_name ::= NK_ID", + /* 185 */ "table_name ::= NK_ID", + /* 186 */ "column_name ::= NK_ID", + /* 187 */ "function_name ::= NK_ID", + /* 188 */ "table_alias ::= NK_ID", + /* 189 */ "column_alias ::= NK_ID", + /* 190 */ "user_name ::= NK_ID", + /* 191 */ "index_name ::= NK_ID", + /* 192 */ "topic_name ::= NK_ID", + /* 193 */ "expression ::= literal", + /* 194 */ "expression ::= column_reference", + /* 195 */ "expression ::= function_name NK_LP expression_list NK_RP", + /* 196 */ "expression ::= function_name NK_LP NK_STAR NK_RP", + /* 197 */ "expression ::= subquery", + /* 198 */ "expression ::= NK_LP expression NK_RP", + /* 199 */ "expression ::= NK_PLUS expression", + /* 200 */ "expression ::= NK_MINUS expression", + /* 201 */ "expression ::= expression NK_PLUS expression", + /* 202 */ "expression ::= expression NK_MINUS expression", + /* 203 */ "expression ::= expression NK_STAR expression", + /* 204 */ "expression ::= expression NK_SLASH expression", + /* 205 */ "expression ::= expression NK_REM expression", + /* 206 */ "expression_list ::= expression", + /* 207 */ "expression_list ::= expression_list NK_COMMA expression", + /* 208 */ "column_reference ::= column_name", + /* 209 */ "column_reference ::= table_name NK_DOT column_name", + /* 210 */ "predicate ::= expression compare_op expression", + /* 211 */ "predicate ::= expression BETWEEN expression AND expression", + /* 212 */ "predicate ::= expression NOT BETWEEN expression AND expression", + /* 213 */ "predicate ::= expression IS NULL", + /* 214 */ "predicate ::= expression IS NOT NULL", + /* 215 */ "predicate ::= expression in_op in_predicate_value", + /* 216 */ "compare_op ::= NK_LT", + /* 217 */ "compare_op ::= NK_GT", + /* 218 */ "compare_op ::= NK_LE", + /* 219 */ "compare_op ::= NK_GE", + /* 220 */ "compare_op ::= NK_NE", + /* 221 */ "compare_op ::= NK_EQ", + /* 222 */ "compare_op ::= LIKE", + /* 223 */ "compare_op ::= NOT LIKE", + /* 224 */ "compare_op ::= MATCH", + /* 225 */ "compare_op ::= NMATCH", + /* 226 */ "in_op ::= IN", + /* 227 */ "in_op ::= NOT IN", + /* 228 */ "in_predicate_value ::= NK_LP expression_list NK_RP", + /* 229 */ "boolean_value_expression ::= boolean_primary", + /* 230 */ "boolean_value_expression ::= NOT boolean_primary", + /* 231 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 232 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 233 */ "boolean_primary ::= predicate", + /* 234 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 235 */ "common_expression ::= expression", + /* 236 */ "common_expression ::= boolean_value_expression", + /* 237 */ "from_clause ::= FROM table_reference_list", + /* 238 */ "table_reference_list ::= table_reference", + /* 239 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 240 */ "table_reference ::= table_primary", + /* 241 */ "table_reference ::= joined_table", + /* 242 */ "table_primary ::= table_name alias_opt", + /* 243 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 244 */ "table_primary ::= subquery alias_opt", + /* 245 */ "table_primary ::= parenthesized_joined_table", + /* 246 */ "alias_opt ::=", + /* 247 */ "alias_opt ::= table_alias", + /* 248 */ "alias_opt ::= AS table_alias", + /* 249 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 250 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 251 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 252 */ "join_type ::=", + /* 253 */ "join_type ::= INNER", + /* 254 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 255 */ "set_quantifier_opt ::=", + /* 256 */ "set_quantifier_opt ::= DISTINCT", + /* 257 */ "set_quantifier_opt ::= ALL", + /* 258 */ "select_list ::= NK_STAR", + /* 259 */ "select_list ::= select_sublist", + /* 260 */ "select_sublist ::= select_item", + /* 261 */ "select_sublist ::= select_sublist NK_COMMA select_item", + /* 262 */ "select_item ::= common_expression", + /* 263 */ "select_item ::= common_expression column_alias", + /* 264 */ "select_item ::= common_expression AS column_alias", + /* 265 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 266 */ "where_clause_opt ::=", + /* 267 */ "where_clause_opt ::= WHERE search_condition", + /* 268 */ "partition_by_clause_opt ::=", + /* 269 */ "partition_by_clause_opt ::= PARTITION BY expression_list", + /* 270 */ "twindow_clause_opt ::=", + /* 271 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP", + /* 272 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP", + /* 273 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 274 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 275 */ "sliding_opt ::=", + /* 276 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 277 */ "fill_opt ::=", + /* 278 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 279 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 280 */ "fill_mode ::= NONE", + /* 281 */ "fill_mode ::= PREV", + /* 282 */ "fill_mode ::= NULL", + /* 283 */ "fill_mode ::= LINEAR", + /* 284 */ "fill_mode ::= NEXT", + /* 285 */ "group_by_clause_opt ::=", + /* 286 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 287 */ "group_by_list ::= expression", + /* 288 */ "group_by_list ::= group_by_list NK_COMMA expression", + /* 289 */ "having_clause_opt ::=", + /* 290 */ "having_clause_opt ::= HAVING search_condition", + /* 291 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 292 */ "query_expression_body ::= query_primary", + /* 293 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", + /* 294 */ "query_primary ::= query_specification", + /* 295 */ "order_by_clause_opt ::=", + /* 296 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 297 */ "slimit_clause_opt ::=", + /* 298 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 299 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 300 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 301 */ "limit_clause_opt ::=", + /* 302 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 303 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 304 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 305 */ "subquery ::= NK_LP query_expression NK_RP", + /* 306 */ "search_condition ::= common_expression", + /* 307 */ "sort_specification_list ::= sort_specification", + /* 308 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 309 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", + /* 310 */ "ordering_specification_opt ::=", + /* 311 */ "ordering_specification_opt ::= ASC", + /* 312 */ "ordering_specification_opt ::= DESC", + /* 313 */ "null_ordering_opt ::=", + /* 314 */ "null_ordering_opt ::= NULLS FIRST", + /* 315 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -1236,121 +1385,139 @@ static void yy_destructor( */ /********* Begin destructor definitions ***************************************/ /* Default NON-TERMINAL Destructor */ - case 141: /* cmd */ - case 147: /* db_options */ - case 149: /* full_table_name */ - case 152: /* table_options */ - case 156: /* create_subtable_clause */ - case 159: /* drop_table_clause */ - case 162: /* column_def */ - case 165: /* col_name */ - case 167: /* index_options */ - case 169: /* duration_literal */ - case 170: /* sliding_opt */ - case 171: /* func */ - case 175: /* query_expression */ - case 176: /* literal */ - case 179: /* expression */ - case 180: /* column_reference */ - case 181: /* subquery */ - case 182: /* predicate */ - case 185: /* in_predicate_value */ - case 186: /* boolean_value_expression */ - case 187: /* boolean_primary */ - case 188: /* common_expression */ - case 189: /* from_clause */ - case 190: /* table_reference_list */ - case 191: /* table_reference */ - case 192: /* table_primary */ - case 193: /* joined_table */ - case 195: /* parenthesized_joined_table */ - case 197: /* search_condition */ - case 198: /* query_specification */ - case 201: /* where_clause_opt */ - case 203: /* twindow_clause_opt */ - case 205: /* having_clause_opt */ - case 207: /* select_item */ - case 208: /* fill_opt */ - case 211: /* query_expression_body */ - case 213: /* slimit_clause_opt */ - case 214: /* limit_clause_opt */ - case 215: /* query_primary */ - case 217: /* sort_specification */ + case 160: /* cmd */ + case 163: /* literal */ + case 170: /* db_options */ + case 172: /* alter_db_options */ + case 174: /* full_table_name */ + case 177: /* table_options */ + case 181: /* alter_table_clause */ + case 182: /* alter_table_options */ + case 185: /* create_subtable_clause */ + case 188: /* drop_table_clause */ + case 191: /* column_def */ + case 194: /* col_name */ + case 195: /* func_name */ + case 198: /* index_options */ + case 200: /* duration_literal */ + case 201: /* sliding_opt */ + case 202: /* func */ + case 205: /* query_expression */ + case 208: /* expression */ + case 209: /* column_reference */ + case 210: /* subquery */ + case 211: /* predicate */ + case 214: /* in_predicate_value */ + case 215: /* boolean_value_expression */ + case 216: /* boolean_primary */ + case 217: /* common_expression */ + case 218: /* from_clause */ + case 219: /* table_reference_list */ + case 220: /* table_reference */ + case 221: /* table_primary */ + case 222: /* joined_table */ + case 224: /* parenthesized_joined_table */ + case 226: /* search_condition */ + case 227: /* query_specification */ + case 230: /* where_clause_opt */ + case 232: /* twindow_clause_opt */ + case 234: /* having_clause_opt */ + case 236: /* select_item */ + case 237: /* fill_opt */ + case 240: /* query_expression_body */ + case 242: /* slimit_clause_opt */ + case 243: /* limit_clause_opt */ + case 244: /* query_primary */ + case 246: /* sort_specification */ { - nodesDestroyNode((yypminor->yy432)); + nodesDestroyNode((yypminor->yy26)); } break; - case 142: /* user_name */ - case 143: /* dnode_endpoint */ - case 144: /* dnode_host_name */ - case 146: /* db_name */ - case 161: /* table_name */ - case 163: /* column_name */ - case 166: /* index_name */ - case 172: /* function_name */ - case 174: /* topic_name */ - case 177: /* table_alias */ - case 178: /* column_alias */ - case 194: /* alias_opt */ + case 161: /* account_options */ + case 162: /* alter_account_options */ + case 164: /* alter_account_option */ { } break; - case 145: /* not_exists_opt */ - case 148: /* exists_opt */ - case 199: /* set_quantifier_opt */ + case 165: /* user_name */ + case 166: /* dnode_endpoint */ + case 167: /* dnode_host_name */ + case 169: /* db_name */ + case 183: /* column_name */ + case 190: /* table_name */ + case 196: /* function_name */ + case 197: /* index_name */ + case 204: /* topic_name */ + case 206: /* table_alias */ + case 207: /* column_alias */ + case 223: /* alias_opt */ { } break; - case 150: /* column_def_list */ - case 151: /* tags_def_opt */ - case 153: /* multi_create_clause */ - case 154: /* tags_def */ - case 155: /* multi_drop_clause */ - case 157: /* specific_tags_opt */ - case 158: /* literal_list */ - case 160: /* col_name_list */ - case 168: /* func_list */ - case 173: /* expression_list */ - case 200: /* select_list */ - case 202: /* partition_by_clause_opt */ - case 204: /* group_by_clause_opt */ - case 206: /* select_sublist */ - case 210: /* group_by_list */ - case 212: /* order_by_clause_opt */ - case 216: /* sort_specification_list */ -{ - nodesDestroyList((yypminor->yy24)); -} - break; - case 164: /* type_name */ + case 168: /* not_exists_opt */ + case 171: /* exists_opt */ + case 228: /* set_quantifier_opt */ { } break; - case 183: /* compare_op */ - case 184: /* in_op */ + case 173: /* alter_db_option */ + case 193: /* alter_table_option */ { } break; - case 196: /* join_type */ + case 175: /* column_def_list */ + case 176: /* tags_def_opt */ + case 178: /* multi_create_clause */ + case 179: /* tags_def */ + case 180: /* multi_drop_clause */ + case 186: /* specific_tags_opt */ + case 187: /* literal_list */ + case 189: /* col_name_list */ + case 192: /* func_name_list */ + case 199: /* func_list */ + case 203: /* expression_list */ + case 229: /* select_list */ + case 231: /* partition_by_clause_opt */ + case 233: /* group_by_clause_opt */ + case 235: /* select_sublist */ + case 239: /* group_by_list */ + case 241: /* order_by_clause_opt */ + case 245: /* sort_specification_list */ +{ + nodesDestroyList((yypminor->yy64)); +} + break; + case 184: /* type_name */ { } break; - case 209: /* fill_mode */ + case 212: /* compare_op */ + case 213: /* in_op */ { } break; - case 218: /* ordering_specification_opt */ + case 225: /* join_type */ { } break; - case 219: /* null_ordering_opt */ + case 238: /* fill_mode */ +{ + +} + break; + case 247: /* ordering_specification_opt */ +{ + +} + break; + case 248: /* null_ordering_opt */ { } @@ -1649,261 +1816,322 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ signed char nrhs; /* Negative of the number of RHS symbols in the rule */ } yyRuleInfo[] = { - { 141, -5 }, /* (0) cmd ::= CREATE USER user_name PASS NK_STRING */ - { 141, -5 }, /* (1) cmd ::= ALTER USER user_name PASS NK_STRING */ - { 141, -5 }, /* (2) cmd ::= ALTER USER user_name PRIVILEGE NK_STRING */ - { 141, -3 }, /* (3) cmd ::= DROP USER user_name */ - { 141, -2 }, /* (4) cmd ::= SHOW USERS */ - { 141, -3 }, /* (5) cmd ::= CREATE DNODE dnode_endpoint */ - { 141, -5 }, /* (6) cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER */ - { 141, -3 }, /* (7) cmd ::= DROP DNODE NK_INTEGER */ - { 141, -3 }, /* (8) cmd ::= DROP DNODE dnode_endpoint */ - { 141, -2 }, /* (9) cmd ::= SHOW DNODES */ - { 143, -1 }, /* (10) dnode_endpoint ::= NK_STRING */ - { 144, -1 }, /* (11) dnode_host_name ::= NK_ID */ - { 144, -1 }, /* (12) dnode_host_name ::= NK_IPTOKEN */ - { 141, -5 }, /* (13) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ - { 141, -5 }, /* (14) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ - { 141, -2 }, /* (15) cmd ::= SHOW QNODES */ - { 141, -5 }, /* (16) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ - { 141, -4 }, /* (17) cmd ::= DROP DATABASE exists_opt db_name */ - { 141, -2 }, /* (18) cmd ::= SHOW DATABASES */ - { 141, -2 }, /* (19) cmd ::= USE db_name */ - { 141, -4 }, /* (20) cmd ::= ALTER DATABASE db_name db_options */ - { 145, -3 }, /* (21) not_exists_opt ::= IF NOT EXISTS */ - { 145, 0 }, /* (22) not_exists_opt ::= */ - { 148, -2 }, /* (23) exists_opt ::= IF EXISTS */ - { 148, 0 }, /* (24) exists_opt ::= */ - { 147, 0 }, /* (25) db_options ::= */ - { 147, -3 }, /* (26) db_options ::= db_options BLOCKS NK_INTEGER */ - { 147, -3 }, /* (27) db_options ::= db_options CACHE NK_INTEGER */ - { 147, -3 }, /* (28) db_options ::= db_options CACHELAST NK_INTEGER */ - { 147, -3 }, /* (29) db_options ::= db_options COMP NK_INTEGER */ - { 147, -3 }, /* (30) db_options ::= db_options DAYS NK_INTEGER */ - { 147, -3 }, /* (31) db_options ::= db_options FSYNC NK_INTEGER */ - { 147, -3 }, /* (32) db_options ::= db_options MAXROWS NK_INTEGER */ - { 147, -3 }, /* (33) db_options ::= db_options MINROWS NK_INTEGER */ - { 147, -3 }, /* (34) db_options ::= db_options KEEP NK_INTEGER */ - { 147, -3 }, /* (35) db_options ::= db_options PRECISION NK_STRING */ - { 147, -3 }, /* (36) db_options ::= db_options QUORUM NK_INTEGER */ - { 147, -3 }, /* (37) db_options ::= db_options REPLICA NK_INTEGER */ - { 147, -3 }, /* (38) db_options ::= db_options TTL NK_INTEGER */ - { 147, -3 }, /* (39) db_options ::= db_options WAL NK_INTEGER */ - { 147, -3 }, /* (40) db_options ::= db_options VGROUPS NK_INTEGER */ - { 147, -3 }, /* (41) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ - { 147, -3 }, /* (42) db_options ::= db_options STREAM_MODE NK_INTEGER */ - { 141, -9 }, /* (43) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - { 141, -3 }, /* (44) cmd ::= CREATE TABLE multi_create_clause */ - { 141, -9 }, /* (45) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ - { 141, -3 }, /* (46) cmd ::= DROP TABLE multi_drop_clause */ - { 141, -4 }, /* (47) cmd ::= DROP STABLE exists_opt full_table_name */ - { 141, -2 }, /* (48) cmd ::= SHOW TABLES */ - { 141, -2 }, /* (49) cmd ::= SHOW STABLES */ - { 153, -1 }, /* (50) multi_create_clause ::= create_subtable_clause */ - { 153, -2 }, /* (51) multi_create_clause ::= multi_create_clause create_subtable_clause */ - { 156, -9 }, /* (52) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP */ - { 155, -1 }, /* (53) multi_drop_clause ::= drop_table_clause */ - { 155, -2 }, /* (54) multi_drop_clause ::= multi_drop_clause drop_table_clause */ - { 159, -2 }, /* (55) drop_table_clause ::= exists_opt full_table_name */ - { 157, 0 }, /* (56) specific_tags_opt ::= */ - { 157, -3 }, /* (57) specific_tags_opt ::= NK_LP col_name_list NK_RP */ - { 149, -1 }, /* (58) full_table_name ::= table_name */ - { 149, -3 }, /* (59) full_table_name ::= db_name NK_DOT table_name */ - { 150, -1 }, /* (60) column_def_list ::= column_def */ - { 150, -3 }, /* (61) column_def_list ::= column_def_list NK_COMMA column_def */ - { 162, -2 }, /* (62) column_def ::= column_name type_name */ - { 162, -4 }, /* (63) column_def ::= column_name type_name COMMENT NK_STRING */ - { 164, -1 }, /* (64) type_name ::= BOOL */ - { 164, -1 }, /* (65) type_name ::= TINYINT */ - { 164, -1 }, /* (66) type_name ::= SMALLINT */ - { 164, -1 }, /* (67) type_name ::= INT */ - { 164, -1 }, /* (68) type_name ::= INTEGER */ - { 164, -1 }, /* (69) type_name ::= BIGINT */ - { 164, -1 }, /* (70) type_name ::= FLOAT */ - { 164, -1 }, /* (71) type_name ::= DOUBLE */ - { 164, -4 }, /* (72) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ - { 164, -1 }, /* (73) type_name ::= TIMESTAMP */ - { 164, -4 }, /* (74) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ - { 164, -2 }, /* (75) type_name ::= TINYINT UNSIGNED */ - { 164, -2 }, /* (76) type_name ::= SMALLINT UNSIGNED */ - { 164, -2 }, /* (77) type_name ::= INT UNSIGNED */ - { 164, -2 }, /* (78) type_name ::= BIGINT UNSIGNED */ - { 164, -1 }, /* (79) type_name ::= JSON */ - { 164, -4 }, /* (80) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ - { 164, -1 }, /* (81) type_name ::= MEDIUMBLOB */ - { 164, -1 }, /* (82) type_name ::= BLOB */ - { 164, -4 }, /* (83) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ - { 164, -1 }, /* (84) type_name ::= DECIMAL */ - { 164, -4 }, /* (85) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ - { 164, -6 }, /* (86) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ - { 151, 0 }, /* (87) tags_def_opt ::= */ - { 151, -1 }, /* (88) tags_def_opt ::= tags_def */ - { 154, -4 }, /* (89) tags_def ::= TAGS NK_LP column_def_list NK_RP */ - { 152, 0 }, /* (90) table_options ::= */ - { 152, -3 }, /* (91) table_options ::= table_options COMMENT NK_STRING */ - { 152, -3 }, /* (92) table_options ::= table_options KEEP NK_INTEGER */ - { 152, -3 }, /* (93) table_options ::= table_options TTL NK_INTEGER */ - { 152, -5 }, /* (94) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ - { 160, -1 }, /* (95) col_name_list ::= col_name */ - { 160, -3 }, /* (96) col_name_list ::= col_name_list NK_COMMA col_name */ - { 165, -1 }, /* (97) col_name ::= column_name */ - { 141, -7 }, /* (98) cmd ::= CREATE SMA INDEX index_name ON table_name index_options */ - { 141, -9 }, /* (99) cmd ::= CREATE FULLTEXT INDEX index_name ON table_name NK_LP col_name_list NK_RP */ - { 141, -5 }, /* (100) cmd ::= DROP INDEX index_name ON table_name */ - { 167, 0 }, /* (101) index_options ::= */ - { 167, -9 }, /* (102) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ - { 167, -11 }, /* (103) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ - { 168, -1 }, /* (104) func_list ::= func */ - { 168, -3 }, /* (105) func_list ::= func_list NK_COMMA func */ - { 171, -4 }, /* (106) func ::= function_name NK_LP expression_list NK_RP */ - { 141, -6 }, /* (107) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ - { 141, -6 }, /* (108) cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name */ - { 141, -4 }, /* (109) cmd ::= DROP TOPIC exists_opt topic_name */ - { 141, -2 }, /* (110) cmd ::= SHOW VGROUPS */ - { 141, -4 }, /* (111) cmd ::= SHOW db_name NK_DOT VGROUPS */ - { 141, -2 }, /* (112) cmd ::= SHOW MNODES */ - { 141, -1 }, /* (113) cmd ::= query_expression */ - { 176, -1 }, /* (114) literal ::= NK_INTEGER */ - { 176, -1 }, /* (115) literal ::= NK_FLOAT */ - { 176, -1 }, /* (116) literal ::= NK_STRING */ - { 176, -1 }, /* (117) literal ::= NK_BOOL */ - { 176, -2 }, /* (118) literal ::= TIMESTAMP NK_STRING */ - { 176, -1 }, /* (119) literal ::= duration_literal */ - { 169, -1 }, /* (120) duration_literal ::= NK_VARIABLE */ - { 158, -1 }, /* (121) literal_list ::= literal */ - { 158, -3 }, /* (122) literal_list ::= literal_list NK_COMMA literal */ - { 146, -1 }, /* (123) db_name ::= NK_ID */ - { 161, -1 }, /* (124) table_name ::= NK_ID */ - { 163, -1 }, /* (125) column_name ::= NK_ID */ - { 172, -1 }, /* (126) function_name ::= NK_ID */ - { 177, -1 }, /* (127) table_alias ::= NK_ID */ - { 178, -1 }, /* (128) column_alias ::= NK_ID */ - { 142, -1 }, /* (129) user_name ::= NK_ID */ - { 166, -1 }, /* (130) index_name ::= NK_ID */ - { 174, -1 }, /* (131) topic_name ::= NK_ID */ - { 179, -1 }, /* (132) expression ::= literal */ - { 179, -1 }, /* (133) expression ::= column_reference */ - { 179, -4 }, /* (134) expression ::= function_name NK_LP expression_list NK_RP */ - { 179, -4 }, /* (135) expression ::= function_name NK_LP NK_STAR NK_RP */ - { 179, -1 }, /* (136) expression ::= subquery */ - { 179, -3 }, /* (137) expression ::= NK_LP expression NK_RP */ - { 179, -2 }, /* (138) expression ::= NK_PLUS expression */ - { 179, -2 }, /* (139) expression ::= NK_MINUS expression */ - { 179, -3 }, /* (140) expression ::= expression NK_PLUS expression */ - { 179, -3 }, /* (141) expression ::= expression NK_MINUS expression */ - { 179, -3 }, /* (142) expression ::= expression NK_STAR expression */ - { 179, -3 }, /* (143) expression ::= expression NK_SLASH expression */ - { 179, -3 }, /* (144) expression ::= expression NK_REM expression */ - { 173, -1 }, /* (145) expression_list ::= expression */ - { 173, -3 }, /* (146) expression_list ::= expression_list NK_COMMA expression */ - { 180, -1 }, /* (147) column_reference ::= column_name */ - { 180, -3 }, /* (148) column_reference ::= table_name NK_DOT column_name */ - { 182, -3 }, /* (149) predicate ::= expression compare_op expression */ - { 182, -5 }, /* (150) predicate ::= expression BETWEEN expression AND expression */ - { 182, -6 }, /* (151) predicate ::= expression NOT BETWEEN expression AND expression */ - { 182, -3 }, /* (152) predicate ::= expression IS NULL */ - { 182, -4 }, /* (153) predicate ::= expression IS NOT NULL */ - { 182, -3 }, /* (154) predicate ::= expression in_op in_predicate_value */ - { 183, -1 }, /* (155) compare_op ::= NK_LT */ - { 183, -1 }, /* (156) compare_op ::= NK_GT */ - { 183, -1 }, /* (157) compare_op ::= NK_LE */ - { 183, -1 }, /* (158) compare_op ::= NK_GE */ - { 183, -1 }, /* (159) compare_op ::= NK_NE */ - { 183, -1 }, /* (160) compare_op ::= NK_EQ */ - { 183, -1 }, /* (161) compare_op ::= LIKE */ - { 183, -2 }, /* (162) compare_op ::= NOT LIKE */ - { 183, -1 }, /* (163) compare_op ::= MATCH */ - { 183, -1 }, /* (164) compare_op ::= NMATCH */ - { 184, -1 }, /* (165) in_op ::= IN */ - { 184, -2 }, /* (166) in_op ::= NOT IN */ - { 185, -3 }, /* (167) in_predicate_value ::= NK_LP expression_list NK_RP */ - { 186, -1 }, /* (168) boolean_value_expression ::= boolean_primary */ - { 186, -2 }, /* (169) boolean_value_expression ::= NOT boolean_primary */ - { 186, -3 }, /* (170) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ - { 186, -3 }, /* (171) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ - { 187, -1 }, /* (172) boolean_primary ::= predicate */ - { 187, -3 }, /* (173) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - { 188, -1 }, /* (174) common_expression ::= expression */ - { 188, -1 }, /* (175) common_expression ::= boolean_value_expression */ - { 189, -2 }, /* (176) from_clause ::= FROM table_reference_list */ - { 190, -1 }, /* (177) table_reference_list ::= table_reference */ - { 190, -3 }, /* (178) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - { 191, -1 }, /* (179) table_reference ::= table_primary */ - { 191, -1 }, /* (180) table_reference ::= joined_table */ - { 192, -2 }, /* (181) table_primary ::= table_name alias_opt */ - { 192, -4 }, /* (182) table_primary ::= db_name NK_DOT table_name alias_opt */ - { 192, -2 }, /* (183) table_primary ::= subquery alias_opt */ - { 192, -1 }, /* (184) table_primary ::= parenthesized_joined_table */ - { 194, 0 }, /* (185) alias_opt ::= */ - { 194, -1 }, /* (186) alias_opt ::= table_alias */ - { 194, -2 }, /* (187) alias_opt ::= AS table_alias */ - { 195, -3 }, /* (188) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - { 195, -3 }, /* (189) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - { 193, -6 }, /* (190) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - { 196, 0 }, /* (191) join_type ::= */ - { 196, -1 }, /* (192) join_type ::= INNER */ - { 198, -9 }, /* (193) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - { 199, 0 }, /* (194) set_quantifier_opt ::= */ - { 199, -1 }, /* (195) set_quantifier_opt ::= DISTINCT */ - { 199, -1 }, /* (196) set_quantifier_opt ::= ALL */ - { 200, -1 }, /* (197) select_list ::= NK_STAR */ - { 200, -1 }, /* (198) select_list ::= select_sublist */ - { 206, -1 }, /* (199) select_sublist ::= select_item */ - { 206, -3 }, /* (200) select_sublist ::= select_sublist NK_COMMA select_item */ - { 207, -1 }, /* (201) select_item ::= common_expression */ - { 207, -2 }, /* (202) select_item ::= common_expression column_alias */ - { 207, -3 }, /* (203) select_item ::= common_expression AS column_alias */ - { 207, -3 }, /* (204) select_item ::= table_name NK_DOT NK_STAR */ - { 201, 0 }, /* (205) where_clause_opt ::= */ - { 201, -2 }, /* (206) where_clause_opt ::= WHERE search_condition */ - { 202, 0 }, /* (207) partition_by_clause_opt ::= */ - { 202, -3 }, /* (208) partition_by_clause_opt ::= PARTITION BY expression_list */ - { 203, 0 }, /* (209) twindow_clause_opt ::= */ - { 203, -6 }, /* (210) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ - { 203, -4 }, /* (211) twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ - { 203, -6 }, /* (212) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - { 203, -8 }, /* (213) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - { 170, 0 }, /* (214) sliding_opt ::= */ - { 170, -4 }, /* (215) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - { 208, 0 }, /* (216) fill_opt ::= */ - { 208, -4 }, /* (217) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - { 208, -6 }, /* (218) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ - { 209, -1 }, /* (219) fill_mode ::= NONE */ - { 209, -1 }, /* (220) fill_mode ::= PREV */ - { 209, -1 }, /* (221) fill_mode ::= NULL */ - { 209, -1 }, /* (222) fill_mode ::= LINEAR */ - { 209, -1 }, /* (223) fill_mode ::= NEXT */ - { 204, 0 }, /* (224) group_by_clause_opt ::= */ - { 204, -3 }, /* (225) group_by_clause_opt ::= GROUP BY group_by_list */ - { 210, -1 }, /* (226) group_by_list ::= expression */ - { 210, -3 }, /* (227) group_by_list ::= group_by_list NK_COMMA expression */ - { 205, 0 }, /* (228) having_clause_opt ::= */ - { 205, -2 }, /* (229) having_clause_opt ::= HAVING search_condition */ - { 175, -4 }, /* (230) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 211, -1 }, /* (231) query_expression_body ::= query_primary */ - { 211, -4 }, /* (232) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ - { 215, -1 }, /* (233) query_primary ::= query_specification */ - { 212, 0 }, /* (234) order_by_clause_opt ::= */ - { 212, -3 }, /* (235) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 213, 0 }, /* (236) slimit_clause_opt ::= */ - { 213, -2 }, /* (237) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - { 213, -4 }, /* (238) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 213, -4 }, /* (239) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 214, 0 }, /* (240) limit_clause_opt ::= */ - { 214, -2 }, /* (241) limit_clause_opt ::= LIMIT NK_INTEGER */ - { 214, -4 }, /* (242) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 214, -4 }, /* (243) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 181, -3 }, /* (244) subquery ::= NK_LP query_expression NK_RP */ - { 197, -1 }, /* (245) search_condition ::= common_expression */ - { 216, -1 }, /* (246) sort_specification_list ::= sort_specification */ - { 216, -3 }, /* (247) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 217, -3 }, /* (248) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ - { 218, 0 }, /* (249) ordering_specification_opt ::= */ - { 218, -1 }, /* (250) ordering_specification_opt ::= ASC */ - { 218, -1 }, /* (251) ordering_specification_opt ::= DESC */ - { 219, 0 }, /* (252) null_ordering_opt ::= */ - { 219, -2 }, /* (253) null_ordering_opt ::= NULLS FIRST */ - { 219, -2 }, /* (254) null_ordering_opt ::= NULLS LAST */ + { 160, -6 }, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ + { 160, -4 }, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ + { 161, 0 }, /* (2) account_options ::= */ + { 161, -3 }, /* (3) account_options ::= account_options PPS literal */ + { 161, -3 }, /* (4) account_options ::= account_options TSERIES literal */ + { 161, -3 }, /* (5) account_options ::= account_options STORAGE literal */ + { 161, -3 }, /* (6) account_options ::= account_options STREAMS literal */ + { 161, -3 }, /* (7) account_options ::= account_options QTIME literal */ + { 161, -3 }, /* (8) account_options ::= account_options DBS literal */ + { 161, -3 }, /* (9) account_options ::= account_options USERS literal */ + { 161, -3 }, /* (10) account_options ::= account_options CONNS literal */ + { 161, -3 }, /* (11) account_options ::= account_options STATE literal */ + { 162, -1 }, /* (12) alter_account_options ::= alter_account_option */ + { 162, -2 }, /* (13) alter_account_options ::= alter_account_options alter_account_option */ + { 164, -2 }, /* (14) alter_account_option ::= PASS literal */ + { 164, -2 }, /* (15) alter_account_option ::= PPS literal */ + { 164, -2 }, /* (16) alter_account_option ::= TSERIES literal */ + { 164, -2 }, /* (17) alter_account_option ::= STORAGE literal */ + { 164, -2 }, /* (18) alter_account_option ::= STREAMS literal */ + { 164, -2 }, /* (19) alter_account_option ::= QTIME literal */ + { 164, -2 }, /* (20) alter_account_option ::= DBS literal */ + { 164, -2 }, /* (21) alter_account_option ::= USERS literal */ + { 164, -2 }, /* (22) alter_account_option ::= CONNS literal */ + { 164, -2 }, /* (23) alter_account_option ::= STATE literal */ + { 160, -5 }, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING */ + { 160, -5 }, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ + { 160, -5 }, /* (26) cmd ::= ALTER USER user_name PRIVILEGE NK_STRING */ + { 160, -3 }, /* (27) cmd ::= DROP USER user_name */ + { 160, -2 }, /* (28) cmd ::= SHOW USERS */ + { 160, -3 }, /* (29) cmd ::= CREATE DNODE dnode_endpoint */ + { 160, -5 }, /* (30) cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER */ + { 160, -3 }, /* (31) cmd ::= DROP DNODE NK_INTEGER */ + { 160, -3 }, /* (32) cmd ::= DROP DNODE dnode_endpoint */ + { 160, -2 }, /* (33) cmd ::= SHOW DNODES */ + { 160, -4 }, /* (34) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ + { 160, -5 }, /* (35) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ + { 160, -4 }, /* (36) cmd ::= ALTER ALL DNODES NK_STRING */ + { 160, -5 }, /* (37) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ + { 166, -1 }, /* (38) dnode_endpoint ::= NK_STRING */ + { 167, -1 }, /* (39) dnode_host_name ::= NK_ID */ + { 167, -1 }, /* (40) dnode_host_name ::= NK_IPTOKEN */ + { 160, -3 }, /* (41) cmd ::= ALTER LOCAL NK_STRING */ + { 160, -4 }, /* (42) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ + { 160, -5 }, /* (43) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ + { 160, -5 }, /* (44) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ + { 160, -2 }, /* (45) cmd ::= SHOW QNODES */ + { 160, -5 }, /* (46) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ + { 160, -4 }, /* (47) cmd ::= DROP DATABASE exists_opt db_name */ + { 160, -2 }, /* (48) cmd ::= SHOW DATABASES */ + { 160, -2 }, /* (49) cmd ::= USE db_name */ + { 160, -4 }, /* (50) cmd ::= ALTER DATABASE db_name alter_db_options */ + { 168, -3 }, /* (51) not_exists_opt ::= IF NOT EXISTS */ + { 168, 0 }, /* (52) not_exists_opt ::= */ + { 171, -2 }, /* (53) exists_opt ::= IF EXISTS */ + { 171, 0 }, /* (54) exists_opt ::= */ + { 170, 0 }, /* (55) db_options ::= */ + { 170, -3 }, /* (56) db_options ::= db_options BLOCKS NK_INTEGER */ + { 170, -3 }, /* (57) db_options ::= db_options CACHE NK_INTEGER */ + { 170, -3 }, /* (58) db_options ::= db_options CACHELAST NK_INTEGER */ + { 170, -3 }, /* (59) db_options ::= db_options COMP NK_INTEGER */ + { 170, -3 }, /* (60) db_options ::= db_options DAYS NK_INTEGER */ + { 170, -3 }, /* (61) db_options ::= db_options FSYNC NK_INTEGER */ + { 170, -3 }, /* (62) db_options ::= db_options MAXROWS NK_INTEGER */ + { 170, -3 }, /* (63) db_options ::= db_options MINROWS NK_INTEGER */ + { 170, -3 }, /* (64) db_options ::= db_options KEEP NK_INTEGER */ + { 170, -3 }, /* (65) db_options ::= db_options PRECISION NK_STRING */ + { 170, -3 }, /* (66) db_options ::= db_options QUORUM NK_INTEGER */ + { 170, -3 }, /* (67) db_options ::= db_options REPLICA NK_INTEGER */ + { 170, -3 }, /* (68) db_options ::= db_options TTL NK_INTEGER */ + { 170, -3 }, /* (69) db_options ::= db_options WAL NK_INTEGER */ + { 170, -3 }, /* (70) db_options ::= db_options VGROUPS NK_INTEGER */ + { 170, -3 }, /* (71) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ + { 170, -3 }, /* (72) db_options ::= db_options STREAM_MODE NK_INTEGER */ + { 170, -3 }, /* (73) db_options ::= db_options RETENTIONS NK_STRING */ + { 170, -3 }, /* (74) db_options ::= db_options FILE_FACTOR NK_FLOAT */ + { 172, -1 }, /* (75) alter_db_options ::= alter_db_option */ + { 172, -2 }, /* (76) alter_db_options ::= alter_db_options alter_db_option */ + { 173, -2 }, /* (77) alter_db_option ::= BLOCKS NK_INTEGER */ + { 173, -2 }, /* (78) alter_db_option ::= FSYNC NK_INTEGER */ + { 173, -2 }, /* (79) alter_db_option ::= KEEP NK_INTEGER */ + { 173, -2 }, /* (80) alter_db_option ::= WAL NK_INTEGER */ + { 173, -2 }, /* (81) alter_db_option ::= QUORUM NK_INTEGER */ + { 173, -2 }, /* (82) alter_db_option ::= CACHELAST NK_INTEGER */ + { 160, -9 }, /* (83) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + { 160, -3 }, /* (84) cmd ::= CREATE TABLE multi_create_clause */ + { 160, -9 }, /* (85) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ + { 160, -3 }, /* (86) cmd ::= DROP TABLE multi_drop_clause */ + { 160, -4 }, /* (87) cmd ::= DROP STABLE exists_opt full_table_name */ + { 160, -2 }, /* (88) cmd ::= SHOW TABLES */ + { 160, -2 }, /* (89) cmd ::= SHOW STABLES */ + { 160, -3 }, /* (90) cmd ::= ALTER TABLE alter_table_clause */ + { 160, -3 }, /* (91) cmd ::= ALTER STABLE alter_table_clause */ + { 181, -2 }, /* (92) alter_table_clause ::= full_table_name alter_table_options */ + { 181, -5 }, /* (93) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ + { 181, -4 }, /* (94) alter_table_clause ::= full_table_name DROP COLUMN column_name */ + { 181, -5 }, /* (95) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ + { 181, -5 }, /* (96) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ + { 181, -5 }, /* (97) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ + { 181, -4 }, /* (98) alter_table_clause ::= full_table_name DROP TAG column_name */ + { 181, -5 }, /* (99) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ + { 181, -5 }, /* (100) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ + { 181, -6 }, /* (101) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal */ + { 178, -1 }, /* (102) multi_create_clause ::= create_subtable_clause */ + { 178, -2 }, /* (103) multi_create_clause ::= multi_create_clause create_subtable_clause */ + { 185, -9 }, /* (104) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP */ + { 180, -1 }, /* (105) multi_drop_clause ::= drop_table_clause */ + { 180, -2 }, /* (106) multi_drop_clause ::= multi_drop_clause drop_table_clause */ + { 188, -2 }, /* (107) drop_table_clause ::= exists_opt full_table_name */ + { 186, 0 }, /* (108) specific_tags_opt ::= */ + { 186, -3 }, /* (109) specific_tags_opt ::= NK_LP col_name_list NK_RP */ + { 174, -1 }, /* (110) full_table_name ::= table_name */ + { 174, -3 }, /* (111) full_table_name ::= db_name NK_DOT table_name */ + { 175, -1 }, /* (112) column_def_list ::= column_def */ + { 175, -3 }, /* (113) column_def_list ::= column_def_list NK_COMMA column_def */ + { 191, -2 }, /* (114) column_def ::= column_name type_name */ + { 191, -4 }, /* (115) column_def ::= column_name type_name COMMENT NK_STRING */ + { 184, -1 }, /* (116) type_name ::= BOOL */ + { 184, -1 }, /* (117) type_name ::= TINYINT */ + { 184, -1 }, /* (118) type_name ::= SMALLINT */ + { 184, -1 }, /* (119) type_name ::= INT */ + { 184, -1 }, /* (120) type_name ::= INTEGER */ + { 184, -1 }, /* (121) type_name ::= BIGINT */ + { 184, -1 }, /* (122) type_name ::= FLOAT */ + { 184, -1 }, /* (123) type_name ::= DOUBLE */ + { 184, -4 }, /* (124) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ + { 184, -1 }, /* (125) type_name ::= TIMESTAMP */ + { 184, -4 }, /* (126) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ + { 184, -2 }, /* (127) type_name ::= TINYINT UNSIGNED */ + { 184, -2 }, /* (128) type_name ::= SMALLINT UNSIGNED */ + { 184, -2 }, /* (129) type_name ::= INT UNSIGNED */ + { 184, -2 }, /* (130) type_name ::= BIGINT UNSIGNED */ + { 184, -1 }, /* (131) type_name ::= JSON */ + { 184, -4 }, /* (132) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ + { 184, -1 }, /* (133) type_name ::= MEDIUMBLOB */ + { 184, -1 }, /* (134) type_name ::= BLOB */ + { 184, -4 }, /* (135) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ + { 184, -1 }, /* (136) type_name ::= DECIMAL */ + { 184, -4 }, /* (137) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ + { 184, -6 }, /* (138) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ + { 176, 0 }, /* (139) tags_def_opt ::= */ + { 176, -1 }, /* (140) tags_def_opt ::= tags_def */ + { 179, -4 }, /* (141) tags_def ::= TAGS NK_LP column_def_list NK_RP */ + { 177, 0 }, /* (142) table_options ::= */ + { 177, -3 }, /* (143) table_options ::= table_options COMMENT NK_STRING */ + { 177, -3 }, /* (144) table_options ::= table_options KEEP NK_INTEGER */ + { 177, -3 }, /* (145) table_options ::= table_options TTL NK_INTEGER */ + { 177, -5 }, /* (146) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ + { 177, -5 }, /* (147) table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP */ + { 182, -1 }, /* (148) alter_table_options ::= alter_table_option */ + { 182, -2 }, /* (149) alter_table_options ::= alter_table_options alter_table_option */ + { 193, -2 }, /* (150) alter_table_option ::= COMMENT NK_STRING */ + { 193, -2 }, /* (151) alter_table_option ::= KEEP NK_INTEGER */ + { 193, -2 }, /* (152) alter_table_option ::= TTL NK_INTEGER */ + { 189, -1 }, /* (153) col_name_list ::= col_name */ + { 189, -3 }, /* (154) col_name_list ::= col_name_list NK_COMMA col_name */ + { 194, -1 }, /* (155) col_name ::= column_name */ + { 192, -1 }, /* (156) func_name_list ::= func_name */ + { 192, -3 }, /* (157) func_name_list ::= func_name_list NK_COMMA col_name */ + { 195, -1 }, /* (158) func_name ::= function_name */ + { 160, -7 }, /* (159) cmd ::= CREATE SMA INDEX index_name ON table_name index_options */ + { 160, -9 }, /* (160) cmd ::= CREATE FULLTEXT INDEX index_name ON table_name NK_LP col_name_list NK_RP */ + { 160, -5 }, /* (161) cmd ::= DROP INDEX index_name ON table_name */ + { 198, 0 }, /* (162) index_options ::= */ + { 198, -9 }, /* (163) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ + { 198, -11 }, /* (164) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ + { 199, -1 }, /* (165) func_list ::= func */ + { 199, -3 }, /* (166) func_list ::= func_list NK_COMMA func */ + { 202, -4 }, /* (167) func ::= function_name NK_LP expression_list NK_RP */ + { 160, -6 }, /* (168) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ + { 160, -6 }, /* (169) cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name */ + { 160, -4 }, /* (170) cmd ::= DROP TOPIC exists_opt topic_name */ + { 160, -2 }, /* (171) cmd ::= SHOW VGROUPS */ + { 160, -4 }, /* (172) cmd ::= SHOW db_name NK_DOT VGROUPS */ + { 160, -2 }, /* (173) cmd ::= SHOW MNODES */ + { 160, -1 }, /* (174) cmd ::= query_expression */ + { 163, -1 }, /* (175) literal ::= NK_INTEGER */ + { 163, -1 }, /* (176) literal ::= NK_FLOAT */ + { 163, -1 }, /* (177) literal ::= NK_STRING */ + { 163, -1 }, /* (178) literal ::= NK_BOOL */ + { 163, -2 }, /* (179) literal ::= TIMESTAMP NK_STRING */ + { 163, -1 }, /* (180) literal ::= duration_literal */ + { 200, -1 }, /* (181) duration_literal ::= NK_VARIABLE */ + { 187, -1 }, /* (182) literal_list ::= literal */ + { 187, -3 }, /* (183) literal_list ::= literal_list NK_COMMA literal */ + { 169, -1 }, /* (184) db_name ::= NK_ID */ + { 190, -1 }, /* (185) table_name ::= NK_ID */ + { 183, -1 }, /* (186) column_name ::= NK_ID */ + { 196, -1 }, /* (187) function_name ::= NK_ID */ + { 206, -1 }, /* (188) table_alias ::= NK_ID */ + { 207, -1 }, /* (189) column_alias ::= NK_ID */ + { 165, -1 }, /* (190) user_name ::= NK_ID */ + { 197, -1 }, /* (191) index_name ::= NK_ID */ + { 204, -1 }, /* (192) topic_name ::= NK_ID */ + { 208, -1 }, /* (193) expression ::= literal */ + { 208, -1 }, /* (194) expression ::= column_reference */ + { 208, -4 }, /* (195) expression ::= function_name NK_LP expression_list NK_RP */ + { 208, -4 }, /* (196) expression ::= function_name NK_LP NK_STAR NK_RP */ + { 208, -1 }, /* (197) expression ::= subquery */ + { 208, -3 }, /* (198) expression ::= NK_LP expression NK_RP */ + { 208, -2 }, /* (199) expression ::= NK_PLUS expression */ + { 208, -2 }, /* (200) expression ::= NK_MINUS expression */ + { 208, -3 }, /* (201) expression ::= expression NK_PLUS expression */ + { 208, -3 }, /* (202) expression ::= expression NK_MINUS expression */ + { 208, -3 }, /* (203) expression ::= expression NK_STAR expression */ + { 208, -3 }, /* (204) expression ::= expression NK_SLASH expression */ + { 208, -3 }, /* (205) expression ::= expression NK_REM expression */ + { 203, -1 }, /* (206) expression_list ::= expression */ + { 203, -3 }, /* (207) expression_list ::= expression_list NK_COMMA expression */ + { 209, -1 }, /* (208) column_reference ::= column_name */ + { 209, -3 }, /* (209) column_reference ::= table_name NK_DOT column_name */ + { 211, -3 }, /* (210) predicate ::= expression compare_op expression */ + { 211, -5 }, /* (211) predicate ::= expression BETWEEN expression AND expression */ + { 211, -6 }, /* (212) predicate ::= expression NOT BETWEEN expression AND expression */ + { 211, -3 }, /* (213) predicate ::= expression IS NULL */ + { 211, -4 }, /* (214) predicate ::= expression IS NOT NULL */ + { 211, -3 }, /* (215) predicate ::= expression in_op in_predicate_value */ + { 212, -1 }, /* (216) compare_op ::= NK_LT */ + { 212, -1 }, /* (217) compare_op ::= NK_GT */ + { 212, -1 }, /* (218) compare_op ::= NK_LE */ + { 212, -1 }, /* (219) compare_op ::= NK_GE */ + { 212, -1 }, /* (220) compare_op ::= NK_NE */ + { 212, -1 }, /* (221) compare_op ::= NK_EQ */ + { 212, -1 }, /* (222) compare_op ::= LIKE */ + { 212, -2 }, /* (223) compare_op ::= NOT LIKE */ + { 212, -1 }, /* (224) compare_op ::= MATCH */ + { 212, -1 }, /* (225) compare_op ::= NMATCH */ + { 213, -1 }, /* (226) in_op ::= IN */ + { 213, -2 }, /* (227) in_op ::= NOT IN */ + { 214, -3 }, /* (228) in_predicate_value ::= NK_LP expression_list NK_RP */ + { 215, -1 }, /* (229) boolean_value_expression ::= boolean_primary */ + { 215, -2 }, /* (230) boolean_value_expression ::= NOT boolean_primary */ + { 215, -3 }, /* (231) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + { 215, -3 }, /* (232) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + { 216, -1 }, /* (233) boolean_primary ::= predicate */ + { 216, -3 }, /* (234) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + { 217, -1 }, /* (235) common_expression ::= expression */ + { 217, -1 }, /* (236) common_expression ::= boolean_value_expression */ + { 218, -2 }, /* (237) from_clause ::= FROM table_reference_list */ + { 219, -1 }, /* (238) table_reference_list ::= table_reference */ + { 219, -3 }, /* (239) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 220, -1 }, /* (240) table_reference ::= table_primary */ + { 220, -1 }, /* (241) table_reference ::= joined_table */ + { 221, -2 }, /* (242) table_primary ::= table_name alias_opt */ + { 221, -4 }, /* (243) table_primary ::= db_name NK_DOT table_name alias_opt */ + { 221, -2 }, /* (244) table_primary ::= subquery alias_opt */ + { 221, -1 }, /* (245) table_primary ::= parenthesized_joined_table */ + { 223, 0 }, /* (246) alias_opt ::= */ + { 223, -1 }, /* (247) alias_opt ::= table_alias */ + { 223, -2 }, /* (248) alias_opt ::= AS table_alias */ + { 224, -3 }, /* (249) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 224, -3 }, /* (250) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 222, -6 }, /* (251) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 225, 0 }, /* (252) join_type ::= */ + { 225, -1 }, /* (253) join_type ::= INNER */ + { 227, -9 }, /* (254) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 228, 0 }, /* (255) set_quantifier_opt ::= */ + { 228, -1 }, /* (256) set_quantifier_opt ::= DISTINCT */ + { 228, -1 }, /* (257) set_quantifier_opt ::= ALL */ + { 229, -1 }, /* (258) select_list ::= NK_STAR */ + { 229, -1 }, /* (259) select_list ::= select_sublist */ + { 235, -1 }, /* (260) select_sublist ::= select_item */ + { 235, -3 }, /* (261) select_sublist ::= select_sublist NK_COMMA select_item */ + { 236, -1 }, /* (262) select_item ::= common_expression */ + { 236, -2 }, /* (263) select_item ::= common_expression column_alias */ + { 236, -3 }, /* (264) select_item ::= common_expression AS column_alias */ + { 236, -3 }, /* (265) select_item ::= table_name NK_DOT NK_STAR */ + { 230, 0 }, /* (266) where_clause_opt ::= */ + { 230, -2 }, /* (267) where_clause_opt ::= WHERE search_condition */ + { 231, 0 }, /* (268) partition_by_clause_opt ::= */ + { 231, -3 }, /* (269) partition_by_clause_opt ::= PARTITION BY expression_list */ + { 232, 0 }, /* (270) twindow_clause_opt ::= */ + { 232, -6 }, /* (271) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ + { 232, -4 }, /* (272) twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ + { 232, -6 }, /* (273) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 232, -8 }, /* (274) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 201, 0 }, /* (275) sliding_opt ::= */ + { 201, -4 }, /* (276) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 237, 0 }, /* (277) fill_opt ::= */ + { 237, -4 }, /* (278) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 237, -6 }, /* (279) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + { 238, -1 }, /* (280) fill_mode ::= NONE */ + { 238, -1 }, /* (281) fill_mode ::= PREV */ + { 238, -1 }, /* (282) fill_mode ::= NULL */ + { 238, -1 }, /* (283) fill_mode ::= LINEAR */ + { 238, -1 }, /* (284) fill_mode ::= NEXT */ + { 233, 0 }, /* (285) group_by_clause_opt ::= */ + { 233, -3 }, /* (286) group_by_clause_opt ::= GROUP BY group_by_list */ + { 239, -1 }, /* (287) group_by_list ::= expression */ + { 239, -3 }, /* (288) group_by_list ::= group_by_list NK_COMMA expression */ + { 234, 0 }, /* (289) having_clause_opt ::= */ + { 234, -2 }, /* (290) having_clause_opt ::= HAVING search_condition */ + { 205, -4 }, /* (291) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 240, -1 }, /* (292) query_expression_body ::= query_primary */ + { 240, -4 }, /* (293) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ + { 244, -1 }, /* (294) query_primary ::= query_specification */ + { 241, 0 }, /* (295) order_by_clause_opt ::= */ + { 241, -3 }, /* (296) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 242, 0 }, /* (297) slimit_clause_opt ::= */ + { 242, -2 }, /* (298) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 242, -4 }, /* (299) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 242, -4 }, /* (300) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 243, 0 }, /* (301) limit_clause_opt ::= */ + { 243, -2 }, /* (302) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 243, -4 }, /* (303) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 243, -4 }, /* (304) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 210, -3 }, /* (305) subquery ::= NK_LP query_expression NK_RP */ + { 226, -1 }, /* (306) search_condition ::= common_expression */ + { 245, -1 }, /* (307) sort_specification_list ::= sort_specification */ + { 245, -3 }, /* (308) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 246, -3 }, /* (309) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + { 247, 0 }, /* (310) ordering_specification_opt ::= */ + { 247, -1 }, /* (311) ordering_specification_opt ::= ASC */ + { 247, -1 }, /* (312) ordering_specification_opt ::= DESC */ + { 248, 0 }, /* (313) null_ordering_opt ::= */ + { 248, -2 }, /* (314) null_ordering_opt ::= NULLS FIRST */ + { 248, -2 }, /* (315) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -1990,794 +2218,964 @@ static YYACTIONTYPE yy_reduce( */ /********** Begin reduce actions **********************************************/ YYMINORTYPE yylhsminor; - case 0: /* cmd ::= CREATE USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-2].minor.yy129, &yymsp[0].minor.yy0); } + case 0: /* cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ +{ pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } + yy_destructor(yypParser,161,&yymsp[0].minor); break; - case 1: /* cmd ::= ALTER USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy129, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } + case 1: /* cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ +{ pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } + yy_destructor(yypParser,162,&yymsp[0].minor); break; - case 2: /* cmd ::= ALTER USER user_name PRIVILEGE NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy129, TSDB_ALTER_USER_PRIVILEGES, &yymsp[0].minor.yy0); } + case 2: /* account_options ::= */ +{ } break; - case 3: /* cmd ::= DROP USER user_name */ -{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy129); } + case 3: /* account_options ::= account_options PPS literal */ + case 4: /* account_options ::= account_options TSERIES literal */ yytestcase(yyruleno==4); + case 5: /* account_options ::= account_options STORAGE literal */ yytestcase(yyruleno==5); + case 6: /* account_options ::= account_options STREAMS literal */ yytestcase(yyruleno==6); + case 7: /* account_options ::= account_options QTIME literal */ yytestcase(yyruleno==7); + case 8: /* account_options ::= account_options DBS literal */ yytestcase(yyruleno==8); + case 9: /* account_options ::= account_options USERS literal */ yytestcase(yyruleno==9); + case 10: /* account_options ::= account_options CONNS literal */ yytestcase(yyruleno==10); + case 11: /* account_options ::= account_options STATE literal */ yytestcase(yyruleno==11); +{ yy_destructor(yypParser,161,&yymsp[-2].minor); +{ } + yy_destructor(yypParser,163,&yymsp[0].minor); +} break; - case 4: /* cmd ::= SHOW USERS */ + case 12: /* alter_account_options ::= alter_account_option */ +{ yy_destructor(yypParser,164,&yymsp[0].minor); +{ } +} + break; + case 13: /* alter_account_options ::= alter_account_options alter_account_option */ +{ yy_destructor(yypParser,162,&yymsp[-1].minor); +{ } + yy_destructor(yypParser,164,&yymsp[0].minor); +} + break; + case 14: /* alter_account_option ::= PASS literal */ + case 15: /* alter_account_option ::= PPS literal */ yytestcase(yyruleno==15); + case 16: /* alter_account_option ::= TSERIES literal */ yytestcase(yyruleno==16); + case 17: /* alter_account_option ::= STORAGE literal */ yytestcase(yyruleno==17); + case 18: /* alter_account_option ::= STREAMS literal */ yytestcase(yyruleno==18); + case 19: /* alter_account_option ::= QTIME literal */ yytestcase(yyruleno==19); + case 20: /* alter_account_option ::= DBS literal */ yytestcase(yyruleno==20); + case 21: /* alter_account_option ::= USERS literal */ yytestcase(yyruleno==21); + case 22: /* alter_account_option ::= CONNS literal */ yytestcase(yyruleno==22); + case 23: /* alter_account_option ::= STATE literal */ yytestcase(yyruleno==23); +{ } + yy_destructor(yypParser,163,&yymsp[0].minor); + break; + case 24: /* cmd ::= CREATE USER user_name PASS NK_STRING */ +{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-2].minor.yy353, &yymsp[0].minor.yy0); } + break; + case 25: /* cmd ::= ALTER USER user_name PASS NK_STRING */ +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy353, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } + break; + case 26: /* cmd ::= ALTER USER user_name PRIVILEGE NK_STRING */ +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy353, TSDB_ALTER_USER_PRIVILEGES, &yymsp[0].minor.yy0); } + break; + case 27: /* cmd ::= DROP USER user_name */ +{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy353); } + break; + case 28: /* cmd ::= SHOW USERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT, NULL); } break; - case 5: /* cmd ::= CREATE DNODE dnode_endpoint */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy129, NULL); } + case 29: /* cmd ::= CREATE DNODE dnode_endpoint */ +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy353, NULL); } break; - case 6: /* cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy129, &yymsp[0].minor.yy0); } + case 30: /* cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER */ +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy353, &yymsp[0].minor.yy0); } break; - case 7: /* cmd ::= DROP DNODE NK_INTEGER */ + case 31: /* cmd ::= DROP DNODE NK_INTEGER */ { pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 8: /* cmd ::= DROP DNODE dnode_endpoint */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy129); } + case 32: /* cmd ::= DROP DNODE dnode_endpoint */ +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy353); } break; - case 9: /* cmd ::= SHOW DNODES */ + case 33: /* cmd ::= SHOW DNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT, NULL); } break; - case 10: /* dnode_endpoint ::= NK_STRING */ - case 11: /* dnode_host_name ::= NK_ID */ yytestcase(yyruleno==11); - case 12: /* dnode_host_name ::= NK_IPTOKEN */ yytestcase(yyruleno==12); - case 123: /* db_name ::= NK_ID */ yytestcase(yyruleno==123); - case 124: /* table_name ::= NK_ID */ yytestcase(yyruleno==124); - case 125: /* column_name ::= NK_ID */ yytestcase(yyruleno==125); - case 126: /* function_name ::= NK_ID */ yytestcase(yyruleno==126); - case 127: /* table_alias ::= NK_ID */ yytestcase(yyruleno==127); - case 128: /* column_alias ::= NK_ID */ yytestcase(yyruleno==128); - case 129: /* user_name ::= NK_ID */ yytestcase(yyruleno==129); - case 130: /* index_name ::= NK_ID */ yytestcase(yyruleno==130); - case 131: /* topic_name ::= NK_ID */ yytestcase(yyruleno==131); -{ yylhsminor.yy129 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy129 = yylhsminor.yy129; + case 34: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ +{ pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, NULL); } break; - case 13: /* cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ + case 35: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ +{ pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-2].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } + break; + case 36: /* cmd ::= ALTER ALL DNODES NK_STRING */ +{ pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &yymsp[0].minor.yy0, NULL); } + break; + case 37: /* cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ +{ pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } + break; + case 38: /* dnode_endpoint ::= NK_STRING */ + case 39: /* dnode_host_name ::= NK_ID */ yytestcase(yyruleno==39); + case 40: /* dnode_host_name ::= NK_IPTOKEN */ yytestcase(yyruleno==40); + case 184: /* db_name ::= NK_ID */ yytestcase(yyruleno==184); + case 185: /* table_name ::= NK_ID */ yytestcase(yyruleno==185); + case 186: /* column_name ::= NK_ID */ yytestcase(yyruleno==186); + case 187: /* function_name ::= NK_ID */ yytestcase(yyruleno==187); + case 188: /* table_alias ::= NK_ID */ yytestcase(yyruleno==188); + case 189: /* column_alias ::= NK_ID */ yytestcase(yyruleno==189); + case 190: /* user_name ::= NK_ID */ yytestcase(yyruleno==190); + case 191: /* index_name ::= NK_ID */ yytestcase(yyruleno==191); + case 192: /* topic_name ::= NK_ID */ yytestcase(yyruleno==192); +{ yylhsminor.yy353 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy353 = yylhsminor.yy353; + break; + case 41: /* cmd ::= ALTER LOCAL NK_STRING */ +{ pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[0].minor.yy0, NULL); } + break; + case 42: /* cmd ::= ALTER LOCAL NK_STRING NK_STRING */ +{ pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } + break; + case 43: /* cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createCreateQnodeStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 14: /* cmd ::= DROP QNODE ON DNODE NK_INTEGER */ + case 44: /* cmd ::= DROP QNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createDropQnodeStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 15: /* cmd ::= SHOW QNODES */ + case 45: /* cmd ::= SHOW QNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT, NULL); } break; - case 16: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ -{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy97, &yymsp[-1].minor.yy129, yymsp[0].minor.yy432); } + case 46: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ +{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy107, &yymsp[-1].minor.yy353, yymsp[0].minor.yy26); } break; - case 17: /* cmd ::= DROP DATABASE exists_opt db_name */ -{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy97, &yymsp[0].minor.yy129); } + case 47: /* cmd ::= DROP DATABASE exists_opt db_name */ +{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy107, &yymsp[0].minor.yy353); } break; - case 18: /* cmd ::= SHOW DATABASES */ + case 48: /* cmd ::= SHOW DATABASES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT, NULL); } break; - case 19: /* cmd ::= USE db_name */ -{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy129); } + case 49: /* cmd ::= USE db_name */ +{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy353); } break; - case 20: /* cmd ::= ALTER DATABASE db_name db_options */ -{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy129, yymsp[0].minor.yy432); } + case 50: /* cmd ::= ALTER DATABASE db_name alter_db_options */ +{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy353, yymsp[0].minor.yy26); } break; - case 21: /* not_exists_opt ::= IF NOT EXISTS */ -{ yymsp[-2].minor.yy97 = true; } + case 51: /* not_exists_opt ::= IF NOT EXISTS */ +{ yymsp[-2].minor.yy107 = true; } break; - case 22: /* not_exists_opt ::= */ - case 24: /* exists_opt ::= */ yytestcase(yyruleno==24); - case 194: /* set_quantifier_opt ::= */ yytestcase(yyruleno==194); -{ yymsp[1].minor.yy97 = false; } + case 52: /* not_exists_opt ::= */ + case 54: /* exists_opt ::= */ yytestcase(yyruleno==54); + case 255: /* set_quantifier_opt ::= */ yytestcase(yyruleno==255); +{ yymsp[1].minor.yy107 = false; } break; - case 23: /* exists_opt ::= IF EXISTS */ -{ yymsp[-1].minor.yy97 = true; } + case 53: /* exists_opt ::= IF EXISTS */ +{ yymsp[-1].minor.yy107 = true; } break; - case 25: /* db_options ::= */ -{ yymsp[1].minor.yy432 = createDefaultDatabaseOptions(pCxt); } + case 55: /* db_options ::= */ +{ yymsp[1].minor.yy26 = createDefaultDatabaseOptions(pCxt); } break; - case 26: /* db_options ::= db_options BLOCKS NK_INTEGER */ -{ yylhsminor.yy432 = setDatabaseOption(pCxt, yymsp[-2].minor.yy432, DB_OPTION_BLOCKS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 56: /* db_options ::= db_options BLOCKS NK_INTEGER */ +{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_BLOCKS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 27: /* db_options ::= db_options CACHE NK_INTEGER */ -{ yylhsminor.yy432 = setDatabaseOption(pCxt, yymsp[-2].minor.yy432, DB_OPTION_CACHE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 57: /* db_options ::= db_options CACHE NK_INTEGER */ +{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_CACHE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 28: /* db_options ::= db_options CACHELAST NK_INTEGER */ -{ yylhsminor.yy432 = setDatabaseOption(pCxt, yymsp[-2].minor.yy432, DB_OPTION_CACHELAST, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 58: /* db_options ::= db_options CACHELAST NK_INTEGER */ +{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_CACHELAST, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 29: /* db_options ::= db_options COMP NK_INTEGER */ -{ yylhsminor.yy432 = setDatabaseOption(pCxt, yymsp[-2].minor.yy432, DB_OPTION_COMP, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 59: /* db_options ::= db_options COMP NK_INTEGER */ +{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_COMP, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 30: /* db_options ::= db_options DAYS NK_INTEGER */ -{ yylhsminor.yy432 = setDatabaseOption(pCxt, yymsp[-2].minor.yy432, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 60: /* db_options ::= db_options DAYS NK_INTEGER */ +{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 31: /* db_options ::= db_options FSYNC NK_INTEGER */ -{ yylhsminor.yy432 = setDatabaseOption(pCxt, yymsp[-2].minor.yy432, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 61: /* db_options ::= db_options FSYNC NK_INTEGER */ +{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 32: /* db_options ::= db_options MAXROWS NK_INTEGER */ -{ yylhsminor.yy432 = setDatabaseOption(pCxt, yymsp[-2].minor.yy432, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 62: /* db_options ::= db_options MAXROWS NK_INTEGER */ +{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 33: /* db_options ::= db_options MINROWS NK_INTEGER */ -{ yylhsminor.yy432 = setDatabaseOption(pCxt, yymsp[-2].minor.yy432, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 63: /* db_options ::= db_options MINROWS NK_INTEGER */ +{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 34: /* db_options ::= db_options KEEP NK_INTEGER */ -{ yylhsminor.yy432 = setDatabaseOption(pCxt, yymsp[-2].minor.yy432, DB_OPTION_KEEP, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 64: /* db_options ::= db_options KEEP NK_INTEGER */ +{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_KEEP, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 35: /* db_options ::= db_options PRECISION NK_STRING */ -{ yylhsminor.yy432 = setDatabaseOption(pCxt, yymsp[-2].minor.yy432, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 65: /* db_options ::= db_options PRECISION NK_STRING */ +{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 36: /* db_options ::= db_options QUORUM NK_INTEGER */ -{ yylhsminor.yy432 = setDatabaseOption(pCxt, yymsp[-2].minor.yy432, DB_OPTION_QUORUM, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 66: /* db_options ::= db_options QUORUM NK_INTEGER */ +{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_QUORUM, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 37: /* db_options ::= db_options REPLICA NK_INTEGER */ -{ yylhsminor.yy432 = setDatabaseOption(pCxt, yymsp[-2].minor.yy432, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 67: /* db_options ::= db_options REPLICA NK_INTEGER */ +{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 38: /* db_options ::= db_options TTL NK_INTEGER */ -{ yylhsminor.yy432 = setDatabaseOption(pCxt, yymsp[-2].minor.yy432, DB_OPTION_TTL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 68: /* db_options ::= db_options TTL NK_INTEGER */ +{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_TTL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 39: /* db_options ::= db_options WAL NK_INTEGER */ -{ yylhsminor.yy432 = setDatabaseOption(pCxt, yymsp[-2].minor.yy432, DB_OPTION_WAL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 69: /* db_options ::= db_options WAL NK_INTEGER */ +{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_WAL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 40: /* db_options ::= db_options VGROUPS NK_INTEGER */ -{ yylhsminor.yy432 = setDatabaseOption(pCxt, yymsp[-2].minor.yy432, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 70: /* db_options ::= db_options VGROUPS NK_INTEGER */ +{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 41: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ -{ yylhsminor.yy432 = setDatabaseOption(pCxt, yymsp[-2].minor.yy432, DB_OPTION_SINGLESTABLE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 71: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ +{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 42: /* db_options ::= db_options STREAM_MODE NK_INTEGER */ -{ yylhsminor.yy432 = setDatabaseOption(pCxt, yymsp[-2].minor.yy432, DB_OPTION_STREAMMODE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 72: /* db_options ::= db_options STREAM_MODE NK_INTEGER */ +{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_STREAM_MODE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 43: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - case 45: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==45); -{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy97, yymsp[-5].minor.yy432, yymsp[-3].minor.yy24, yymsp[-1].minor.yy24, yymsp[0].minor.yy432);} + case 73: /* db_options ::= db_options RETENTIONS NK_STRING */ +{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_RETENTIONS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 44: /* cmd ::= CREATE TABLE multi_create_clause */ -{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy24);} + case 74: /* db_options ::= db_options FILE_FACTOR NK_FLOAT */ +{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_FILE_FACTOR, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 46: /* cmd ::= DROP TABLE multi_drop_clause */ -{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy24); } + case 75: /* alter_db_options ::= alter_db_option */ +{ yylhsminor.yy26 = createDefaultAlterDatabaseOptions(pCxt); yylhsminor.yy26 = setDatabaseOption(pCxt, yylhsminor.yy26, yymsp[0].minor.yy443.type, &yymsp[0].minor.yy443.val); } + yymsp[0].minor.yy26 = yylhsminor.yy26; break; - case 47: /* cmd ::= DROP STABLE exists_opt full_table_name */ -{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy97, yymsp[0].minor.yy432); } + case 76: /* alter_db_options ::= alter_db_options alter_db_option */ +{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-1].minor.yy26, yymsp[0].minor.yy443.type, &yymsp[0].minor.yy443.val); } + yymsp[-1].minor.yy26 = yylhsminor.yy26; break; - case 48: /* cmd ::= SHOW TABLES */ + case 77: /* alter_db_option ::= BLOCKS NK_INTEGER */ +{ yymsp[-1].minor.yy443.type = DB_OPTION_BLOCKS; yymsp[-1].minor.yy443.val = yymsp[0].minor.yy0; } + break; + case 78: /* alter_db_option ::= FSYNC NK_INTEGER */ +{ yymsp[-1].minor.yy443.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy443.val = yymsp[0].minor.yy0; } + break; + case 79: /* alter_db_option ::= KEEP NK_INTEGER */ +{ yymsp[-1].minor.yy443.type = DB_OPTION_KEEP; yymsp[-1].minor.yy443.val = yymsp[0].minor.yy0; } + break; + case 80: /* alter_db_option ::= WAL NK_INTEGER */ +{ yymsp[-1].minor.yy443.type = DB_OPTION_WAL; yymsp[-1].minor.yy443.val = yymsp[0].minor.yy0; } + break; + case 81: /* alter_db_option ::= QUORUM NK_INTEGER */ +{ yymsp[-1].minor.yy443.type = DB_OPTION_QUORUM; yymsp[-1].minor.yy443.val = yymsp[0].minor.yy0; } + break; + case 82: /* alter_db_option ::= CACHELAST NK_INTEGER */ +{ yymsp[-1].minor.yy443.type = DB_OPTION_CACHELAST; yymsp[-1].minor.yy443.val = yymsp[0].minor.yy0; } + break; + case 83: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + case 85: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==85); +{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy107, yymsp[-5].minor.yy26, yymsp[-3].minor.yy64, yymsp[-1].minor.yy64, yymsp[0].minor.yy26); } + break; + case 84: /* cmd ::= CREATE TABLE multi_create_clause */ +{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy64); } + break; + case 86: /* cmd ::= DROP TABLE multi_drop_clause */ +{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy64); } + break; + case 87: /* cmd ::= DROP STABLE exists_opt full_table_name */ +{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy107, yymsp[0].minor.yy26); } + break; + case 88: /* cmd ::= SHOW TABLES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, NULL); } break; - case 49: /* cmd ::= SHOW STABLES */ + case 89: /* cmd ::= SHOW STABLES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, NULL); } break; - case 50: /* multi_create_clause ::= create_subtable_clause */ - case 53: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==53); - case 60: /* column_def_list ::= column_def */ yytestcase(yyruleno==60); - case 95: /* col_name_list ::= col_name */ yytestcase(yyruleno==95); - case 104: /* func_list ::= func */ yytestcase(yyruleno==104); - case 199: /* select_sublist ::= select_item */ yytestcase(yyruleno==199); - case 246: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==246); -{ yylhsminor.yy24 = createNodeList(pCxt, yymsp[0].minor.yy432); } - yymsp[0].minor.yy24 = yylhsminor.yy24; + case 90: /* cmd ::= ALTER TABLE alter_table_clause */ + case 91: /* cmd ::= ALTER STABLE alter_table_clause */ yytestcase(yyruleno==91); + case 174: /* cmd ::= query_expression */ yytestcase(yyruleno==174); +{ pCxt->pRootNode = yymsp[0].minor.yy26; } break; - case 51: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ - case 54: /* multi_drop_clause ::= multi_drop_clause drop_table_clause */ yytestcase(yyruleno==54); -{ yylhsminor.yy24 = addNodeToList(pCxt, yymsp[-1].minor.yy24, yymsp[0].minor.yy432); } - yymsp[-1].minor.yy24 = yylhsminor.yy24; + case 92: /* alter_table_clause ::= full_table_name alter_table_options */ +{ yylhsminor.yy26 = createAlterTableOption(pCxt, yymsp[-1].minor.yy26, yymsp[0].minor.yy26); } + yymsp[-1].minor.yy26 = yylhsminor.yy26; break; - case 52: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP */ -{ yylhsminor.yy432 = createCreateSubTableClause(pCxt, yymsp[-8].minor.yy97, yymsp[-7].minor.yy432, yymsp[-5].minor.yy432, yymsp[-4].minor.yy24, yymsp[-1].minor.yy24); } - yymsp[-8].minor.yy432 = yylhsminor.yy432; + case 93: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ +{ yylhsminor.yy26 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy26, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy353, yymsp[0].minor.yy370); } + yymsp[-4].minor.yy26 = yylhsminor.yy26; break; - case 55: /* drop_table_clause ::= exists_opt full_table_name */ -{ yylhsminor.yy432 = createDropTableClause(pCxt, yymsp[-1].minor.yy97, yymsp[0].minor.yy432); } - yymsp[-1].minor.yy432 = yylhsminor.yy432; + case 94: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ +{ yylhsminor.yy26 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy26, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy353); } + yymsp[-3].minor.yy26 = yylhsminor.yy26; break; - case 56: /* specific_tags_opt ::= */ - case 87: /* tags_def_opt ::= */ yytestcase(yyruleno==87); - case 207: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==207); - case 224: /* group_by_clause_opt ::= */ yytestcase(yyruleno==224); - case 234: /* order_by_clause_opt ::= */ yytestcase(yyruleno==234); -{ yymsp[1].minor.yy24 = NULL; } + case 95: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ +{ yylhsminor.yy26 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy26, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy353, yymsp[0].minor.yy370); } + yymsp[-4].minor.yy26 = yylhsminor.yy26; break; - case 57: /* specific_tags_opt ::= NK_LP col_name_list NK_RP */ -{ yymsp[-2].minor.yy24 = yymsp[-1].minor.yy24; } + case 96: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ +{ yylhsminor.yy26 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy26, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy353, &yymsp[0].minor.yy353); } + yymsp[-4].minor.yy26 = yylhsminor.yy26; break; - case 58: /* full_table_name ::= table_name */ -{ yylhsminor.yy432 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy129, NULL); } - yymsp[0].minor.yy432 = yylhsminor.yy432; + case 97: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ +{ yylhsminor.yy26 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy26, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy353, yymsp[0].minor.yy370); } + yymsp[-4].minor.yy26 = yylhsminor.yy26; break; - case 59: /* full_table_name ::= db_name NK_DOT table_name */ -{ yylhsminor.yy432 = createRealTableNode(pCxt, &yymsp[-2].minor.yy129, &yymsp[0].minor.yy129, NULL); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 98: /* alter_table_clause ::= full_table_name DROP TAG column_name */ +{ yylhsminor.yy26 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy26, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy353); } + yymsp[-3].minor.yy26 = yylhsminor.yy26; break; - case 61: /* column_def_list ::= column_def_list NK_COMMA column_def */ - case 96: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==96); - case 105: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==105); - case 200: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==200); - case 247: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==247); -{ yylhsminor.yy24 = addNodeToList(pCxt, yymsp[-2].minor.yy24, yymsp[0].minor.yy432); } - yymsp[-2].minor.yy24 = yylhsminor.yy24; + case 99: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ +{ yylhsminor.yy26 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy26, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy353, yymsp[0].minor.yy370); } + yymsp[-4].minor.yy26 = yylhsminor.yy26; break; - case 62: /* column_def ::= column_name type_name */ -{ yylhsminor.yy432 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy129, yymsp[0].minor.yy224, NULL); } - yymsp[-1].minor.yy432 = yylhsminor.yy432; + case 100: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ +{ yylhsminor.yy26 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy26, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy353, &yymsp[0].minor.yy353); } + yymsp[-4].minor.yy26 = yylhsminor.yy26; break; - case 63: /* column_def ::= column_name type_name COMMENT NK_STRING */ -{ yylhsminor.yy432 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy129, yymsp[-2].minor.yy224, &yymsp[0].minor.yy0); } - yymsp[-3].minor.yy432 = yylhsminor.yy432; + case 101: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal */ +{ yylhsminor.yy26 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy26, &yymsp[-2].minor.yy353, yymsp[0].minor.yy26); } + yymsp[-5].minor.yy26 = yylhsminor.yy26; break; - case 64: /* type_name ::= BOOL */ -{ yymsp[0].minor.yy224 = createDataType(TSDB_DATA_TYPE_BOOL); } + case 102: /* multi_create_clause ::= create_subtable_clause */ + case 105: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==105); + case 112: /* column_def_list ::= column_def */ yytestcase(yyruleno==112); + case 153: /* col_name_list ::= col_name */ yytestcase(yyruleno==153); + case 156: /* func_name_list ::= func_name */ yytestcase(yyruleno==156); + case 165: /* func_list ::= func */ yytestcase(yyruleno==165); + case 260: /* select_sublist ::= select_item */ yytestcase(yyruleno==260); + case 307: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==307); +{ yylhsminor.yy64 = createNodeList(pCxt, yymsp[0].minor.yy26); } + yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 65: /* type_name ::= TINYINT */ -{ yymsp[0].minor.yy224 = createDataType(TSDB_DATA_TYPE_TINYINT); } + case 103: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ + case 106: /* multi_drop_clause ::= multi_drop_clause drop_table_clause */ yytestcase(yyruleno==106); +{ yylhsminor.yy64 = addNodeToList(pCxt, yymsp[-1].minor.yy64, yymsp[0].minor.yy26); } + yymsp[-1].minor.yy64 = yylhsminor.yy64; break; - case 66: /* type_name ::= SMALLINT */ -{ yymsp[0].minor.yy224 = createDataType(TSDB_DATA_TYPE_SMALLINT); } + case 104: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP */ +{ yylhsminor.yy26 = createCreateSubTableClause(pCxt, yymsp[-8].minor.yy107, yymsp[-7].minor.yy26, yymsp[-5].minor.yy26, yymsp[-4].minor.yy64, yymsp[-1].minor.yy64); } + yymsp[-8].minor.yy26 = yylhsminor.yy26; break; - case 67: /* type_name ::= INT */ - case 68: /* type_name ::= INTEGER */ yytestcase(yyruleno==68); -{ yymsp[0].minor.yy224 = createDataType(TSDB_DATA_TYPE_INT); } + case 107: /* drop_table_clause ::= exists_opt full_table_name */ +{ yylhsminor.yy26 = createDropTableClause(pCxt, yymsp[-1].minor.yy107, yymsp[0].minor.yy26); } + yymsp[-1].minor.yy26 = yylhsminor.yy26; break; - case 69: /* type_name ::= BIGINT */ -{ yymsp[0].minor.yy224 = createDataType(TSDB_DATA_TYPE_BIGINT); } + case 108: /* specific_tags_opt ::= */ + case 139: /* tags_def_opt ::= */ yytestcase(yyruleno==139); + case 268: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==268); + case 285: /* group_by_clause_opt ::= */ yytestcase(yyruleno==285); + case 295: /* order_by_clause_opt ::= */ yytestcase(yyruleno==295); +{ yymsp[1].minor.yy64 = NULL; } break; - case 70: /* type_name ::= FLOAT */ -{ yymsp[0].minor.yy224 = createDataType(TSDB_DATA_TYPE_FLOAT); } + case 109: /* specific_tags_opt ::= NK_LP col_name_list NK_RP */ +{ yymsp[-2].minor.yy64 = yymsp[-1].minor.yy64; } break; - case 71: /* type_name ::= DOUBLE */ -{ yymsp[0].minor.yy224 = createDataType(TSDB_DATA_TYPE_DOUBLE); } + case 110: /* full_table_name ::= table_name */ +{ yylhsminor.yy26 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy353, NULL); } + yymsp[0].minor.yy26 = yylhsminor.yy26; break; - case 72: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy224 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } + case 111: /* full_table_name ::= db_name NK_DOT table_name */ +{ yylhsminor.yy26 = createRealTableNode(pCxt, &yymsp[-2].minor.yy353, &yymsp[0].minor.yy353, NULL); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 73: /* type_name ::= TIMESTAMP */ -{ yymsp[0].minor.yy224 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } + case 113: /* column_def_list ::= column_def_list NK_COMMA column_def */ + case 154: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==154); + case 157: /* func_name_list ::= func_name_list NK_COMMA col_name */ yytestcase(yyruleno==157); + case 166: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==166); + case 261: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==261); + case 308: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==308); +{ yylhsminor.yy64 = addNodeToList(pCxt, yymsp[-2].minor.yy64, yymsp[0].minor.yy26); } + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 74: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy224 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } + case 114: /* column_def ::= column_name type_name */ +{ yylhsminor.yy26 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy353, yymsp[0].minor.yy370, NULL); } + yymsp[-1].minor.yy26 = yylhsminor.yy26; break; - case 75: /* type_name ::= TINYINT UNSIGNED */ -{ yymsp[-1].minor.yy224 = createDataType(TSDB_DATA_TYPE_UTINYINT); } + case 115: /* column_def ::= column_name type_name COMMENT NK_STRING */ +{ yylhsminor.yy26 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy353, yymsp[-2].minor.yy370, &yymsp[0].minor.yy0); } + yymsp[-3].minor.yy26 = yylhsminor.yy26; break; - case 76: /* type_name ::= SMALLINT UNSIGNED */ -{ yymsp[-1].minor.yy224 = createDataType(TSDB_DATA_TYPE_USMALLINT); } + case 116: /* type_name ::= BOOL */ +{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_BOOL); } break; - case 77: /* type_name ::= INT UNSIGNED */ -{ yymsp[-1].minor.yy224 = createDataType(TSDB_DATA_TYPE_UINT); } + case 117: /* type_name ::= TINYINT */ +{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_TINYINT); } break; - case 78: /* type_name ::= BIGINT UNSIGNED */ -{ yymsp[-1].minor.yy224 = createDataType(TSDB_DATA_TYPE_UBIGINT); } + case 118: /* type_name ::= SMALLINT */ +{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_SMALLINT); } break; - case 79: /* type_name ::= JSON */ -{ yymsp[0].minor.yy224 = createDataType(TSDB_DATA_TYPE_JSON); } + case 119: /* type_name ::= INT */ + case 120: /* type_name ::= INTEGER */ yytestcase(yyruleno==120); +{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_INT); } break; - case 80: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy224 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } + case 121: /* type_name ::= BIGINT */ +{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_BIGINT); } break; - case 81: /* type_name ::= MEDIUMBLOB */ -{ yymsp[0].minor.yy224 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } + case 122: /* type_name ::= FLOAT */ +{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_FLOAT); } break; - case 82: /* type_name ::= BLOB */ -{ yymsp[0].minor.yy224 = createDataType(TSDB_DATA_TYPE_BLOB); } + case 123: /* type_name ::= DOUBLE */ +{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_DOUBLE); } break; - case 83: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy224 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } + case 124: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy370 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } break; - case 84: /* type_name ::= DECIMAL */ -{ yymsp[0].minor.yy224 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 125: /* type_name ::= TIMESTAMP */ +{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } break; - case 85: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy224 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 126: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy370 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } break; - case 86: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ -{ yymsp[-5].minor.yy224 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 127: /* type_name ::= TINYINT UNSIGNED */ +{ yymsp[-1].minor.yy370 = createDataType(TSDB_DATA_TYPE_UTINYINT); } break; - case 88: /* tags_def_opt ::= tags_def */ - case 198: /* select_list ::= select_sublist */ yytestcase(yyruleno==198); -{ yylhsminor.yy24 = yymsp[0].minor.yy24; } - yymsp[0].minor.yy24 = yylhsminor.yy24; + case 128: /* type_name ::= SMALLINT UNSIGNED */ +{ yymsp[-1].minor.yy370 = createDataType(TSDB_DATA_TYPE_USMALLINT); } break; - case 89: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ -{ yymsp[-3].minor.yy24 = yymsp[-1].minor.yy24; } + case 129: /* type_name ::= INT UNSIGNED */ +{ yymsp[-1].minor.yy370 = createDataType(TSDB_DATA_TYPE_UINT); } break; - case 90: /* table_options ::= */ -{ yymsp[1].minor.yy432 = createDefaultTableOptions(pCxt);} + case 130: /* type_name ::= BIGINT UNSIGNED */ +{ yymsp[-1].minor.yy370 = createDataType(TSDB_DATA_TYPE_UBIGINT); } break; - case 91: /* table_options ::= table_options COMMENT NK_STRING */ -{ yylhsminor.yy432 = setTableOption(pCxt, yymsp[-2].minor.yy432, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 131: /* type_name ::= JSON */ +{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_JSON); } break; - case 92: /* table_options ::= table_options KEEP NK_INTEGER */ -{ yylhsminor.yy432 = setTableOption(pCxt, yymsp[-2].minor.yy432, TABLE_OPTION_KEEP, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 132: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy370 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } break; - case 93: /* table_options ::= table_options TTL NK_INTEGER */ -{ yylhsminor.yy432 = setTableOption(pCxt, yymsp[-2].minor.yy432, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 133: /* type_name ::= MEDIUMBLOB */ +{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } break; - case 94: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ -{ yylhsminor.yy432 = setTableSmaOption(pCxt, yymsp[-4].minor.yy432, yymsp[-1].minor.yy24); } - yymsp[-4].minor.yy432 = yylhsminor.yy432; + case 134: /* type_name ::= BLOB */ +{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_BLOB); } break; - case 97: /* col_name ::= column_name */ -{ yylhsminor.yy432 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy129); } - yymsp[0].minor.yy432 = yylhsminor.yy432; + case 135: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy370 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } break; - case 98: /* cmd ::= CREATE SMA INDEX index_name ON table_name index_options */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, &yymsp[-3].minor.yy129, &yymsp[-1].minor.yy129, NULL, yymsp[0].minor.yy432); } + case 136: /* type_name ::= DECIMAL */ +{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 99: /* cmd ::= CREATE FULLTEXT INDEX index_name ON table_name NK_LP col_name_list NK_RP */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, &yymsp[-5].minor.yy129, &yymsp[-3].minor.yy129, yymsp[-1].minor.yy24, NULL); } + case 137: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy370 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 100: /* cmd ::= DROP INDEX index_name ON table_name */ -{ pCxt->pRootNode = createDropIndexStmt(pCxt, &yymsp[-2].minor.yy129, &yymsp[0].minor.yy129); } + case 138: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ +{ yymsp[-5].minor.yy370 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 101: /* index_options ::= */ - case 205: /* where_clause_opt ::= */ yytestcase(yyruleno==205); - case 209: /* twindow_clause_opt ::= */ yytestcase(yyruleno==209); - case 214: /* sliding_opt ::= */ yytestcase(yyruleno==214); - case 216: /* fill_opt ::= */ yytestcase(yyruleno==216); - case 228: /* having_clause_opt ::= */ yytestcase(yyruleno==228); - case 236: /* slimit_clause_opt ::= */ yytestcase(yyruleno==236); - case 240: /* limit_clause_opt ::= */ yytestcase(yyruleno==240); -{ yymsp[1].minor.yy432 = NULL; } + case 140: /* tags_def_opt ::= tags_def */ + case 259: /* select_list ::= select_sublist */ yytestcase(yyruleno==259); +{ yylhsminor.yy64 = yymsp[0].minor.yy64; } + yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 102: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ -{ yymsp[-8].minor.yy432 = createIndexOption(pCxt, yymsp[-6].minor.yy24, releaseRawExprNode(pCxt, yymsp[-2].minor.yy432), NULL, yymsp[0].minor.yy432); } + case 141: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ +{ yymsp[-3].minor.yy64 = yymsp[-1].minor.yy64; } break; - case 103: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ -{ yymsp[-10].minor.yy432 = createIndexOption(pCxt, yymsp[-8].minor.yy24, releaseRawExprNode(pCxt, yymsp[-4].minor.yy432), releaseRawExprNode(pCxt, yymsp[-2].minor.yy432), yymsp[0].minor.yy432); } + case 142: /* table_options ::= */ +{ yymsp[1].minor.yy26 = createDefaultTableOptions(pCxt); } break; - case 106: /* func ::= function_name NK_LP expression_list NK_RP */ -{ yylhsminor.yy432 = createFunctionNode(pCxt, &yymsp[-3].minor.yy129, yymsp[-1].minor.yy24); } - yymsp[-3].minor.yy432 = yylhsminor.yy432; + case 143: /* table_options ::= table_options COMMENT NK_STRING */ +{ yylhsminor.yy26 = setTableOption(pCxt, yymsp[-2].minor.yy26, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 107: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ -{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-3].minor.yy97, &yymsp[-2].minor.yy129, yymsp[0].minor.yy432, NULL); } + case 144: /* table_options ::= table_options KEEP NK_INTEGER */ +{ yylhsminor.yy26 = setTableOption(pCxt, yymsp[-2].minor.yy26, TABLE_OPTION_KEEP, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 108: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name */ -{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-3].minor.yy97, &yymsp[-2].minor.yy129, NULL, &yymsp[0].minor.yy129); } + case 145: /* table_options ::= table_options TTL NK_INTEGER */ +{ yylhsminor.yy26 = setTableOption(pCxt, yymsp[-2].minor.yy26, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 109: /* cmd ::= DROP TOPIC exists_opt topic_name */ -{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy97, &yymsp[0].minor.yy129); } + case 146: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ +{ yylhsminor.yy26 = setTableSmaOption(pCxt, yymsp[-4].minor.yy26, yymsp[-1].minor.yy64); } + yymsp[-4].minor.yy26 = yylhsminor.yy26; break; - case 110: /* cmd ::= SHOW VGROUPS */ + case 147: /* table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP */ +{ yylhsminor.yy26 = setTableRollupOption(pCxt, yymsp[-4].minor.yy26, yymsp[-1].minor.yy64); } + yymsp[-4].minor.yy26 = yylhsminor.yy26; + break; + case 148: /* alter_table_options ::= alter_table_option */ +{ yylhsminor.yy26 = createDefaultAlterTableOptions(pCxt); yylhsminor.yy26 = setTableOption(pCxt, yylhsminor.yy26, yymsp[0].minor.yy443.type, &yymsp[0].minor.yy443.val); } + yymsp[0].minor.yy26 = yylhsminor.yy26; + break; + case 149: /* alter_table_options ::= alter_table_options alter_table_option */ +{ yylhsminor.yy26 = setTableOption(pCxt, yymsp[-1].minor.yy26, yymsp[0].minor.yy443.type, &yymsp[0].minor.yy443.val); } + yymsp[-1].minor.yy26 = yylhsminor.yy26; + break; + case 150: /* alter_table_option ::= COMMENT NK_STRING */ +{ yymsp[-1].minor.yy443.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy443.val = yymsp[0].minor.yy0; } + break; + case 151: /* alter_table_option ::= KEEP NK_INTEGER */ +{ yymsp[-1].minor.yy443.type = TABLE_OPTION_KEEP; yymsp[-1].minor.yy443.val = yymsp[0].minor.yy0; } + break; + case 152: /* alter_table_option ::= TTL NK_INTEGER */ +{ yymsp[-1].minor.yy443.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy443.val = yymsp[0].minor.yy0; } + break; + case 155: /* col_name ::= column_name */ +{ yylhsminor.yy26 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy353); } + yymsp[0].minor.yy26 = yylhsminor.yy26; + break; + case 158: /* func_name ::= function_name */ +{ yylhsminor.yy26 = createFunctionNode(pCxt, &yymsp[0].minor.yy353, NULL); } + yymsp[0].minor.yy26 = yylhsminor.yy26; + break; + case 159: /* cmd ::= CREATE SMA INDEX index_name ON table_name index_options */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, &yymsp[-3].minor.yy353, &yymsp[-1].minor.yy353, NULL, yymsp[0].minor.yy26); } + break; + case 160: /* cmd ::= CREATE FULLTEXT INDEX index_name ON table_name NK_LP col_name_list NK_RP */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, &yymsp[-5].minor.yy353, &yymsp[-3].minor.yy353, yymsp[-1].minor.yy64, NULL); } + break; + case 161: /* cmd ::= DROP INDEX index_name ON table_name */ +{ pCxt->pRootNode = createDropIndexStmt(pCxt, &yymsp[-2].minor.yy353, &yymsp[0].minor.yy353); } + break; + case 162: /* index_options ::= */ + case 266: /* where_clause_opt ::= */ yytestcase(yyruleno==266); + case 270: /* twindow_clause_opt ::= */ yytestcase(yyruleno==270); + case 275: /* sliding_opt ::= */ yytestcase(yyruleno==275); + case 277: /* fill_opt ::= */ yytestcase(yyruleno==277); + case 289: /* having_clause_opt ::= */ yytestcase(yyruleno==289); + case 297: /* slimit_clause_opt ::= */ yytestcase(yyruleno==297); + case 301: /* limit_clause_opt ::= */ yytestcase(yyruleno==301); +{ yymsp[1].minor.yy26 = NULL; } + break; + case 163: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ +{ yymsp[-8].minor.yy26 = createIndexOption(pCxt, yymsp[-6].minor.yy64, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), NULL, yymsp[0].minor.yy26); } + break; + case 164: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ +{ yymsp[-10].minor.yy26 = createIndexOption(pCxt, yymsp[-8].minor.yy64, releaseRawExprNode(pCxt, yymsp[-4].minor.yy26), releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), yymsp[0].minor.yy26); } + break; + case 167: /* func ::= function_name NK_LP expression_list NK_RP */ +{ yylhsminor.yy26 = createFunctionNode(pCxt, &yymsp[-3].minor.yy353, yymsp[-1].minor.yy64); } + yymsp[-3].minor.yy26 = yylhsminor.yy26; + break; + case 168: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ +{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-3].minor.yy107, &yymsp[-2].minor.yy353, yymsp[0].minor.yy26, NULL); } + break; + case 169: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name */ +{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-3].minor.yy107, &yymsp[-2].minor.yy353, NULL, &yymsp[0].minor.yy353); } + break; + case 170: /* cmd ::= DROP TOPIC exists_opt topic_name */ +{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy107, &yymsp[0].minor.yy353); } + break; + case 171: /* cmd ::= SHOW VGROUPS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, NULL); } break; - case 111: /* cmd ::= SHOW db_name NK_DOT VGROUPS */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, &yymsp[-2].minor.yy129); } + case 172: /* cmd ::= SHOW db_name NK_DOT VGROUPS */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, &yymsp[-2].minor.yy353); } break; - case 112: /* cmd ::= SHOW MNODES */ + case 173: /* cmd ::= SHOW MNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT, NULL); } break; - case 113: /* cmd ::= query_expression */ -{ pCxt->pRootNode = yymsp[0].minor.yy432; } + case 175: /* literal ::= NK_INTEGER */ +{ yylhsminor.yy26 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy26 = yylhsminor.yy26; break; - case 114: /* literal ::= NK_INTEGER */ -{ yylhsminor.yy432 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy432 = yylhsminor.yy432; + case 176: /* literal ::= NK_FLOAT */ +{ yylhsminor.yy26 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy26 = yylhsminor.yy26; break; - case 115: /* literal ::= NK_FLOAT */ -{ yylhsminor.yy432 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy432 = yylhsminor.yy432; + case 177: /* literal ::= NK_STRING */ +{ yylhsminor.yy26 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy26 = yylhsminor.yy26; break; - case 116: /* literal ::= NK_STRING */ -{ yylhsminor.yy432 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy432 = yylhsminor.yy432; + case 178: /* literal ::= NK_BOOL */ +{ yylhsminor.yy26 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy26 = yylhsminor.yy26; break; - case 117: /* literal ::= NK_BOOL */ -{ yylhsminor.yy432 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy432 = yylhsminor.yy432; + case 179: /* literal ::= TIMESTAMP NK_STRING */ +{ yylhsminor.yy26 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy26 = yylhsminor.yy26; break; - case 118: /* literal ::= TIMESTAMP NK_STRING */ -{ yylhsminor.yy432 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } - yymsp[-1].minor.yy432 = yylhsminor.yy432; + case 180: /* literal ::= duration_literal */ + case 193: /* expression ::= literal */ yytestcase(yyruleno==193); + case 194: /* expression ::= column_reference */ yytestcase(yyruleno==194); + case 197: /* expression ::= subquery */ yytestcase(yyruleno==197); + case 229: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==229); + case 233: /* boolean_primary ::= predicate */ yytestcase(yyruleno==233); + case 235: /* common_expression ::= expression */ yytestcase(yyruleno==235); + case 236: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==236); + case 238: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==238); + case 240: /* table_reference ::= table_primary */ yytestcase(yyruleno==240); + case 241: /* table_reference ::= joined_table */ yytestcase(yyruleno==241); + case 245: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==245); + case 292: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==292); + case 294: /* query_primary ::= query_specification */ yytestcase(yyruleno==294); +{ yylhsminor.yy26 = yymsp[0].minor.yy26; } + yymsp[0].minor.yy26 = yylhsminor.yy26; break; - case 119: /* literal ::= duration_literal */ - case 132: /* expression ::= literal */ yytestcase(yyruleno==132); - case 133: /* expression ::= column_reference */ yytestcase(yyruleno==133); - case 136: /* expression ::= subquery */ yytestcase(yyruleno==136); - case 168: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==168); - case 172: /* boolean_primary ::= predicate */ yytestcase(yyruleno==172); - case 174: /* common_expression ::= expression */ yytestcase(yyruleno==174); - case 175: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==175); - case 177: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==177); - case 179: /* table_reference ::= table_primary */ yytestcase(yyruleno==179); - case 180: /* table_reference ::= joined_table */ yytestcase(yyruleno==180); - case 184: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==184); - case 231: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==231); - case 233: /* query_primary ::= query_specification */ yytestcase(yyruleno==233); -{ yylhsminor.yy432 = yymsp[0].minor.yy432; } - yymsp[0].minor.yy432 = yylhsminor.yy432; + case 181: /* duration_literal ::= NK_VARIABLE */ +{ yylhsminor.yy26 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy26 = yylhsminor.yy26; break; - case 120: /* duration_literal ::= NK_VARIABLE */ -{ yylhsminor.yy432 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy432 = yylhsminor.yy432; + case 182: /* literal_list ::= literal */ + case 206: /* expression_list ::= expression */ yytestcase(yyruleno==206); +{ yylhsminor.yy64 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy26)); } + yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 121: /* literal_list ::= literal */ - case 145: /* expression_list ::= expression */ yytestcase(yyruleno==145); -{ yylhsminor.yy24 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy432)); } - yymsp[0].minor.yy24 = yylhsminor.yy24; + case 183: /* literal_list ::= literal_list NK_COMMA literal */ + case 207: /* expression_list ::= expression_list NK_COMMA expression */ yytestcase(yyruleno==207); +{ yylhsminor.yy64 = addNodeToList(pCxt, yymsp[-2].minor.yy64, releaseRawExprNode(pCxt, yymsp[0].minor.yy26)); } + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 122: /* literal_list ::= literal_list NK_COMMA literal */ - case 146: /* expression_list ::= expression_list NK_COMMA expression */ yytestcase(yyruleno==146); -{ yylhsminor.yy24 = addNodeToList(pCxt, yymsp[-2].minor.yy24, releaseRawExprNode(pCxt, yymsp[0].minor.yy432)); } - yymsp[-2].minor.yy24 = yylhsminor.yy24; + case 195: /* expression ::= function_name NK_LP expression_list NK_RP */ +{ yylhsminor.yy26 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy353, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy353, yymsp[-1].minor.yy64)); } + yymsp[-3].minor.yy26 = yylhsminor.yy26; break; - case 134: /* expression ::= function_name NK_LP expression_list NK_RP */ -{ yylhsminor.yy432 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy129, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy129, yymsp[-1].minor.yy24)); } - yymsp[-3].minor.yy432 = yylhsminor.yy432; + case 196: /* expression ::= function_name NK_LP NK_STAR NK_RP */ +{ yylhsminor.yy26 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy353, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy353, createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy0)))); } + yymsp[-3].minor.yy26 = yylhsminor.yy26; break; - case 135: /* expression ::= function_name NK_LP NK_STAR NK_RP */ -{ yylhsminor.yy432 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy129, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy129, createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy0)))); } - yymsp[-3].minor.yy432 = yylhsminor.yy432; + case 198: /* expression ::= NK_LP expression NK_RP */ + case 234: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==234); +{ yylhsminor.yy26 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy26)); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 137: /* expression ::= NK_LP expression NK_RP */ - case 173: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==173); -{ yylhsminor.yy432 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy432)); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; - break; - case 138: /* expression ::= NK_PLUS expression */ + case 199: /* expression ::= NK_PLUS expression */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy432); - yylhsminor.yy432 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy432)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); + yylhsminor.yy26 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy26)); } - yymsp[-1].minor.yy432 = yylhsminor.yy432; + yymsp[-1].minor.yy26 = yylhsminor.yy26; break; - case 139: /* expression ::= NK_MINUS expression */ + case 200: /* expression ::= NK_MINUS expression */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy432); - yylhsminor.yy432 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[0].minor.yy432), NULL)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); + yylhsminor.yy26 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[0].minor.yy26), NULL)); } - yymsp[-1].minor.yy432 = yylhsminor.yy432; + yymsp[-1].minor.yy26 = yylhsminor.yy26; break; - case 140: /* expression ::= expression NK_PLUS expression */ + case 201: /* expression ::= expression NK_PLUS expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy432); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy432); - yylhsminor.yy432 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy432), releaseRawExprNode(pCxt, yymsp[0].minor.yy432))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy26); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); + yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 141: /* expression ::= expression NK_MINUS expression */ + case 202: /* expression ::= expression NK_MINUS expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy432); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy432); - yylhsminor.yy432 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy432), releaseRawExprNode(pCxt, yymsp[0].minor.yy432))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy26); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); + yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 142: /* expression ::= expression NK_STAR expression */ + case 203: /* expression ::= expression NK_STAR expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy432); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy432); - yylhsminor.yy432 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy432), releaseRawExprNode(pCxt, yymsp[0].minor.yy432))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy26); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); + yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 143: /* expression ::= expression NK_SLASH expression */ + case 204: /* expression ::= expression NK_SLASH expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy432); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy432); - yylhsminor.yy432 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy432), releaseRawExprNode(pCxt, yymsp[0].minor.yy432))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy26); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); + yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 144: /* expression ::= expression NK_REM expression */ + case 205: /* expression ::= expression NK_REM expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy432); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy432); - yylhsminor.yy432 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy432), releaseRawExprNode(pCxt, yymsp[0].minor.yy432))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy26); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); + yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 147: /* column_reference ::= column_name */ -{ yylhsminor.yy432 = createRawExprNode(pCxt, &yymsp[0].minor.yy129, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy129)); } - yymsp[0].minor.yy432 = yylhsminor.yy432; + case 208: /* column_reference ::= column_name */ +{ yylhsminor.yy26 = createRawExprNode(pCxt, &yymsp[0].minor.yy353, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy353)); } + yymsp[0].minor.yy26 = yylhsminor.yy26; break; - case 148: /* column_reference ::= table_name NK_DOT column_name */ -{ yylhsminor.yy432 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy129, &yymsp[0].minor.yy129, createColumnNode(pCxt, &yymsp[-2].minor.yy129, &yymsp[0].minor.yy129)); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 209: /* column_reference ::= table_name NK_DOT column_name */ +{ yylhsminor.yy26 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy353, &yymsp[0].minor.yy353, createColumnNode(pCxt, &yymsp[-2].minor.yy353, &yymsp[0].minor.yy353)); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 149: /* predicate ::= expression compare_op expression */ - case 154: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==154); + case 210: /* predicate ::= expression compare_op expression */ + case 215: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==215); { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy432); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy432); - yylhsminor.yy432 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy260, releaseRawExprNode(pCxt, yymsp[-2].minor.yy432), releaseRawExprNode(pCxt, yymsp[0].minor.yy432))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy26); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); + yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy80, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 150: /* predicate ::= expression BETWEEN expression AND expression */ + case 211: /* predicate ::= expression BETWEEN expression AND expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy432); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy432); - yylhsminor.yy432 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy432), releaseRawExprNode(pCxt, yymsp[-2].minor.yy432), releaseRawExprNode(pCxt, yymsp[0].minor.yy432))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy26); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); + yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy26), releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); } - yymsp[-4].minor.yy432 = yylhsminor.yy432; + yymsp[-4].minor.yy26 = yylhsminor.yy26; break; - case 151: /* predicate ::= expression NOT BETWEEN expression AND expression */ + case 212: /* predicate ::= expression NOT BETWEEN expression AND expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy432); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy432); - yylhsminor.yy432 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy432), releaseRawExprNode(pCxt, yymsp[-5].minor.yy432), releaseRawExprNode(pCxt, yymsp[0].minor.yy432))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy26); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); + yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), releaseRawExprNode(pCxt, yymsp[-5].minor.yy26), releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); } - yymsp[-5].minor.yy432 = yylhsminor.yy432; + yymsp[-5].minor.yy26 = yylhsminor.yy26; break; - case 152: /* predicate ::= expression IS NULL */ + case 213: /* predicate ::= expression IS NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy432); - yylhsminor.yy432 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy432), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy26); + yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), NULL)); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 153: /* predicate ::= expression IS NOT NULL */ + case 214: /* predicate ::= expression IS NOT NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy432); - yylhsminor.yy432 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy432), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy26); + yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy26), NULL)); } - yymsp[-3].minor.yy432 = yylhsminor.yy432; + yymsp[-3].minor.yy26 = yylhsminor.yy26; break; - case 155: /* compare_op ::= NK_LT */ -{ yymsp[0].minor.yy260 = OP_TYPE_LOWER_THAN; } + case 216: /* compare_op ::= NK_LT */ +{ yymsp[0].minor.yy80 = OP_TYPE_LOWER_THAN; } break; - case 156: /* compare_op ::= NK_GT */ -{ yymsp[0].minor.yy260 = OP_TYPE_GREATER_THAN; } + case 217: /* compare_op ::= NK_GT */ +{ yymsp[0].minor.yy80 = OP_TYPE_GREATER_THAN; } break; - case 157: /* compare_op ::= NK_LE */ -{ yymsp[0].minor.yy260 = OP_TYPE_LOWER_EQUAL; } + case 218: /* compare_op ::= NK_LE */ +{ yymsp[0].minor.yy80 = OP_TYPE_LOWER_EQUAL; } break; - case 158: /* compare_op ::= NK_GE */ -{ yymsp[0].minor.yy260 = OP_TYPE_GREATER_EQUAL; } + case 219: /* compare_op ::= NK_GE */ +{ yymsp[0].minor.yy80 = OP_TYPE_GREATER_EQUAL; } break; - case 159: /* compare_op ::= NK_NE */ -{ yymsp[0].minor.yy260 = OP_TYPE_NOT_EQUAL; } + case 220: /* compare_op ::= NK_NE */ +{ yymsp[0].minor.yy80 = OP_TYPE_NOT_EQUAL; } break; - case 160: /* compare_op ::= NK_EQ */ -{ yymsp[0].minor.yy260 = OP_TYPE_EQUAL; } + case 221: /* compare_op ::= NK_EQ */ +{ yymsp[0].minor.yy80 = OP_TYPE_EQUAL; } break; - case 161: /* compare_op ::= LIKE */ -{ yymsp[0].minor.yy260 = OP_TYPE_LIKE; } + case 222: /* compare_op ::= LIKE */ +{ yymsp[0].minor.yy80 = OP_TYPE_LIKE; } break; - case 162: /* compare_op ::= NOT LIKE */ -{ yymsp[-1].minor.yy260 = OP_TYPE_NOT_LIKE; } + case 223: /* compare_op ::= NOT LIKE */ +{ yymsp[-1].minor.yy80 = OP_TYPE_NOT_LIKE; } break; - case 163: /* compare_op ::= MATCH */ -{ yymsp[0].minor.yy260 = OP_TYPE_MATCH; } + case 224: /* compare_op ::= MATCH */ +{ yymsp[0].minor.yy80 = OP_TYPE_MATCH; } break; - case 164: /* compare_op ::= NMATCH */ -{ yymsp[0].minor.yy260 = OP_TYPE_NMATCH; } + case 225: /* compare_op ::= NMATCH */ +{ yymsp[0].minor.yy80 = OP_TYPE_NMATCH; } break; - case 165: /* in_op ::= IN */ -{ yymsp[0].minor.yy260 = OP_TYPE_IN; } + case 226: /* in_op ::= IN */ +{ yymsp[0].minor.yy80 = OP_TYPE_IN; } break; - case 166: /* in_op ::= NOT IN */ -{ yymsp[-1].minor.yy260 = OP_TYPE_NOT_IN; } + case 227: /* in_op ::= NOT IN */ +{ yymsp[-1].minor.yy80 = OP_TYPE_NOT_IN; } break; - case 167: /* in_predicate_value ::= NK_LP expression_list NK_RP */ -{ yylhsminor.yy432 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy24)); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 228: /* in_predicate_value ::= NK_LP expression_list NK_RP */ +{ yylhsminor.yy26 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy64)); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 169: /* boolean_value_expression ::= NOT boolean_primary */ + case 230: /* boolean_value_expression ::= NOT boolean_primary */ { - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy432); - yylhsminor.yy432 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy432), NULL)); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); + yylhsminor.yy26 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy26), NULL)); } - yymsp[-1].minor.yy432 = yylhsminor.yy432; + yymsp[-1].minor.yy26 = yylhsminor.yy26; break; - case 170: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + case 231: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy432); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy432); - yylhsminor.yy432 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy432), releaseRawExprNode(pCxt, yymsp[0].minor.yy432))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy26); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); + yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 171: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + case 232: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy432); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy432); - yylhsminor.yy432 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy432), releaseRawExprNode(pCxt, yymsp[0].minor.yy432))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy26); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); + yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 176: /* from_clause ::= FROM table_reference_list */ - case 206: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==206); - case 229: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==229); -{ yymsp[-1].minor.yy432 = yymsp[0].minor.yy432; } + case 237: /* from_clause ::= FROM table_reference_list */ + case 267: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==267); + case 290: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==290); +{ yymsp[-1].minor.yy26 = yymsp[0].minor.yy26; } break; - case 178: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -{ yylhsminor.yy432 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy432, yymsp[0].minor.yy432, NULL); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 239: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ yylhsminor.yy26 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy26, yymsp[0].minor.yy26, NULL); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 181: /* table_primary ::= table_name alias_opt */ -{ yylhsminor.yy432 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy129, &yymsp[0].minor.yy129); } - yymsp[-1].minor.yy432 = yylhsminor.yy432; + case 242: /* table_primary ::= table_name alias_opt */ +{ yylhsminor.yy26 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy353, &yymsp[0].minor.yy353); } + yymsp[-1].minor.yy26 = yylhsminor.yy26; break; - case 182: /* table_primary ::= db_name NK_DOT table_name alias_opt */ -{ yylhsminor.yy432 = createRealTableNode(pCxt, &yymsp[-3].minor.yy129, &yymsp[-1].minor.yy129, &yymsp[0].minor.yy129); } - yymsp[-3].minor.yy432 = yylhsminor.yy432; + case 243: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ yylhsminor.yy26 = createRealTableNode(pCxt, &yymsp[-3].minor.yy353, &yymsp[-1].minor.yy353, &yymsp[0].minor.yy353); } + yymsp[-3].minor.yy26 = yylhsminor.yy26; break; - case 183: /* table_primary ::= subquery alias_opt */ -{ yylhsminor.yy432 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy432), &yymsp[0].minor.yy129); } - yymsp[-1].minor.yy432 = yylhsminor.yy432; + case 244: /* table_primary ::= subquery alias_opt */ +{ yylhsminor.yy26 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy26), &yymsp[0].minor.yy353); } + yymsp[-1].minor.yy26 = yylhsminor.yy26; break; - case 185: /* alias_opt ::= */ -{ yymsp[1].minor.yy129 = nil_token; } + case 246: /* alias_opt ::= */ +{ yymsp[1].minor.yy353 = nil_token; } break; - case 186: /* alias_opt ::= table_alias */ -{ yylhsminor.yy129 = yymsp[0].minor.yy129; } - yymsp[0].minor.yy129 = yylhsminor.yy129; + case 247: /* alias_opt ::= table_alias */ +{ yylhsminor.yy353 = yymsp[0].minor.yy353; } + yymsp[0].minor.yy353 = yylhsminor.yy353; break; - case 187: /* alias_opt ::= AS table_alias */ -{ yymsp[-1].minor.yy129 = yymsp[0].minor.yy129; } + case 248: /* alias_opt ::= AS table_alias */ +{ yymsp[-1].minor.yy353 = yymsp[0].minor.yy353; } break; - case 188: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - case 189: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==189); -{ yymsp[-2].minor.yy432 = yymsp[-1].minor.yy432; } + case 249: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + case 250: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==250); +{ yymsp[-2].minor.yy26 = yymsp[-1].minor.yy26; } break; - case 190: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -{ yylhsminor.yy432 = createJoinTableNode(pCxt, yymsp[-4].minor.yy332, yymsp[-5].minor.yy432, yymsp[-2].minor.yy432, yymsp[0].minor.yy432); } - yymsp[-5].minor.yy432 = yylhsminor.yy432; + case 251: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ yylhsminor.yy26 = createJoinTableNode(pCxt, yymsp[-4].minor.yy372, yymsp[-5].minor.yy26, yymsp[-2].minor.yy26, yymsp[0].minor.yy26); } + yymsp[-5].minor.yy26 = yylhsminor.yy26; break; - case 191: /* join_type ::= */ -{ yymsp[1].minor.yy332 = JOIN_TYPE_INNER; } + case 252: /* join_type ::= */ +{ yymsp[1].minor.yy372 = JOIN_TYPE_INNER; } break; - case 192: /* join_type ::= INNER */ -{ yymsp[0].minor.yy332 = JOIN_TYPE_INNER; } + case 253: /* join_type ::= INNER */ +{ yymsp[0].minor.yy372 = JOIN_TYPE_INNER; } break; - case 193: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 254: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { - yymsp[-8].minor.yy432 = createSelectStmt(pCxt, yymsp[-7].minor.yy97, yymsp[-6].minor.yy24, yymsp[-5].minor.yy432); - yymsp[-8].minor.yy432 = addWhereClause(pCxt, yymsp[-8].minor.yy432, yymsp[-4].minor.yy432); - yymsp[-8].minor.yy432 = addPartitionByClause(pCxt, yymsp[-8].minor.yy432, yymsp[-3].minor.yy24); - yymsp[-8].minor.yy432 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy432, yymsp[-2].minor.yy432); - yymsp[-8].minor.yy432 = addGroupByClause(pCxt, yymsp[-8].minor.yy432, yymsp[-1].minor.yy24); - yymsp[-8].minor.yy432 = addHavingClause(pCxt, yymsp[-8].minor.yy432, yymsp[0].minor.yy432); + yymsp[-8].minor.yy26 = createSelectStmt(pCxt, yymsp[-7].minor.yy107, yymsp[-6].minor.yy64, yymsp[-5].minor.yy26); + yymsp[-8].minor.yy26 = addWhereClause(pCxt, yymsp[-8].minor.yy26, yymsp[-4].minor.yy26); + yymsp[-8].minor.yy26 = addPartitionByClause(pCxt, yymsp[-8].minor.yy26, yymsp[-3].minor.yy64); + yymsp[-8].minor.yy26 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy26, yymsp[-2].minor.yy26); + yymsp[-8].minor.yy26 = addGroupByClause(pCxt, yymsp[-8].minor.yy26, yymsp[-1].minor.yy64); + yymsp[-8].minor.yy26 = addHavingClause(pCxt, yymsp[-8].minor.yy26, yymsp[0].minor.yy26); } break; - case 195: /* set_quantifier_opt ::= DISTINCT */ -{ yymsp[0].minor.yy97 = true; } + case 256: /* set_quantifier_opt ::= DISTINCT */ +{ yymsp[0].minor.yy107 = true; } break; - case 196: /* set_quantifier_opt ::= ALL */ -{ yymsp[0].minor.yy97 = false; } + case 257: /* set_quantifier_opt ::= ALL */ +{ yymsp[0].minor.yy107 = false; } break; - case 197: /* select_list ::= NK_STAR */ -{ yymsp[0].minor.yy24 = NULL; } + case 258: /* select_list ::= NK_STAR */ +{ yymsp[0].minor.yy64 = NULL; } break; - case 201: /* select_item ::= common_expression */ + case 262: /* select_item ::= common_expression */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy432); - yylhsminor.yy432 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy432), &t); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); + yylhsminor.yy26 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy26), &t); } - yymsp[0].minor.yy432 = yylhsminor.yy432; + yymsp[0].minor.yy26 = yylhsminor.yy26; break; - case 202: /* select_item ::= common_expression column_alias */ -{ yylhsminor.yy432 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy432), &yymsp[0].minor.yy129); } - yymsp[-1].minor.yy432 = yylhsminor.yy432; + case 263: /* select_item ::= common_expression column_alias */ +{ yylhsminor.yy26 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy26), &yymsp[0].minor.yy353); } + yymsp[-1].minor.yy26 = yylhsminor.yy26; break; - case 203: /* select_item ::= common_expression AS column_alias */ -{ yylhsminor.yy432 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy432), &yymsp[0].minor.yy129); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 264: /* select_item ::= common_expression AS column_alias */ +{ yylhsminor.yy26 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), &yymsp[0].minor.yy353); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 204: /* select_item ::= table_name NK_DOT NK_STAR */ -{ yylhsminor.yy432 = createColumnNode(pCxt, &yymsp[-2].minor.yy129, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 265: /* select_item ::= table_name NK_DOT NK_STAR */ +{ yylhsminor.yy26 = createColumnNode(pCxt, &yymsp[-2].minor.yy353, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 208: /* partition_by_clause_opt ::= PARTITION BY expression_list */ - case 225: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==225); - case 235: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==235); -{ yymsp[-2].minor.yy24 = yymsp[0].minor.yy24; } + case 269: /* partition_by_clause_opt ::= PARTITION BY expression_list */ + case 286: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==286); + case 296: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==296); +{ yymsp[-2].minor.yy64 = yymsp[0].minor.yy64; } break; - case 210: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ -{ yymsp[-5].minor.yy432 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy432), &yymsp[-1].minor.yy0); } + case 271: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ +{ yymsp[-5].minor.yy26 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy26), &yymsp[-1].minor.yy0); } break; - case 211: /* twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ -{ yymsp[-3].minor.yy432 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy432)); } + case 272: /* twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ +{ yymsp[-3].minor.yy26 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy26)); } break; - case 212: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-5].minor.yy432 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy432), NULL, yymsp[-1].minor.yy432, yymsp[0].minor.yy432); } + case 273: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-5].minor.yy26 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy26), NULL, yymsp[-1].minor.yy26, yymsp[0].minor.yy26); } break; - case 213: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-7].minor.yy432 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy432), releaseRawExprNode(pCxt, yymsp[-3].minor.yy432), yymsp[-1].minor.yy432, yymsp[0].minor.yy432); } + case 274: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-7].minor.yy26 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy26), releaseRawExprNode(pCxt, yymsp[-3].minor.yy26), yymsp[-1].minor.yy26, yymsp[0].minor.yy26); } break; - case 215: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ -{ yymsp[-3].minor.yy432 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy432); } + case 276: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ +{ yymsp[-3].minor.yy26 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy26); } break; - case 217: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -{ yymsp[-3].minor.yy432 = createFillNode(pCxt, yymsp[-1].minor.yy294, NULL); } + case 278: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ yymsp[-3].minor.yy26 = createFillNode(pCxt, yymsp[-1].minor.yy192, NULL); } break; - case 218: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ -{ yymsp[-5].minor.yy432 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy24)); } + case 279: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ +{ yymsp[-5].minor.yy26 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy64)); } break; - case 219: /* fill_mode ::= NONE */ -{ yymsp[0].minor.yy294 = FILL_MODE_NONE; } + case 280: /* fill_mode ::= NONE */ +{ yymsp[0].minor.yy192 = FILL_MODE_NONE; } break; - case 220: /* fill_mode ::= PREV */ -{ yymsp[0].minor.yy294 = FILL_MODE_PREV; } + case 281: /* fill_mode ::= PREV */ +{ yymsp[0].minor.yy192 = FILL_MODE_PREV; } break; - case 221: /* fill_mode ::= NULL */ -{ yymsp[0].minor.yy294 = FILL_MODE_NULL; } + case 282: /* fill_mode ::= NULL */ +{ yymsp[0].minor.yy192 = FILL_MODE_NULL; } break; - case 222: /* fill_mode ::= LINEAR */ -{ yymsp[0].minor.yy294 = FILL_MODE_LINEAR; } + case 283: /* fill_mode ::= LINEAR */ +{ yymsp[0].minor.yy192 = FILL_MODE_LINEAR; } break; - case 223: /* fill_mode ::= NEXT */ -{ yymsp[0].minor.yy294 = FILL_MODE_NEXT; } + case 284: /* fill_mode ::= NEXT */ +{ yymsp[0].minor.yy192 = FILL_MODE_NEXT; } break; - case 226: /* group_by_list ::= expression */ -{ yylhsminor.yy24 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy432))); } - yymsp[0].minor.yy24 = yylhsminor.yy24; + case 287: /* group_by_list ::= expression */ +{ yylhsminor.yy64 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); } + yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 227: /* group_by_list ::= group_by_list NK_COMMA expression */ -{ yylhsminor.yy24 = addNodeToList(pCxt, yymsp[-2].minor.yy24, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy432))); } - yymsp[-2].minor.yy24 = yylhsminor.yy24; + case 288: /* group_by_list ::= group_by_list NK_COMMA expression */ +{ yylhsminor.yy64 = addNodeToList(pCxt, yymsp[-2].minor.yy64, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); } + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 230: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 291: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ { - yylhsminor.yy432 = addOrderByClause(pCxt, yymsp[-3].minor.yy432, yymsp[-2].minor.yy24); - yylhsminor.yy432 = addSlimitClause(pCxt, yylhsminor.yy432, yymsp[-1].minor.yy432); - yylhsminor.yy432 = addLimitClause(pCxt, yylhsminor.yy432, yymsp[0].minor.yy432); + yylhsminor.yy26 = addOrderByClause(pCxt, yymsp[-3].minor.yy26, yymsp[-2].minor.yy64); + yylhsminor.yy26 = addSlimitClause(pCxt, yylhsminor.yy26, yymsp[-1].minor.yy26); + yylhsminor.yy26 = addLimitClause(pCxt, yylhsminor.yy26, yymsp[0].minor.yy26); } - yymsp[-3].minor.yy432 = yylhsminor.yy432; + yymsp[-3].minor.yy26 = yylhsminor.yy26; break; - case 232: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ -{ yylhsminor.yy432 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy432, yymsp[0].minor.yy432); } - yymsp[-3].minor.yy432 = yylhsminor.yy432; + case 293: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ +{ yylhsminor.yy26 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy26, yymsp[0].minor.yy26); } + yymsp[-3].minor.yy26 = yylhsminor.yy26; break; - case 237: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 241: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==241); -{ yymsp[-1].minor.yy432 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + case 298: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 302: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==302); +{ yymsp[-1].minor.yy26 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 238: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 242: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==242); -{ yymsp[-3].minor.yy432 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 299: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 303: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==303); +{ yymsp[-3].minor.yy26 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 239: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 243: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==243); -{ yymsp[-3].minor.yy432 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 300: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 304: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==304); +{ yymsp[-3].minor.yy26 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 244: /* subquery ::= NK_LP query_expression NK_RP */ -{ yylhsminor.yy432 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy432); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 305: /* subquery ::= NK_LP query_expression NK_RP */ +{ yylhsminor.yy26 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy26); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 245: /* search_condition ::= common_expression */ -{ yylhsminor.yy432 = releaseRawExprNode(pCxt, yymsp[0].minor.yy432); } - yymsp[0].minor.yy432 = yylhsminor.yy432; + case 306: /* search_condition ::= common_expression */ +{ yylhsminor.yy26 = releaseRawExprNode(pCxt, yymsp[0].minor.yy26); } + yymsp[0].minor.yy26 = yylhsminor.yy26; break; - case 248: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ -{ yylhsminor.yy432 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy432), yymsp[-1].minor.yy378, yymsp[0].minor.yy257); } - yymsp[-2].minor.yy432 = yylhsminor.yy432; + case 309: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ +{ yylhsminor.yy26 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), yymsp[-1].minor.yy32, yymsp[0].minor.yy391); } + yymsp[-2].minor.yy26 = yylhsminor.yy26; break; - case 249: /* ordering_specification_opt ::= */ -{ yymsp[1].minor.yy378 = ORDER_ASC; } + case 310: /* ordering_specification_opt ::= */ +{ yymsp[1].minor.yy32 = ORDER_ASC; } break; - case 250: /* ordering_specification_opt ::= ASC */ -{ yymsp[0].minor.yy378 = ORDER_ASC; } + case 311: /* ordering_specification_opt ::= ASC */ +{ yymsp[0].minor.yy32 = ORDER_ASC; } break; - case 251: /* ordering_specification_opt ::= DESC */ -{ yymsp[0].minor.yy378 = ORDER_DESC; } + case 312: /* ordering_specification_opt ::= DESC */ +{ yymsp[0].minor.yy32 = ORDER_DESC; } break; - case 252: /* null_ordering_opt ::= */ -{ yymsp[1].minor.yy257 = NULL_ORDER_DEFAULT; } + case 313: /* null_ordering_opt ::= */ +{ yymsp[1].minor.yy391 = NULL_ORDER_DEFAULT; } break; - case 253: /* null_ordering_opt ::= NULLS FIRST */ -{ yymsp[-1].minor.yy257 = NULL_ORDER_FIRST; } + case 314: /* null_ordering_opt ::= NULLS FIRST */ +{ yymsp[-1].minor.yy391 = NULL_ORDER_FIRST; } break; - case 254: /* null_ordering_opt ::= NULLS LAST */ -{ yymsp[-1].minor.yy257 = NULL_ORDER_LAST; } + case 315: /* null_ordering_opt ::= NULLS LAST */ +{ yymsp[-1].minor.yy391 = NULL_ORDER_LAST; } break; default: break; @@ -2839,13 +3237,15 @@ static void yy_syntax_error( ParseCTX_FETCH #define TOKEN yyminor /************ Begin %syntax_error code ****************************************/ - - if(TOKEN.z) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); - } else { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL); + + if (pCxt->valid) { + if(TOKEN.z) { + generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); + } else { + generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL); + } + pCxt->valid = false; } - pCxt->valid = false; /************ End %syntax_error code ******************************************/ ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ ParseCTX_STORE diff --git a/source/libs/parser/test/parserAstTest.cpp b/source/libs/parser/test/parserAstTest.cpp index f4c1bde478..bb2f2539e9 100644 --- a/source/libs/parser/test/parserAstTest.cpp +++ b/source/libs/parser/test/parserAstTest.cpp @@ -302,6 +302,13 @@ TEST_F(ParserTest, createUser) { ASSERT_TRUE(run()); } +TEST_F(ParserTest, alterAccount) { + setDatabase("root", "test"); + + bind("alter account ac_wxy pass '123456'"); + ASSERT_TRUE(run(TSDB_CODE_PAR_EXPRIE_STATEMENT)); +} + TEST_F(ParserTest, createDnode) { setDatabase("root", "test"); @@ -312,6 +319,16 @@ TEST_F(ParserTest, createDnode) { ASSERT_TRUE(run()); } +TEST_F(ParserTest, alterDnode) { + setDatabase("root", "test"); + + bind("alter dnode 1 'resetLog'"); + ASSERT_TRUE(run()); + + bind("alter dnode 1 'debugFlag' '134'"); + ASSERT_TRUE(run()); +} + TEST_F(ParserTest, createDatabase) { setDatabase("root", "test"); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 76a8fc3bd6..017de7e70d 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -18,13 +18,13 @@ #include "functionMgt.h" typedef struct SLogicPlanContext { - int32_t errCode; - int32_t planNodeId; - int32_t acctId; + SPlanContext* pPlanCxt; } SLogicPlanContext; -static SLogicNode* createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt); -static SLogicNode* createLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable); +typedef int32_t (*FCreateLogicNode)(SLogicPlanContext*, SSelectStmt*, SLogicNode**); + +static int32_t doCreateLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable, SLogicNode** pLogicNode); +static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogicNode** pLogicNode); typedef struct SRewriteExprCxt { int32_t errCode; @@ -66,7 +66,6 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { } typedef struct SNameExprCxt { - int32_t planNodeId; int32_t rewriteId; } SNameExprCxt; @@ -76,7 +75,7 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { case QUERY_NODE_LOGIC_CONDITION: case QUERY_NODE_FUNCTION: { SNameExprCxt* pCxt = (SNameExprCxt*)pContext; - sprintf(((SExprNode*)pNode)->aliasName, "#expr_%d_%d", pCxt->planNodeId, pCxt->rewriteId++); + sprintf(((SExprNode*)pNode)->aliasName, "#expr_%d", pCxt->rewriteId++); return DEAL_RES_IGNORE_CHILD; } default: @@ -86,131 +85,194 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static int32_t rewriteExpr(int32_t planNodeId, int32_t rewriteId, SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { - SNameExprCxt nameCxt = { .planNodeId = planNodeId, .rewriteId = rewriteId }; +static int32_t rewriteExpr(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { + static int32_t rewriteId = 1; + SNameExprCxt nameCxt = { .rewriteId = rewriteId }; nodesWalkList(pExprs, doNameExpr, &nameCxt); SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs }; nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); return cxt.errCode; } -static SLogicNode* pushLogicNode(SLogicPlanContext* pCxt, SLogicNode* pRoot, SLogicNode* pNode) { - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - goto error; - } - - if (NULL == pRoot) { - return pNode; - } - - if (NULL == pNode) { - return pRoot; - } - - if (NULL == pNode->pChildren) { - pNode->pChildren = nodesMakeList(); - if (NULL == pNode->pChildren) { - goto error; +static int32_t pushLogicNode(SLogicPlanContext* pCxt, SLogicNode** pOldRoot, SLogicNode* pNewRoot) { + if (NULL == pNewRoot->pChildren) { + pNewRoot->pChildren = nodesMakeList(); + if (NULL == pNewRoot->pChildren) { + return TSDB_CODE_OUT_OF_MEMORY; } } - if (TSDB_CODE_SUCCESS != nodesListAppend(pNode->pChildren, (SNode*)pRoot)) { - goto error; + if (TSDB_CODE_SUCCESS != nodesListAppend(pNewRoot->pChildren, (SNode*)*pOldRoot)) { + return TSDB_CODE_OUT_OF_MEMORY; } - pRoot->pParent = pNode; - return pNode; -error: - nodesDestroyNode((SNode*)pNode); - return pRoot; + + (*pOldRoot)->pParent = pNewRoot; + *pOldRoot = pNewRoot; + + return TSDB_CODE_SUCCESS; } -static SLogicNode* createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable) { +static int32_t createChildLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, FCreateLogicNode func, SLogicNode** pRoot) { + SLogicNode* pNode = NULL; + int32_t code = func(pCxt, pSelect, &pNode); + if (TSDB_CODE_SUCCESS == code && NULL != pNode) { + code = pushLogicNode(pCxt, pRoot, pNode); + } + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pNode); + } + return code; +} + +static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable, SLogicNode** pLogicNode) { SScanLogicNode* pScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN); - CHECK_ALLOC(pScan, NULL); - pScan->node.id = pCxt->planNodeId++; + if (NULL == pScan) { + return TSDB_CODE_OUT_OF_MEMORY; + } TSWAP(pScan->pMeta, pRealTable->pMeta, STableMeta*); TSWAP(pScan->pVgroupList, pRealTable->pVgroupList, SVgroupsInfo*); - - // set columns to scan - SNodeList* pCols = NULL; - CHECK_CODE(nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, &pCols), (SLogicNode*)pScan); - if (NULL != pCols) { - pScan->pScanCols = nodesCloneList(pCols); - CHECK_ALLOC(pScan->pScanCols, (SLogicNode*)pScan); - } - - // set output - if (NULL != pCols) { - pScan->node.pTargets = nodesCloneList(pCols); - CHECK_ALLOC(pScan->node.pTargets, (SLogicNode*)pScan); - } - - pScan->scanType = SCAN_TYPE_TABLE; + pScan->scanType = pCxt->pPlanCxt->topicQuery ? SCAN_TYPE_TOPIC : SCAN_TYPE_TABLE; pScan->scanFlag = MAIN_SCAN; pScan->scanRange = TSWINDOW_INITIALIZER; pScan->tableName.type = TSDB_TABLE_NAME_T; - pScan->tableName.acctId = pCxt->acctId; + pScan->tableName.acctId = pCxt->pPlanCxt->acctId; strcpy(pScan->tableName.dbname, pRealTable->table.dbName); strcpy(pScan->tableName.tname, pRealTable->table.tableName); - return (SLogicNode*)pScan; -} - -static SLogicNode* createSubqueryLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, STempTableNode* pTable) { - SLogicNode* pRoot = createQueryLogicNode(pCxt, pTable->pSubquery); - CHECK_ALLOC(pRoot, NULL); - SNode* pNode; - FOREACH(pNode, pRoot->pTargets) { - strcpy(((SColumnNode*)pNode)->tableAlias, pTable->table.tableAlias); + // set columns to scan + SNodeList* pCols = NULL; + int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, &pCols); + if (TSDB_CODE_SUCCESS == code && NULL != pCols) { + pScan->pScanCols = nodesCloneList(pCols); + if (NULL == pScan) { + code = TSDB_CODE_OUT_OF_MEMORY; + } } - return pRoot; + + // set output + if (TSDB_CODE_SUCCESS == code && NULL != pCols) { + pScan->node.pTargets = nodesCloneList(pCols); + if (NULL == pScan) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pScan; + } else { + nodesDestroyNode(pScan); + } + + return code; } -static SLogicNode* createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SJoinTableNode* pJoinTable) { +static int32_t createSubqueryLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, STempTableNode* pTable, SLogicNode** pLogicNode) { + int32_t code = createQueryLogicNode(pCxt, pTable->pSubquery, pLogicNode); + if (TSDB_CODE_SUCCESS == code) { + SNode* pNode; + FOREACH(pNode, (*pLogicNode)->pTargets) { + strcpy(((SColumnNode*)pNode)->tableAlias, pTable->table.tableAlias); + } + } + return code; +} + +static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SJoinTableNode* pJoinTable, SLogicNode** pLogicNode) { SJoinLogicNode* pJoin = (SJoinLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_JOIN); - CHECK_ALLOC(pJoin, NULL); - pJoin->node.id = pCxt->planNodeId++; + if (NULL == pJoin) { + return TSDB_CODE_OUT_OF_MEMORY; + } pJoin->joinType = pJoinTable->joinType; + int32_t code = TSDB_CODE_SUCCESS; + // set left and right node pJoin->node.pChildren = nodesMakeList(); - CHECK_ALLOC(pJoin->node.pChildren, (SLogicNode*)pJoin); - SLogicNode* pLeft = createLogicNodeByTable(pCxt, pSelect, pJoinTable->pLeft); - CHECK_ALLOC(pLeft, (SLogicNode*)pJoin); - CHECK_CODE(nodesListAppend(pJoin->node.pChildren, (SNode*)pLeft), (SLogicNode*)pJoin); - SLogicNode* pRight = createLogicNodeByTable(pCxt, pSelect, pJoinTable->pRight); - CHECK_ALLOC(pRight, (SLogicNode*)pJoin); - CHECK_CODE(nodesListAppend(pJoin->node.pChildren, (SNode*)pRight), (SLogicNode*)pJoin); + if (NULL == pJoin->node.pChildren) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + + SLogicNode* pLeft = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = doCreateLogicNodeByTable(pCxt, pSelect, pJoinTable->pLeft, &pLeft); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppend(pJoin->node.pChildren, (SNode*)pLeft); + } + } + + SLogicNode* pRight = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = doCreateLogicNodeByTable(pCxt, pSelect, pJoinTable->pRight, &pRight); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppend(pJoin->node.pChildren, (SNode*)pRight); + } + } // set on conditions - if (NULL != pJoinTable->pOnCond) { + if (TSDB_CODE_SUCCESS == code && NULL != pJoinTable->pOnCond) { pJoin->pOnConditions = nodesCloneNode(pJoinTable->pOnCond); - CHECK_ALLOC(pJoin->pOnConditions, (SLogicNode*)pJoin); + if (NULL == pJoin->pOnConditions) { + code = TSDB_CODE_OUT_OF_MEMORY; + } } // set the output - pJoin->node.pTargets = nodesCloneList(pLeft->pTargets); - CHECK_ALLOC(pJoin->node.pTargets, (SLogicNode*)pJoin); - SNodeList* pTargets = nodesCloneList(pRight->pTargets); - CHECK_ALLOC(pTargets, (SLogicNode*)pJoin); - nodesListAppendList(pJoin->node.pTargets, pTargets); + if (TSDB_CODE_SUCCESS == code) { + pJoin->node.pTargets = nodesCloneList(pLeft->pTargets); + if (NULL == pJoin->pOnConditions) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppendList(pJoin->node.pTargets, nodesCloneList(pRight->pTargets)); + } + } - return (SLogicNode*)pJoin; + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pJoin; + } else { + nodesDestroyNode(pJoin); + } + + return code; } -static SLogicNode* createLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable) { +static int32_t doCreateLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable, SLogicNode** pLogicNode) { switch (nodeType(pTable)) { case QUERY_NODE_REAL_TABLE: - return createScanLogicNode(pCxt, pSelect, (SRealTableNode*)pTable); + return createScanLogicNode(pCxt, pSelect, (SRealTableNode*)pTable, pLogicNode); case QUERY_NODE_TEMP_TABLE: - return createSubqueryLogicNode(pCxt, pSelect, (STempTableNode*)pTable); + return createSubqueryLogicNode(pCxt, pSelect, (STempTableNode*)pTable, pLogicNode); case QUERY_NODE_JOIN_TABLE: - return createJoinLogicNode(pCxt, pSelect, (SJoinTableNode*)pTable); + return createJoinLogicNode(pCxt, pSelect, (SJoinTableNode*)pTable, pLogicNode); default: break; } - return NULL; + return TSDB_CODE_FAILED; +} + +static int32_t createLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable, SLogicNode** pLogicNode) { + SLogicNode* pNode = NULL; + int32_t code = doCreateLogicNodeByTable(pCxt, pSelect, pTable, &pNode); + if (TSDB_CODE_SUCCESS == code) { + pNode->pConditions = nodesCloneNode(pSelect->pWhere); + if (NULL != pSelect->pWhere && NULL == pNode->pConditions) { + nodesDestroyNode(pNode); + return TSDB_CODE_OUT_OF_MEMORY; + } + *pLogicNode = pNode; + } + return code; +} + +static SColumnNode* createColumnByExpr(SExprNode* pExpr) { + SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return NULL; + } + pCol->node.resType = pExpr->resType; + strcpy(pCol->colName, pExpr->aliasName); + return pCol; } typedef struct SCreateColumnCxt { @@ -245,197 +307,241 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static SNodeList* createColumnByRewriteExps(SLogicPlanContext* pCxt, SNodeList* pExprs) { - SCreateColumnCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pList = nodesMakeList() }; - CHECK_ALLOC(cxt.pList, NULL); +static int32_t createColumnByRewriteExps(SLogicPlanContext* pCxt, SNodeList* pExprs, SNodeList** pList) { + SCreateColumnCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pList = (NULL == *pList ? nodesMakeList() : *pList) }; + if (NULL == cxt.pList) { + return TSDB_CODE_OUT_OF_MEMORY; + } nodesWalkList(pExprs, doCreateColumn, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(cxt.pList); - return NULL; + return cxt.errCode; } - return cxt.pList; + if (NULL == *pList) { + *pList = cxt.pList; + } + return cxt.errCode; } -static SLogicNode* createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { +static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { SNodeList* pAggFuncs = NULL; - CHECK_CODE(nodesCollectFuncs(pSelect, fmIsAggFunc, &pAggFuncs), NULL); + int32_t code = nodesCollectFuncs(pSelect, fmIsAggFunc, &pAggFuncs); + if (TSDB_CODE_SUCCESS != code) { + return code; + } if (NULL == pAggFuncs && NULL == pSelect->pGroupByList) { - return NULL; + return TSDB_CODE_SUCCESS; } SAggLogicNode* pAgg = (SAggLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_AGG); - CHECK_ALLOC(pAgg, NULL); - pAgg->node.id = pCxt->planNodeId++; + if (NULL == pAgg) { + return TSDB_CODE_OUT_OF_MEMORY; + } // set grouyp keys, agg funcs and having conditions if (NULL != pSelect->pGroupByList) { pAgg->pGroupKeys = nodesCloneList(pSelect->pGroupByList); - CHECK_ALLOC(pAgg->pGroupKeys, (SLogicNode*)pAgg); + if (NULL == pAgg->pGroupKeys) { + code = TSDB_CODE_OUT_OF_MEMORY; + } } - if (NULL != pAggFuncs) { + + if (TSDB_CODE_SUCCESS == code && NULL != pAggFuncs) { pAgg->pAggFuncs = nodesCloneList(pAggFuncs); - CHECK_ALLOC(pAgg->pAggFuncs, (SLogicNode*)pAgg); + if (NULL == pAgg->pAggFuncs) { + code = TSDB_CODE_OUT_OF_MEMORY; + } } // rewrite the expression in subsequent clauses - CHECK_CODE(rewriteExpr(pAgg->node.id, 1, pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY), (SLogicNode*)pAgg); - CHECK_CODE(rewriteExpr(pAgg->node.id, 1 + LIST_LENGTH(pAgg->pGroupKeys), pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY), (SLogicNode*)pAgg); + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExpr(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY); + } + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExpr(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY); + } - if (NULL != pSelect->pHaving) { + if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving) { pAgg->node.pConditions = nodesCloneNode(pSelect->pHaving); - CHECK_ALLOC(pAgg->node.pConditions, (SLogicNode*)pAgg); + if (NULL == pAgg->node.pConditions) { + code = TSDB_CODE_OUT_OF_MEMORY; + } } // set the output - pAgg->node.pTargets = nodesMakeList(); - CHECK_ALLOC(pAgg->node.pTargets, (SLogicNode*)pAgg); - if (NULL != pAgg->pGroupKeys) { - SNodeList* pTargets = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys); - CHECK_ALLOC(pAgg->node.pTargets, (SLogicNode*)pAgg); - nodesListAppendList(pAgg->node.pTargets, pTargets); + if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pGroupKeys) { + code = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys, &pAgg->node.pTargets); } - if (NULL != pAgg->pAggFuncs) { - SNodeList* pTargets = createColumnByRewriteExps(pCxt, pAgg->pAggFuncs); - CHECK_ALLOC(pTargets, (SLogicNode*)pAgg); - nodesListAppendList(pAgg->node.pTargets, pTargets); + if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pAggFuncs) { + code = createColumnByRewriteExps(pCxt, pAgg->pAggFuncs, &pAgg->node.pTargets); } - - return (SLogicNode*)pAgg; + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pAgg; + } else { + nodesDestroyNode(pAgg); + } + + return code; } -static SLogicNode* createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SIntervalWindowNode* pInterval, SSelectStmt* pSelect) { +static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SIntervalWindowNode* pInterval, SSelectStmt* pSelect, SLogicNode** pLogicNode) { SWindowLogicNode* pWindow = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_WINDOW); - CHECK_ALLOC(pWindow, NULL); - pWindow->node.id = pCxt->planNodeId++; + if (NULL == pWindow) { + return TSDB_CODE_OUT_OF_MEMORY; + } pWindow->winType = WINDOW_TYPE_INTERVAL; - //SValueNode* pIntervalNode = (SValueNode*)((SRawExprNode*)(pInterval->pInterval))->pNode; - SValueNode* pIntervalNode = (SValueNode*)(pInterval->pInterval); - - pWindow->interval = pIntervalNode->datum.i; + pWindow->interval = ((SValueNode*)pInterval->pInterval)->datum.i; + pWindow->intervalUnit = ((SValueNode*)pInterval->pInterval)->unit; pWindow->offset = (NULL != pInterval->pOffset ? ((SValueNode*)pInterval->pOffset)->datum.i : 0); pWindow->sliding = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->datum.i : pWindow->interval); + pWindow->slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit); + + int32_t code = TSDB_CODE_SUCCESS; if (NULL != pInterval->pFill) { pWindow->pFill = nodesCloneNode(pInterval->pFill); - CHECK_ALLOC(pWindow->pFill, (SLogicNode*)pWindow); + if (NULL == pWindow->pFill) { + code = TSDB_CODE_OUT_OF_MEMORY; + } } - SNodeList* pFuncs = NULL; - CHECK_CODE(nodesCollectFuncs(pSelect, fmIsAggFunc, &pFuncs), NULL); - if (NULL != pFuncs) { - pWindow->pFuncs = nodesCloneList(pFuncs); - CHECK_ALLOC(pWindow->pFuncs, (SLogicNode*)pWindow); + if (TSDB_CODE_SUCCESS == code) { + code = nodesCollectFuncs(pSelect, fmIsAggFunc, &pWindow->pFuncs); } - CHECK_CODE(rewriteExpr(pWindow->node.id, 1, pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW), (SLogicNode*)pWindow); + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExpr(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW); + } - pWindow->node.pTargets = createColumnByRewriteExps(pCxt, pWindow->pFuncs); - CHECK_ALLOC(pWindow->node.pTargets, (SLogicNode*)pWindow); + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByRewriteExps(pCxt, pWindow->pFuncs, &pWindow->node.pTargets); + } - return (SLogicNode*)pWindow; + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pWindow; + } else { + nodesDestroyNode(pWindow); + } + + return code; } -static SLogicNode* createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { +static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { if (NULL == pSelect->pWindow) { - return NULL; + return TSDB_CODE_SUCCESS; } switch (nodeType(pSelect->pWindow)) { case QUERY_NODE_INTERVAL_WINDOW: - return createWindowLogicNodeByInterval(pCxt, (SIntervalWindowNode*)pSelect->pWindow, pSelect); + return createWindowLogicNodeByInterval(pCxt, (SIntervalWindowNode*)pSelect->pWindow, pSelect, pLogicNode); default: break; } - return NULL; + return TSDB_CODE_FAILED; } -static SNodeList* createColumnByProjections(SLogicPlanContext* pCxt, SNodeList* pExprs) { +static int32_t createColumnByProjections(SLogicPlanContext* pCxt, SNodeList* pExprs, SNodeList** pCols) { SNodeList* pList = nodesMakeList(); - CHECK_ALLOC(pList, NULL); + if (NULL == pList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + SNode* pNode; - FOREACH(pNode, pExprs) { - SExprNode* pExpr = (SExprNode*)pNode; - SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); - if (NULL == pCol) { - goto error; - } - pCol->node.resType = pExpr->resType; - strcpy(pCol->colName, pExpr->aliasName); - if (TSDB_CODE_SUCCESS != nodesListAppend(pList, (SNode*)pCol)) { - goto error; + FOREACH(pNode, pExprs) { + if (TSDB_CODE_SUCCESS != nodesListAppend(pList, createColumnByExpr((SExprNode*)pNode))) { + nodesDestroyList(pList); + return TSDB_CODE_OUT_OF_MEMORY; } } - return pList; -error: - nodesDestroyList(pList); - return NULL; + + *pCols = pList; + return TSDB_CODE_SUCCESS; } -static SLogicNode* createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { +static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { SProjectLogicNode* pProject = (SProjectLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_PROJECT); - CHECK_ALLOC(pProject, NULL); - pProject->node.id = pCxt->planNodeId++; + if (NULL == pProject) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; pProject->pProjections = nodesCloneList(pSelect->pProjectionList); + if (NULL == pProject->pProjections) { + code = TSDB_CODE_OUT_OF_MEMORY; + } - pProject->node.pTargets = createColumnByProjections(pCxt,pSelect->pProjectionList); - CHECK_ALLOC(pProject->node.pTargets, (SLogicNode*)pProject); + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByProjections(pCxt,pSelect->pProjectionList, &pProject->node.pTargets); + } - return (SLogicNode*)pProject; + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pProject; + } else { + nodesDestroyNode(pProject); + } + + return code; } -static SLogicNode* createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { - SLogicNode* pRoot = createLogicNodeByTable(pCxt, pSelect, pSelect->pFromTable); - if (TSDB_CODE_SUCCESS == pCxt->errCode && NULL != pSelect->pWhere) { - pRoot->pConditions = nodesCloneNode(pSelect->pWhere); - CHECK_ALLOC(pRoot->pConditions, pRoot); +static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { + SLogicNode* pRoot = NULL; + int32_t code = createLogicNodeByTable(pCxt, pSelect, pSelect->pFromTable, &pRoot); + if (TSDB_CODE_SUCCESS == code) { + code = createChildLogicNode(pCxt, pSelect, createWindowLogicNode, &pRoot); } - if (TSDB_CODE_SUCCESS == pCxt->errCode) { - pRoot = pushLogicNode(pCxt, pRoot, createWindowLogicNode(pCxt, pSelect)); + if (TSDB_CODE_SUCCESS == code) { + code = createChildLogicNode(pCxt, pSelect, createAggLogicNode, &pRoot); } - if (TSDB_CODE_SUCCESS == pCxt->errCode) { - pRoot = pushLogicNode(pCxt, pRoot, createAggLogicNode(pCxt, pSelect)); + if (TSDB_CODE_SUCCESS == code) { + code = createChildLogicNode(pCxt, pSelect, createProjectLogicNode, &pRoot); } - if (TSDB_CODE_SUCCESS == pCxt->errCode) { - pRoot = pushLogicNode(pCxt, pRoot, createProjectLogicNode(pCxt, pSelect)); + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = pRoot; + } else { + nodesDestroyNode(pRoot); } - return pRoot; + + return code; } static int32_t getMsgType(ENodeType sqlType) { return (QUERY_NODE_CREATE_TABLE_STMT == sqlType || QUERY_NODE_CREATE_MULTI_TABLE_STMT == sqlType) ? TDMT_VND_CREATE_TABLE : TDMT_VND_SUBMIT; } -static SLogicNode* createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifOpStmt* pStmt) { - SVnodeModifLogicNode* pModif = (SVnodeModifLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIF); - CHECK_ALLOC(pModif, NULL); +static int32_t createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifOpStmt* pStmt, SLogicNode** pLogicNode) { + SVnodeModifLogicNode* pModif = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIF); + if (NULL == pModif) { + return TSDB_CODE_OUT_OF_MEMORY; + } pModif->pDataBlocks = pStmt->pDataBlocks; pModif->msgType = getMsgType(pStmt->sqlNodeType); - return (SLogicNode*)pModif; + *pLogicNode = (SLogicNode*)pModif; + return TSDB_CODE_SUCCESS; } -static SLogicNode* createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt) { +static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogicNode** pLogicNode) { switch (nodeType(pStmt)) { case QUERY_NODE_SELECT_STMT: - return createSelectLogicNode(pCxt, (SSelectStmt*)pStmt); + return createSelectLogicNode(pCxt, (SSelectStmt*)pStmt, pLogicNode); case QUERY_NODE_VNODE_MODIF_STMT: - return createVnodeModifLogicNode(pCxt, (SVnodeModifOpStmt*)pStmt); + return createVnodeModifLogicNode(pCxt, (SVnodeModifOpStmt*)pStmt, pLogicNode); default: break; } - return NULL; // to avoid compiler error + return TSDB_CODE_FAILED; } int32_t createLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode) { - SLogicPlanContext cxt = { .errCode = TSDB_CODE_SUCCESS, .planNodeId = 1, .acctId = pCxt->acctId }; - SLogicNode* pRoot = createQueryLogicNode(&cxt, pCxt->pAstRoot); - if (TSDB_CODE_SUCCESS != cxt.errCode) { - nodesDestroyNode((SNode*)pRoot); - return cxt.errCode; + SLogicPlanContext cxt = { .pPlanCxt = pCxt }; + int32_t code = createQueryLogicNode(&cxt, pCxt->pAstRoot, pLogicNode); + if (TSDB_CODE_SUCCESS != code) { + return code; } - *pLogicNode = pRoot; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 0affd93f4d..ebff05e2b7 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -259,15 +259,22 @@ static SPhysiNode* createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* p return (SPhysiNode*)pTableScan; } +static SPhysiNode* createStreamScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode) { + SStreamScanPhysiNode* pTableScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); + CHECK_ALLOC(pTableScan, NULL); + CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan), (SPhysiNode*)pTableScan); + return (SPhysiNode*)pTableScan; +} + static SPhysiNode* createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode) { switch (pScanLogicNode->scanType) { case SCAN_TYPE_TAG: return createTagScanPhysiNode(pCxt, pScanLogicNode); case SCAN_TYPE_TABLE: return createTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode); - case SCAN_TYPE_STABLE: + case SCAN_TYPE_TOPIC: case SCAN_TYPE_STREAM: - break; + return createStreamScanPhysiNode(pCxt, pSubplan, pScanLogicNode); default: break; } @@ -466,11 +473,20 @@ static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pC } static SPhysiNode* createExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode) { - SExchangePhysiNode* pExchange = (SExchangePhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE); - CHECK_ALLOC(pExchange, NULL); - CHECK_CODE(addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pExchange->node.pOutputDataBlockDesc), (SPhysiNode*)pExchange); - pExchange->srcGroupId = pExchangeLogicNode->srcGroupId; - return (SPhysiNode*)pExchange; + if (pCxt->pPlanCxt->streamQuery) { + SStreamScanPhysiNode* pScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); + CHECK_ALLOC(pScan, NULL); + pScan->pScanCols = nodesCloneList(pExchangeLogicNode->node.pTargets); + CHECK_ALLOC(pScan->pScanCols, (SPhysiNode*)pScan); + CHECK_CODE(addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pScan->node.pOutputDataBlockDesc), (SPhysiNode*)pScan); + return (SPhysiNode*)pScan; + } else { + SExchangePhysiNode* pExchange = (SExchangePhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE); + CHECK_ALLOC(pExchange, NULL); + CHECK_CODE(addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pExchange->node.pOutputDataBlockDesc), (SPhysiNode*)pExchange); + pExchange->srcGroupId = pExchangeLogicNode->srcGroupId; + return (SPhysiNode*)pExchange; + } } static SPhysiNode* createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode) { @@ -608,7 +624,9 @@ static SSubplan* createPhysiSubplan(SPhysiPlanContext* pCxt, SSubLogicPlan* pLog taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); } else { pSubplan->pNode = createPhysiNode(pCxt, pSubplan, pLogicSubplan->pNode); - pSubplan->pDataSink = createDataDispatcher(pCxt, pSubplan->pNode); + if (!pCxt->pPlanCxt->streamQuery && !pCxt->pPlanCxt->topicQuery) { + pSubplan->pDataSink = createDataDispatcher(pCxt, pSubplan->pNode); + } pSubplan->msgType = TDMT_VND_QUERY; } return pSubplan; diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 1d0cbf22df..c8e0852b46 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -44,7 +44,8 @@ typedef struct SStsInfo { } SStsInfo; static SLogicNode* stsMatchByNode(SLogicNode* pNode) { - if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode) && TSDB_SUPER_TABLE == ((SScanLogicNode*)pNode)->pMeta->tableType) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode) && TSDB_SUPER_TABLE == ((SScanLogicNode*)pNode)->pMeta->tableType && + SCAN_TYPE_TOPIC != ((SScanLogicNode*)pNode)->scanType) { return pNode; } SNode* pChild; diff --git a/source/libs/planner/test/plannerTest.cpp b/source/libs/planner/test/plannerTest.cpp index 8baa64d8fb..ccfdb19885 100644 --- a/source/libs/planner/test/plannerTest.cpp +++ b/source/libs/planner/test/plannerTest.cpp @@ -17,6 +17,7 @@ #include +#include "cmdnodes.h" #include "parser.h" #include "planInt.h" @@ -56,7 +57,8 @@ protected: const string syntaxTreeStr = toString(query_->pRoot, false); SLogicNode* pLogicPlan = nullptr; - SPlanContext cxt = { .queryId = 1, .acctId = 0, .pAstRoot = query_->pRoot }; + SPlanContext cxt = { .queryId = 1, .acctId = 0 }; + setPlanContext(query_, &cxt); code = createLogicPlan(&cxt, &pLogicPlan); if (code != TSDB_CODE_SUCCESS) { cout << "sql:[" << cxt_.pSql << "] logic plan code:" << code << ", strerror:" << tstrerror(code) << endl; @@ -94,6 +96,15 @@ protected: private: static const int max_err_len = 1024; + void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) { + if (QUERY_NODE_CREATE_TOPIC_STMT == nodeType(pQuery->pRoot)) { + pCxt->pAstRoot = ((SCreateTopicStmt*)pQuery->pRoot)->pQuery; + pCxt->topicQuery = true; + } else { + pCxt->pAstRoot = pQuery->pRoot; + } + } + void reset() { memset(&cxt_, 0, sizeof(cxt_)); memset(errMagBuf_, 0, max_err_len); @@ -173,3 +184,10 @@ TEST_F(PlannerTest, interval) { bind("SELECT count(*) FROM t1 interval(10s)"); ASSERT_TRUE(run()); } + +TEST_F(PlannerTest, createTopic) { + setDatabase("root", "test"); + + bind("create topic tp as SELECT * FROM st1"); + ASSERT_TRUE(run()); +} diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 63fbf59c06..b39d3e6e37 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -155,6 +155,10 @@ int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransp .ahandle = (void*)pInfo, .handle = pInfo->msgInfo.handle, .code = 0}; + if (pInfo->msgType == TDMT_VND_QUERY || pInfo->msgType == TDMT_VND_FETCH || + pInfo->msgType == TDMT_VND_QUERY_CONTINUE) { + rpcMsg.persistHandle = 1; + } assert(pInfo->fp != NULL); diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index b5b8726a4c..f806a7b212 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -150,7 +150,7 @@ typedef struct SQWorkerMgmt { SHashObj *ctxHash; //key: queryId+taskId, value: SQWTaskCtx void *nodeObj; putReqToQueryQFp putToQueueFp; - sendReqToDnodeFp sendReqFp; + sendReqFp sendReqFp; } SQWorkerMgmt; #define QW_FPARAMS_DEF SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 42890ab38a..73611a5ed9 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1443,7 +1443,7 @@ _return: } int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, - putReqToQueryQFp fp1, sendReqToDnodeFp fp2) { + putReqToQueryQFp fp1, sendReqFp fp2) { if (NULL == qWorkerMgmt || NULL == nodeObj || NULL == fp1 || NULL == fp2) { qError("invalid param to init qworker"); QW_RET(TSDB_CODE_QRY_INVALID_INPUT); diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index d7814a5dfb..a45ec41e6c 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -1030,15 +1030,15 @@ TEST(seqTest, multithreadRand) { code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t t1,t2,t3,t4,t5; - pthread_create(&(t1), &thattr, queryThread, mgmt); - pthread_create(&(t2), &thattr, readyThread, NULL); - pthread_create(&(t3), &thattr, fetchThread, NULL); - pthread_create(&(t4), &thattr, dropThread, NULL); - pthread_create(&(t5), &thattr, statusThread, NULL); + TdThread t1,t2,t3,t4,t5; + taosThreadCreate(&(t1), &thattr, queryThread, mgmt); + taosThreadCreate(&(t2), &thattr, readyThread, NULL); + taosThreadCreate(&(t3), &thattr, fetchThread, NULL); + taosThreadCreate(&(t4), &thattr, dropThread, NULL); + taosThreadCreate(&(t5), &thattr, statusThread, NULL); while (true) { if (qwtTestDeadLoop) { @@ -1089,13 +1089,13 @@ TEST(rcTest, shortExecshortDelay) { tsem_init(&qwtTestQuerySem, 0, 0); tsem_init(&qwtTestFetchSem, 0, 0); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t t1,t2,t3,t4,t5; - pthread_create(&(t1), &thattr, qwtclientThread, mgmt); - pthread_create(&(t2), &thattr, queryQueueThread, mgmt); - pthread_create(&(t3), &thattr, fetchQueueThread, mgmt); + TdThread t1,t2,t3,t4,t5; + taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); + taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); + taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); while (true) { if (qwtTestDeadLoop) { @@ -1170,13 +1170,13 @@ TEST(rcTest, longExecshortDelay) { tsem_init(&qwtTestQuerySem, 0, 0); tsem_init(&qwtTestFetchSem, 0, 0); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t t1,t2,t3,t4,t5; - pthread_create(&(t1), &thattr, qwtclientThread, mgmt); - pthread_create(&(t2), &thattr, queryQueueThread, mgmt); - pthread_create(&(t3), &thattr, fetchQueueThread, mgmt); + TdThread t1,t2,t3,t4,t5; + taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); + taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); + taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); while (true) { if (qwtTestDeadLoop) { @@ -1253,13 +1253,13 @@ TEST(rcTest, shortExeclongDelay) { tsem_init(&qwtTestQuerySem, 0, 0); tsem_init(&qwtTestFetchSem, 0, 0); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t t1,t2,t3,t4,t5; - pthread_create(&(t1), &thattr, qwtclientThread, mgmt); - pthread_create(&(t2), &thattr, queryQueueThread, mgmt); - pthread_create(&(t3), &thattr, fetchQueueThread, mgmt); + TdThread t1,t2,t3,t4,t5; + taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); + taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); + taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); while (true) { if (qwtTestDeadLoop) { @@ -1332,13 +1332,13 @@ TEST(rcTest, dropTest) { tsem_init(&qwtTestQuerySem, 0, 0); tsem_init(&qwtTestFetchSem, 0, 0); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t t1,t2,t3,t4,t5; - pthread_create(&(t1), &thattr, clientThread, mgmt); - pthread_create(&(t2), &thattr, queryQueueThread, mgmt); - pthread_create(&(t3), &thattr, fetchQueueThread, mgmt); + TdThread t1,t2,t3,t4,t5; + taosThreadCreate(&(t1), &thattr, clientThread, mgmt); + taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); + taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); while (true) { if (qwtTestDeadLoop) { diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 376ed1e2bc..b5bd68ba86 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -684,11 +684,11 @@ TEST(queryTest, normalCase) { pIter = taosHashIterate(pJob->execTasks, pIter); } - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t thread1; - pthread_create(&(thread1), &thattr, schtCreateFetchRspThread, &job); + TdThread thread1; + taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); void *data = NULL; code = schedulerFetchRows(job, &data); @@ -780,11 +780,11 @@ TEST(queryTest, flowCtrlCase) { } - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t thread1; - pthread_create(&(thread1), &thattr, schtCreateFetchRspThread, &job); + TdThread thread1; + taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); void *data = NULL; code = schedulerFetchRows(job, &data); @@ -834,11 +834,11 @@ TEST(insertTest, normalCase) { schtSetPlanToString(); schtSetAsyncSendMsgToServer(); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t thread1; - pthread_create(&(thread1), &thattr, schtSendRsp, &insertJobRefId); + TdThread thread1; + taosThreadCreate(&(thread1), &thattr, schtSendRsp, &insertJobRefId); SQueryResult res = {0}; code = schedulerExecJob(mockPointer, qnodeList, &dag, &insertJobRefId, "insert into tb values(now,1)", &res); @@ -851,13 +851,13 @@ TEST(insertTest, normalCase) { } TEST(multiThread, forceFree) { - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t thread1, thread2, thread3; - pthread_create(&(thread1), &thattr, schtRunJobThread, NULL); - pthread_create(&(thread2), &thattr, schtFreeJobThread, NULL); - pthread_create(&(thread3), &thattr, schtFetchRspThread, NULL); + TdThread thread1, thread2, thread3; + taosThreadCreate(&(thread1), &thattr, schtRunJobThread, NULL); + taosThreadCreate(&(thread2), &thattr, schtFreeJobThread, NULL); + taosThreadCreate(&(thread3), &thattr, schtFetchRspThread, NULL); while (true) { if (schtTestDeadLoop) { diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index 5a4b7555bf..0185d5f211 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -35,7 +35,7 @@ extern "C" { typedef struct SSyncIO { STaosQueue *pMsgQ; STaosQset * pQset; - pthread_t consumerTid; + TdThread consumerTid; void * serverRpc; void * clientRpc; diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index c841c2ac06..19121632e2 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -211,7 +211,7 @@ static int32_t syncIOStartInternal(SSyncIO *io) { // start consumer thread { - if (pthread_create(&io->consumerTid, NULL, syncIOConsumerFunc, io) != 0) { + if (taosThreadCreate(&io->consumerTid, NULL, syncIOConsumerFunc, io) != 0) { sError("failed to create sync consumer thread since %s", strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -228,7 +228,7 @@ static int32_t syncIOStartInternal(SSyncIO *io) { static int32_t syncIOStopInternal(SSyncIO *io) { int32_t ret = 0; atomic_store_8(&io->isStart, 0); - pthread_join(io->consumerTid, NULL); + taosThreadJoin(io->consumerTid, NULL); taosTmrCleanUp(io->timerMgr); return ret; } diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 3cb5a65572..978649499a 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -1,10 +1,17 @@ - -set(TDB_SUBDIRS "db") -foreach(TDB_SUBDIR ${TDB_SUBDIRS}) - aux_source_directory("src/${TDB_SUBDIR}" TDB_SRC) -endforeach() - -add_library(tdb STATIC ${TDB_SRC}) +# tdb +add_library(tdb "") +target_sources(tdb + PRIVATE + "src/db/tdbPCache.c" + "src/db/tdbPager.c" + "src/db/tdbUtil.c" + "src/db/tdbBtree.c" + "src/db/tdbDb.c" + "src/db/tdbEnv.c" + # "src/db/tdbPage.c" + "src/page/tdbPage.c" + "src/page/tdbPageL.c" +) target_include_directories( tdb @@ -17,6 +24,7 @@ target_link_libraries( PUBLIC util ) +# for test if(${BUILD_TEST}) add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index cc0d20ef3c..467e40325e 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -22,44 +22,42 @@ extern "C" { #endif -typedef struct STDb TDB; -typedef struct STDbEnv TENV; -typedef struct STDbCurosr TDBC; +// typedef struct STDb TDB; +// typedef struct STDbEnv TENV; +// typedef struct STDbCurosr TDBC; -typedef int32_t pgsz_t; -typedef int32_t cachesz_t; +// typedef int32_t pgsz_t; +// typedef int32_t cachesz_t; -typedef int (*TdbKeyCmprFn)(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2); +// typedef int (*TdbKeyCmprFn)(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2); -// TEVN -int tdbEnvCreate(TENV **ppEnv, const char *rootDir); -int tdbEnvOpen(TENV *ppEnv); -int tdbEnvClose(TENV *pEnv); +// // TEVN +// int tdbEnvCreate(TENV **ppEnv, const char *rootDir); +// int tdbEnvOpen(TENV *ppEnv); +// int tdbEnvClose(TENV *pEnv); -int tdbEnvSetCache(TENV *pEnv, pgsz_t pgSize, cachesz_t cacheSize); -pgsz_t tdbEnvGetPageSize(TENV *pEnv); -cachesz_t tdbEnvGetCacheSize(TENV *pEnv); +// int tdbEnvSetCache(TENV *pEnv, pgsz_t pgSize, cachesz_t cacheSize); +// pgsz_t tdbEnvGetPageSize(TENV *pEnv); +// cachesz_t tdbEnvGetCacheSize(TENV *pEnv); -int tdbEnvBeginTxn(TENV *pEnv); -int tdbEnvCommit(TENV *pEnv); +// int tdbEnvBeginTxn(TENV *pEnv); +// int tdbEnvCommit(TENV *pEnv); -// TDB -int tdbCreate(TDB **ppDb); -int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv); -int tdbClose(TDB *pDb); -int tdbDrop(TDB *pDb); +// // TDB +// int tdbCreate(TDB **ppDb); +// int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv); +// int tdbClose(TDB *pDb); +// int tdbDrop(TDB *pDb); -int tdbSetKeyLen(TDB *pDb, int klen); -int tdbSetValLen(TDB *pDb, int vlen); -int tdbSetDup(TDB *pDb, int dup); -int tdbSetCmprFunc(TDB *pDb, TdbKeyCmprFn fn); -int tdbGetKeyLen(TDB *pDb); -int tdbGetValLen(TDB *pDb); -int tdbGetDup(TDB *pDb); +// int tdbSetKeyLen(TDB *pDb, int klen); +// int tdbSetValLen(TDB *pDb, int vlen); +// int tdbSetDup(TDB *pDb, int dup); +// int tdbSetCmprFunc(TDB *pDb, TdbKeyCmprFn fn); +// int tdbGetKeyLen(TDB *pDb); +// int tdbGetValLen(TDB *pDb); +// int tdbGetDup(TDB *pDb); -int tdbInsert(TDB *pDb, const void *pKey, int nKey, const void *pData, int nData); - -// TDBC +// int tdbInsert(TDB *pDb, const void *pKey, int nKey, const void *pData, int nData); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/db/tdb.c b/source/libs/tdb/src/db/tdb.c deleted file mode 100644 index 65d4cf80cc..0000000000 --- a/source/libs/tdb/src/db/tdb.c +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdbInt.h" - -struct STDb { - char dbname[TDB_MAX_DBNAME_LEN]; - SBTree * pBt; // current access method (may extend) - SPgFile * pPgFile; // backend page file this DB is using - TENV * pEnv; // TENV containing the DB - int klen; // key length if know - int vlen; // value length if know - bool dup; // dup mode - TdbKeyCmprFn cFn; // compare function -}; - -struct STDbCurosr { - SBtCursor *pBtCur; -}; - -static int tdbDefaultKeyCmprFn(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2); - -int tdbCreate(TDB **ppDb) { - TDB *pDb; - - // create the handle - pDb = (TDB *)calloc(1, sizeof(*pDb)); - if (pDb == NULL) { - return -1; - } - - pDb->klen = TDB_VARIANT_LEN; - pDb->vlen = TDB_VARIANT_LEN; - pDb->dup = false; - pDb->cFn = tdbDefaultKeyCmprFn; - - *ppDb = pDb; - return 0; -} - -static int tdbDestroy(TDB *pDb) { - if (pDb) { - free(pDb); - } - return 0; -} - -int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv) { - int ret; - uint8_t fileid[TDB_FILE_ID_LEN]; - SPgFile * pPgFile; - SPgCache *pPgCache; - SBTree * pBt; - bool fileExist; - size_t dbNameLen; - pgno_t dbRootPgno; - char dbfname[128]; // TODO: make this as a macro or malloc on the heap - - ASSERT(pDb != NULL); - ASSERT(fname != NULL); - // TODO: Here we simply put an assert here. In the future, make `pEnv` - // can be set as NULL. - ASSERT(pEnv != NULL); - - // check the DB name - dbNameLen = 0; - if (dbname) { - dbNameLen = strlen(dbname); - if (dbNameLen >= TDB_MAX_DBNAME_LEN) { - return -1; - } - - memcpy(pDb->dbname, dbname, dbNameLen); - } - - pDb->dbname[dbNameLen] = '\0'; - - // get page file from the env, if not opened yet, open it - pPgFile = NULL; - snprintf(dbfname, 128, "%s/%s", tdbEnvGetRootDir(pEnv), fname); - fileExist = taosCheckExistFile(fname); - if (fileExist) { - tdbGnrtFileID(dbfname, fileid, false); - pPgFile = tdbEnvGetPageFile(pEnv, fileid); - } - - if (pPgFile == NULL) { - ret = pgFileOpen(&pPgFile, dbfname, pEnv); - if (ret != 0) { - // TODO: handle error - return -1; - } - } - - // TODO: get the root page number from the master DB of the page file - // tdbGet(&dbRootPgno); - if (dbRootPgno == 0) { - // DB not exist, create one - ret = pgFileAllocatePage(pPgFile, &dbRootPgno); - if (ret != 0) { - // TODO: handle error - } - // tdbInsert(pPgFile->pMasterDB, dbname, strlen(dbname), &dbRootPgno, sizeof(dbRootPgno)); - } - - ASSERT(dbRootPgno > 1); - - // pDb->pBt->root = dbRootPgno; - - // register - pDb->pPgFile = pPgFile; - tdbEnvRgstDB(pEnv, pDb); - pDb->pEnv = pEnv; - - return 0; -} - -int tdbClose(TDB *pDb) { - if (pDb == NULL) return 0; - return tdbDestroy(pDb); -} - -int tdbDrop(TDB *pDb) { - // TODO - return 0; -} - -int tdbSetKeyLen(TDB *pDb, int klen) { - // TODO: check `klen` - pDb->klen = klen; - return 0; -} - -int tdbSetValLen(TDB *pDb, int vlen) { - // TODO: check `vlen` - pDb->vlen = vlen; - return 0; -} - -int tdbSetDup(TDB *pDb, int dup) { - if (dup) { - pDb->dup = true; - } else { - pDb->dup = false; - } - return 0; -} - -int tdbSetCmprFunc(TDB *pDb, TdbKeyCmprFn fn) { - if (fn == NULL) { - return -1; - } else { - pDb->cFn = fn; - } - return 0; -} - -int tdbGetKeyLen(TDB *pDb) { return pDb->klen; } - -int tdbGetValLen(TDB *pDb) { return pDb->vlen; } - -int tdbGetDup(TDB *pDb) { - if (pDb->dup) { - return 1; - } else { - return 0; - } -} - -int tdbInsert(TDB *pDb, const void *pKey, int nKey, const void *pData, int nData) { - // TODO - return 0; -} - -static int tdbDefaultKeyCmprFn(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2) { - int mlen; - int cret; - - ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); - - mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; - cret = memcmp(pKey1, pKey2, mlen); - if (cret == 0) { - if (keyLen1 < keyLen2) { - cret = -1; - } else if (keyLen1 > keyLen2) { - cret = 1; - } else { - cret = 0; - } - } - return cret; -} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index a2520babe5..c6e8c9dca9 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -15,46 +15,99 @@ #include "tdbInt.h" -struct SBtCursor { - SBTree *pBtree; - pgno_t pgno; - SPage * pPage; // current page traversing +#define TDB_BTREE_ROOT 0x1 +#define TDB_BTREE_LEAF 0x2 + +#define TDB_BTREE_PAGE_IS_ROOT(flags) TDB_FLAG_HAS(flags, TDB_BTREE_ROOT) +#define TDB_BTREE_PAGE_IS_LEAF(flags) TDB_FLAG_HAS(flags, TDB_BTREE_LEAF) +#define TDB_BTREE_ASSERT_FLAG(flags) \ + ASSERT(TDB_FLAG_IS(flags, TDB_BTREE_ROOT) || TDB_FLAG_IS(flags, TDB_BTREE_LEAF) || \ + TDB_FLAG_IS(flags, TDB_BTREE_ROOT | TDB_BTREE_LEAF) || TDB_FLAG_IS(flags, 0)) + +struct SBTree { + SPgno root; + int keyLen; + int valLen; + SPager *pPager; + FKeyComparator kcmpr; + u8 fanout; + int pageSize; + int maxLocal; + int minLocal; + int maxLeaf; + int minLeaf; + u8 *pTmp; }; +typedef struct __attribute__((__packed__)) { + SPgno rChild; +} SBtPageHdr; + typedef struct { - pgno_t pgno; - pgsz_t offset; -} SBtIdx; + u16 flags; + SBTree *pBt; +} SBtreeZeroPageArg; -// Btree page header definition -#pragma pack (push,1) typedef struct { - uint8_t flag; // page flag - int32_t vlen; // value length of current page, TDB_VARIANT_LEN for variant length - uint16_t nPayloads; // number of total payloads - pgoff_t freeOff; // free payload offset - pgsz_t fragSize; // total fragment size - pgoff_t offPayload; // payload offset - pgno_t rChildPgno; // right most child page number -} SBtPgHdr; -#pragma pack(pop) + int kLen; + u8 *pKey; + int vLen; + u8 *pVal; + SPgno pgno; + u8 *pTmpSpace; +} SCellDecoder; -typedef int (*BtreeCmprFn)(const void *, const void *); +static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *pCRst); +static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2); +static int tdbBtreeOpenImpl(SBTree *pBt); +static int tdbBtreeZeroPage(SPage *pPage, void *arg); +static int tdbBtreeInitPage(SPage *pPage, void *arg); +static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell, + int *szCell); +static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder); +static int tdbBtreeBalance(SBtCursor *pCur); -#define BTREE_PAGE_HDR(pPage) NULL /* TODO */ -#define BTREE_PAGE_PAYLOAD_AT(pPage, idx) NULL /*TODO*/ -#define BTREE_PAGE_IS_LEAF(pPage) 0 /* TODO */ - -static int btreeCreate(SBTree **ppBt); -static int btreeDestroy(SBTree *pBt); -static int btreeCursorMoveToChild(SBtCursor *pBtCur, pgno_t pgno); - -int btreeOpen(SBTree **ppBt, SPgFile *pPgFile) { +int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, SBTree **ppBt) { SBTree *pBt; int ret; - ret = btreeCreate(&pBt); - if (ret != 0) { + *ppBt = NULL; + + pBt = (SBTree *)calloc(1, sizeof(*pBt)); + if (pBt == NULL) { + return -1; + } + + // pBt->keyLen + pBt->keyLen = keyLen; + // pBt->valLen + pBt->valLen = valLen; + // pBt->pPager + pBt->pPager = pPager; + // pBt->kcmpr + pBt->kcmpr = kcmpr ? kcmpr : tdbDefaultKeyCmprFn; + // pBt->fanout + if (keyLen == TDB_VARIANT_LEN) { + pBt->fanout = TDB_DEFAULT_FANOUT; + } else { + ASSERT(0); + // TODO: pBt->fanout = 0; + } + // pBt->pageSize + pBt->pageSize = tdbPagerGetPageSize(pPager); + // pBt->maxLocal + pBt->maxLocal = (pBt->pageSize - 14) / pBt->fanout; + // pBt->minLocal: Should not be allowed smaller than 15, which is [nPayload][nKey][nData] + pBt->minLocal = (pBt->pageSize - 14) / pBt->fanout / 2; + // pBt->maxLeaf + pBt->maxLeaf = pBt->pageSize - 14; + // pBt->minLeaf + pBt->minLeaf = pBt->minLocal; + + // TODO: pBt->root + ret = tdbBtreeOpenImpl(pBt); + if (ret < 0) { + free(pBt); return -1; } @@ -62,105 +115,828 @@ int btreeOpen(SBTree **ppBt, SPgFile *pPgFile) { return 0; } -int btreeClose(SBTree *pBt) { +int tdbBtreeClose(SBTree *pBt) { // TODO return 0; } -static int btreeCreate(SBTree **ppBt) { +int tdbBtreeCursor(SBtCursor *pCur, SBTree *pBt) { + pCur->pBt = pBt; + pCur->iPage = -1; + pCur->pPage = NULL; + pCur->idx = -1; + + return 0; +} + +int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *pVal, int vLen) { + int ret; + int idx; + SPager *pPager; + SCell *pCell; + int szCell; + int cret; SBTree *pBt; - pBt = (SBTree *)calloc(1, sizeof(*pBt)); - if (pBt == NULL) { + ret = tdbBtCursorMoveTo(pCur, pKey, kLen, &cret); + if (ret < 0) { + // TODO: handle error return -1; } - // TODO - return 0; -} - -static int btreeDestroy(SBTree *pBt) { - if (pBt) { - free(pBt); - } - return 0; -} - -int btreeCursorOpen(SBtCursor *pBtCur, SBTree *pBt) { - // TODO - return 0; -} - -int btreeCursorClose(SBtCursor *pBtCur) { - // TODO - return 0; -} - -int btreeCursorMoveTo(SBtCursor *pBtCur, int kLen, const void *pKey) { - SPage * pPage; - SBtPgHdr * pBtPgHdr; - SPgFile * pPgFile; - pgno_t childPgno; - pgno_t rootPgno; - int nPayloads; - void * pPayload; - BtreeCmprFn cmpFn; - - // 1. Move the cursor to the root page - if (rootPgno == TDB_IVLD_PGNO) { - // No any data in this btree, just return not found (TODO) - return 0; + if (pCur->idx == -1) { + ASSERT(TDB_PAGE_NCELLS(pCur->pPage) == 0); + idx = 0; } else { - // Load the page from the file by the SPgFile handle - pPage = pgFileFetch(pPgFile, rootPgno); - - pBtCur->pPage = pPage; - } - - // 2. Loop to search over the whole tree - for (;;) { - int lidx, ridx, midx, cret; - - pPage = pBtCur->pPage; - pBtPgHdr = BTREE_PAGE_HDR(pPage); - nPayloads = pBtPgHdr->nPayloads; - - // Binary search the page - lidx = 0; - ridx = nPayloads - 1; - midx = (lidx + ridx) >> 1; - for (;;) { - // get the payload ptr at midx - pPayload = BTREE_PAGE_PAYLOAD_AT(pPage, midx); - - // the payload and the key - cret = cmpFn(pKey, pPayload); - - if (cret < 0) { - /* TODO */ - } else if (cret > 0) { - /* TODO */ - } else { - /* TODO */ - } - - if (lidx > ridx) break; - midx = (lidx + ridx) >> 1; - } - if (BTREE_PAGE_IS_LEAF(pPage)) { - /* TODO */ - break; + if (cret > 0) { + idx = pCur->idx + 1; + } else if (cret < 0) { + idx = pCur->idx; } else { /* TODO */ - btreeCursorMoveToChild(pBtCur, childPgno); + ASSERT(0); + } + } + + // TODO: refact code here + pBt = pCur->pBt; + if (!pBt->pTmp) { + pBt->pTmp = (u8 *)malloc(pBt->pageSize); + if (pBt->pTmp == NULL) { + return -1; + } + } + + pCell = pBt->pTmp; + + // Encode the cell + ret = tdbBtreeEncodeCell(pCur->pPage, pKey, kLen, pVal, vLen, pCell, &szCell); + if (ret < 0) { + return -1; + } + + // Insert the cell to the index + ret = tdbPageInsertCell(pCur->pPage, idx, pCell, szCell); + if (ret < 0) { + return -1; + } + + // If page is overflow, balance the tree + if (pCur->pPage->nOverflow > 0) { + ret = tdbBtreeBalance(pCur); + if (ret < 0) { + return -1; } } return 0; } -static int btreeCursorMoveToChild(SBtCursor *pBtCur, pgno_t pgno) { - SPgFile *pPgFile; +static int tdbBtCursorMoveToChild(SBtCursor *pCur, SPgno pgno) { // TODO return 0; -} \ No newline at end of file +} + +static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *pCRst) { + int ret; + SBTree *pBt; + SPager *pPager; + + pBt = pCur->pBt; + pPager = pBt->pPager; + + if (pCur->iPage < 0) { + ASSERT(pCur->iPage == -1); + ASSERT(pCur->idx == -1); + + // Move from the root + ret = tdbPagerFetchPage(pPager, pBt->root, &(pCur->pPage), tdbBtreeInitPage, pBt); + if (ret < 0) { + ASSERT(0); + return -1; + } + + pCur->iPage = 0; + + if (TDB_PAGE_NCELLS(pCur->pPage) == 0) { + // Current page is empty + ASSERT(TDB_FLAG_IS(TDB_PAGE_FLAGS(pCur->pPage), TDB_BTREE_ROOT | TDB_BTREE_LEAF)); + return 0; + } + + for (;;) { + int lidx, ridx, midx, c, nCells; + SCell *pCell; + SPage *pPage; + SCellDecoder cd = {0}; + + pPage = pCur->pPage; + nCells = TDB_PAGE_NCELLS(pPage); + lidx = 0; + ridx = nCells - 1; + + ASSERT(nCells > 0); + + for (;;) { + if (lidx > ridx) break; + + midx = (lidx + ridx) >> 1; + + pCell = TDB_PAGE_CELL_AT(pPage, midx); + ret = tdbBtreeDecodeCell(pPage, pCell, &cd); + if (ret < 0) { + // TODO: handle error + ASSERT(0); + return -1; + } + + // Compare the key values + c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen); + if (c < 0) { + /* input-key < cell-key */ + ridx = midx - 1; + } else if (c > 0) { + /* input-key > cell-key */ + lidx = midx + 1; + } else { + /* input-key == cell-key */ + break; + } + } + + // Move downward or break + u16 flags = TDB_PAGE_FLAGS(pPage); + u8 leaf = TDB_BTREE_PAGE_IS_LEAF(flags); + if (leaf) { + pCur->idx = midx; + *pCRst = c; + break; + } else { + if (c <= 0) { + pCur->idx = midx; + tdbBtCursorMoveToChild(pCur, cd.pgno); + } else { + if (midx == nCells - 1) { + /* Move to right-most child */ + pCur->idx = midx + 1; + tdbBtCursorMoveToChild(pCur, ((SBtPageHdr *)(pPage->pAmHdr))->rChild); + } else { + // TODO: reset cd as uninitialized + pCur->idx = midx + 1; + pCell = TDB_PAGE_CELL_AT(pPage, midx + 1); + tdbBtreeDecodeCell(pPage, pCell, &cd); + tdbBtCursorMoveToChild(pCur, cd.pgno); + } + } + } + } + + } else { + // TODO: Move the cursor from a some position instead of a clear state + ASSERT(0); + } + + return 0; +} + +static int tdbBtCursorMoveToRoot(SBtCursor *pCur) { + SBTree *pBt; + SPager *pPager; + SPage *pPage; + int ret; + + pBt = pCur->pBt; + pPager = pBt->pPager; + + // pPage = tdbPagerGet(pPager, pBt->root, true); + // if (pPage == NULL) { + // // TODO: handle error + // } + + // ret = tdbInitBtPage(pPage, &pBtPage); + // if (ret < 0) { + // // TODO + // return 0; + // } + + // pCur->pPage = pBtPage; + // pCur->iPage = 0; + + return 0; +} + +static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { + int mlen; + int cret; + + ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); + + mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; + cret = memcmp(pKey1, pKey2, mlen); + if (cret == 0) { + if (keyLen1 < keyLen2) { + cret = -1; + } else if (keyLen1 > keyLen2) { + cret = 1; + } else { + cret = 0; + } + } + return cret; +} + +static int tdbBtreeOpenImpl(SBTree *pBt) { + // Try to get the root page of the an existing btree + + SPgno pgno; + SPage *pPage; + int ret; + + { + // 1. TODO: Search the main DB to check if the DB exists + pgno = 0; + } + + if (pgno != 0) { + pBt->root = pgno; + return 0; + } + + // Try to create a new database + SBtreeZeroPageArg zArg = {.flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF, .pBt = pBt}; + ret = tdbPagerNewPage(pBt->pPager, &pgno, &pPage, tdbBtreeZeroPage, &zArg); + if (ret < 0) { + return -1; + } + + // TODO: Unref the page + + ASSERT(pgno != 0); + pBt->root = pgno; + + return 0; +} + +static int tdbBtreeInitPage(SPage *pPage, void *arg) { + SBTree *pBt; + u16 flags; + u8 isLeaf; + + pBt = (SBTree *)arg; + + flags = TDB_PAGE_FLAGS(pPage); + isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); + if (isLeaf) { + pPage->szAmHdr = 0; + } else { + pPage->szAmHdr = sizeof(SBtPageHdr); + } + pPage->pPageHdr = pPage->pData; + pPage->pAmHdr = pPage->pPageHdr + pPage->pPageMethods->szPageHdr; + pPage->pCellIdx = pPage->pAmHdr + pPage->szAmHdr; + pPage->pFreeStart = pPage->pCellIdx + pPage->pPageMethods->szOffset * TDB_PAGE_NCELLS(pPage); + pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); + pPage->pPageFtr = (SPageFtr *)(pPage->pData + pPage->pageSize - sizeof(SPageFtr)); + + TDB_BTREE_ASSERT_FLAG(flags); + + // Init other fields + if (isLeaf) { + pPage->kLen = pBt->keyLen; + pPage->vLen = pBt->valLen; + pPage->maxLocal = pBt->maxLeaf; + pPage->minLocal = pBt->minLeaf; + } else { + pPage->kLen = pBt->keyLen; + pPage->vLen = sizeof(SPgno); + pPage->maxLocal = pBt->maxLocal; + pPage->minLocal = pBt->minLocal; + } + + // TODO: need to update the SPage.nFree + pPage->nFree = pPage->pFreeEnd - pPage->pFreeStart; + pPage->nOverflow = 0; + + return 0; +} + +static int tdbBtreeZeroPage(SPage *pPage, void *arg) { + u16 flags; + SBTree *pBt; + + flags = ((SBtreeZeroPageArg *)arg)->flags; + pBt = ((SBtreeZeroPageArg *)arg)->pBt; + + pPage->pPageHdr = pPage->pData; + + // Init the page header + TDB_PAGE_FLAGS_SET(pPage, flags); + TDB_PAGE_NCELLS_SET(pPage, 0); + TDB_PAGE_CCELLS_SET(pPage, pBt->pageSize - sizeof(SPageFtr)); + TDB_PAGE_FCELL_SET(pPage, 0); + TDB_PAGE_NFREE_SET(pPage, 0); + + tdbBtreeInitPage(pPage, (void *)pBt); + + return 0; +} + +#ifndef TDB_BTREE_BALANCE +typedef struct { + SBTree *pBt; + SPage *pParent; + int idx; + i8 nOld; + SPage *pOldPages[3]; + i8 nNewPages; + SPage *pNewPages[5]; +} SBtreeBalanceHelper; + +static int tdbBtreeCopyPageContent(SPage *pFrom, SPage *pTo) { + int nCells = TDB_PAGE_NCELLS(pFrom); + int cCells = TDB_PAGE_CCELLS(pFrom); + int fCell = TDB_PAGE_FCELL(pFrom); + int nFree = TDB_PAGE_NFREE(pFrom); + + pTo->pFreeStart = pTo->pCellIdx + nCells * pFrom->pPageMethods->szOffset; + memcpy(pTo->pCellIdx, pFrom->pCellIdx, nCells * pFrom->pPageMethods->szOffset); + pTo->pFreeEnd = (u8 *)pTo->pPageFtr - (u8 *)(pFrom->pPageFtr) + pFrom->pFreeEnd; + memcpy(pTo->pFreeEnd, pFrom->pFreeEnd, (u8 *)pFrom->pPageFtr - pFrom->pFreeEnd); + + TDB_PAGE_NCELLS_SET(pTo, nCells); + TDB_PAGE_CCELLS_SET(pTo, cCells); + TDB_PAGE_FCELL_SET(pTo, fCell); + TDB_PAGE_NFREE_SET(pTo, nFree); + + // TODO: update other fields + + return 0; +} + +static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { + SPager *pPager; + SPage *pChild; + SPgno pgnoChild; + int ret; + SBtreeZeroPageArg zArg; + + pPager = pRoot->pPager; + + // Allocate a new child page + zArg.flags = TDB_BTREE_LEAF; + zArg.pBt = pBt; + ret = tdbPagerNewPage(pPager, &pgnoChild, &pChild, tdbBtreeZeroPage, &zArg); + if (ret < 0) { + return -1; + } + + // Copy the root page content to the child page + ret = tdbBtreeCopyPageContent(pRoot, pChild); + if (ret < 0) { + return -1; + } + + pChild->nOverflow = pRoot->nOverflow; + for (int i = 0; i < pChild->nOverflow; i++) { + pChild->apOvfl[i] = pRoot->apOvfl[i]; + pChild->aiOvfl[i] = pRoot->aiOvfl[i]; + } + + // Reinitialize the root page + zArg.flags = TDB_BTREE_ROOT; + zArg.pBt = pBt; + ret = tdbBtreeZeroPage(pRoot, &zArg); + if (ret < 0) { + return -1; + } + + ((SBtPageHdr *)pRoot->pAmHdr)[0].rChild = pgnoChild; + + *ppChild = pChild; + return 0; +} + +static int tdbBtreeBalanceStep1(SBtreeBalanceHelper *pBlh) { + int nCells; + int i; + int idxStart; + int nChild; + int ret; + SPage *pParent; + SPgno pgno; + SCell *pCell; + SCellDecoder cd; + SBTree *pBt; + + pParent = pBlh->pParent; + nCells = TDB_PAGE_NCELLS(pParent); + nChild = nCells + 1; + pBt = pBlh->pBt; + + // TODO: ASSERT below needs to be removed + ASSERT(pParent->nOverflow == 0); + ASSERT(pBlh->idx <= nCells); + + if (nChild < 3) { + idxStart = 0; + pBlh->nOld = nChild; + } else { + if (pBlh->idx == 0) { + idxStart = 0; + } else if (pBlh->idx == nCells) { + idxStart = pBlh->idx - 2; + } else { + idxStart = pBlh->idx - 1; + } + pBlh->nOld = 3; + } + + i = pBlh->nOld - 1; + + if (idxStart + i == nCells) { + pgno = ((SBtPageHdr *)(pParent->pAmHdr))[0].rChild; + } else { + pCell = TDB_PAGE_CELL_AT(pParent, idxStart + i); + // TODO: no need to decode the payload part, and even the kLen, vLen part + // we only need the pgno part + ret = tdbBtreeDecodeCell(pParent, pCell, &cd); + if (ret < 0) { + ASSERT(0); + return -1; + } + pgno = cd.pgno; + } + for (;;) { + ret = tdbPagerFetchPage(pBt->pPager, pgno, &(pBlh->pOldPages[i]), tdbBtreeInitPage, pBt); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // Loop over + if ((i--) == 0) break; + + { + // TODO + // ASSERT(0); + } + } + + return 0; +} + +static int tdbBtreeBalanceStep2(SBtreeBalanceHelper *pBlh) { +#if 0 + SPage *pPage; + int oidx; + int cidx; + int limit; + SCell *pCell; + + for (int i = 0; i < pBlh->nOld; i++) { + pPage = pBlh->pOldPages[i]; + oidx = 0; + cidx = 0; + + if (oidx < pPage->nOverflow) { + limit = pPage->aiOvfl[oidx]; + } else { + limit = pPage->pPageHdr->nCells; + } + + // Loop to copy each cell pointer out + for (;;) { + if (oidx >= pPage->nOverflow && cidx >= pPage->pPageHdr->nCells) break; + + if (cidx < limit) { + // Get local cells + pCell = TDB_PAGE_CELL_AT(pPage, cidx); + } else if (cidx == limit) { + // Get overflow cells + pCell = pPage->apOvfl[oidx++]; + + if (oidx < pPage->nOverflow) { + limit = pPage->aiOvfl[oidx]; + } else { + limit = pPage->pPageHdr->nCells; + } + } else { + ASSERT(0); + } + } + + { + // TODO: Copy divider cells here + } + } + + /* TODO */ + +#endif + return 0; +} + +static int tdbBtreeBalanceStep3(SBtreeBalanceHelper *pBlh) { + // Figure out number of pages needed after balance + for (int i = 0; i < pBlh->nOld; i++) { + /* TODO */ + } + + return 0; +} + +static int tdbBtreeBalanceStep4(SBtreeBalanceHelper *pBlh) { + // TODO + return 0; +} + +static int tdbBtreeBalanceStep5(SBtreeBalanceHelper *pBlh) { + // TODO + return 0; +} + +static int tdbBtreeBalanceStep6(SBtreeBalanceHelper *pBlh) { + // TODO + return 0; +} + +static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { + int ret; + SBtreeBalanceHelper blh; + + ASSERT(!TDB_BTREE_PAGE_IS_LEAF(TDB_PAGE_FLAGS(pParent))); + + blh.pBt = pBt; + blh.pParent = pParent; + blh.idx = idx; + + // Step 1: find two sibling pages and get engough info about the old pages + ret = tdbBtreeBalanceStep1(&blh); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // Step 2: Load all cells on the old page and the divider cells + ret = tdbBtreeBalanceStep2(&blh); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // Step 3: Get the number of pages needed to hold all cells + ret = tdbBtreeBalanceStep3(&blh); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // Step 4: Allocate enough new pages. Reuse old pages as much as possible + ret = tdbBtreeBalanceStep4(&blh); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // Step 5: Insert new divider cells into pParent + ret = tdbBtreeBalanceStep5(&blh); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // Step 6: Update the sibling pages + ret = tdbBtreeBalanceStep6(&blh); + if (ret < 0) { + ASSERT(0); + return -1; + } + + { + // TODO: Reset states + } + + { + // TODO: Clear resources + } + + return 0; +} + +static int tdbBtreeBalance(SBtCursor *pCur) { + int iPage; + SPage *pParent; + SPage *pPage; + int ret; + u16 flags; + u8 leaf; + u8 root; + + // Main loop to balance the BTree + for (;;) { + iPage = pCur->iPage; + pPage = pCur->pPage; + flags = TDB_PAGE_FLAGS(pPage); + leaf = TDB_BTREE_PAGE_IS_LEAF(flags); + root = TDB_BTREE_PAGE_IS_ROOT(flags); + + // TODO: Get the page free space if not get yet + // if (pPage->nFree < 0) { + // if (tdbBtreeComputeFreeSpace(pPage) < 0) { + // return -1; + // } + // } + + // when the page is not overflow and not too empty, the balance work + // is finished. Just break out the balance loop. + if (pPage->nOverflow == 0 /* TODO: && pPage->nFree <= */) { + break; + } + + if (iPage == 0) { + // For the root page, only balance when the page is overfull, + // ignore the case of empty + if (pPage->nOverflow == 0) break; + + ret = tdbBtreeBalanceDeeper(pCur->pBt, pCur->pPage, &(pCur->pgStack[1])); + if (ret < 0) { + return -1; + } + + pCur->idx = 0; + pCur->idxStack[0] = 0; + pCur->pgStack[0] = pCur->pPage; + pCur->iPage = 1; + pCur->pPage = pCur->pgStack[1]; + } else { + // Generalized balance step + pParent = pCur->pgStack[iPage - 1]; + + ret = tdbBtreeBalanceNonRoot(pCur->pBt, pParent, pCur->idxStack[pCur->iPage - 1]); + if (ret < 0) { + return -1; + } + + pCur->iPage--; + pCur->pPage = pCur->pgStack[pCur->iPage]; + } + } + + return 0; +} +#endif + +#ifndef TDB_BTREE_CELL // ========================================================= +static int tdbBtreeEncodePayload(SPage *pPage, u8 *pPayload, const void *pKey, int kLen, const void *pVal, int vLen, + int *szPayload) { + int nPayload; + + ASSERT(pKey != NULL); + + if (pVal == NULL) { + vLen = 0; + } + + nPayload = kLen + vLen; + if (nPayload <= pPage->maxLocal) { + // General case without overflow + memcpy(pPayload, pKey, kLen); + if (pVal) { + memcpy(pPayload + kLen, pVal, vLen); + } + + *szPayload = nPayload; + return 0; + } + + { + // TODO: handle overflow case + ASSERT(0); + } + + return 0; +} + +static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell, + int *szCell) { + u16 flags; + u8 leaf; + int nHeader; + int nPayload; + int ret; + + ASSERT(pPage->kLen == TDB_VARIANT_LEN || pPage->kLen == kLen); + ASSERT(pPage->vLen == TDB_VARIANT_LEN || pPage->vLen == vLen); + + nPayload = 0; + nHeader = 0; + flags = TDB_PAGE_FLAGS(pPage); + leaf = TDB_BTREE_PAGE_IS_LEAF(flags); + + // 1. Encode Header part + /* Encode kLen if need */ + if (pPage->kLen == TDB_VARIANT_LEN) { + nHeader += tdbPutVarInt(pCell + nHeader, kLen); + } + + /* Encode vLen if need */ + if (pPage->vLen == TDB_VARIANT_LEN) { + nHeader += tdbPutVarInt(pCell + nHeader, vLen); + } + + /* Encode SPgno if interior page */ + if (!leaf) { + ASSERT(pPage->vLen == sizeof(SPgno)); + + ((SPgno *)(pCell + nHeader))[0] = ((SPgno *)pVal)[0]; + nHeader = nHeader + sizeof(SPgno); + } + + // 2. Encode payload part + if (leaf) { + ret = tdbBtreeEncodePayload(pPage, pCell + nHeader, pKey, kLen, pVal, vLen, &nPayload); + } else { + ret = tdbBtreeEncodePayload(pPage, pCell + nHeader, pKey, kLen, NULL, 0, &nPayload); + } + if (ret < 0) { + // TODO: handle error + return -1; + } + + *szCell = nHeader + nPayload; + return 0; +} + +static int tdbBtreeDecodePayload(SPage *pPage, const u8 *pPayload, SCellDecoder *pDecoder) { + int nPayload; + + ASSERT(pDecoder->pKey == NULL); + + if (pDecoder->pVal) { + nPayload = pDecoder->kLen + pDecoder->vLen; + } else { + nPayload = pDecoder->kLen; + } + + if (nPayload <= pPage->maxLocal) { + // General case without overflow + pDecoder->pKey = (void *)pPayload; + if (!pDecoder->pVal) { + pDecoder->pVal = (void *)(pPayload + pDecoder->kLen); + } + } else { + // TODO: handle overflow case + ASSERT(0); + } + + return 0; +} + +static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder) { + u16 flags; + u8 leaf; + int nHeader; + int ret; + + nHeader = 0; + flags = TDB_PAGE_FLAGS(pPage); + leaf = TDB_BTREE_PAGE_IS_LEAF(flags); + + // Clear the state of decoder + pDecoder->kLen = -1; + pDecoder->pKey = NULL; + pDecoder->vLen = -1; + pDecoder->pVal = NULL; + pDecoder->pgno = 0; + + // 1. Decode header part + if (pPage->kLen == TDB_VARIANT_LEN) { + nHeader += tdbGetVarInt(pCell + nHeader, &(pDecoder->kLen)); + } else { + pDecoder->kLen = pPage->kLen; + } + + if (pPage->vLen == TDB_VARIANT_LEN) { + nHeader += tdbGetVarInt(pCell + nHeader, &(pDecoder->vLen)); + } else { + pDecoder->vLen = pPage->vLen; + } + + if (!leaf) { + ASSERT(pPage->vLen == sizeof(SPgno)); + + pDecoder->pgno = ((SPgno *)(pCell + nHeader))[0]; + pDecoder->pVal = (u8 *)(&(pDecoder->pgno)); + nHeader = nHeader + sizeof(SPgno); + } + + // 2. Decode payload part + ret = tdbBtreeDecodePayload(pPage, pCell + nHeader, pDecoder); + if (ret < 0) { + return -1; + } + + return 0; +} + +#endif \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c new file mode 100644 index 0000000000..00f38a19bb --- /dev/null +++ b/source/libs/tdb/src/db/tdbDb.c @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdbInt.h" + +struct STDb { + STEnv *pEnv; + SBTree *pBt; +}; + +int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDb **ppDb) { + STDb *pDb; + SPager *pPager; + int ret; + char fFullName[TDB_FILENAME_LEN]; + SPage *pPage; + SPgno pgno; + + *ppDb = NULL; + + pDb = (STDb *)calloc(1, sizeof(*pDb)); + if (pDb == NULL) { + return -1; + } + + // pDb->pEnv + pDb->pEnv = pEnv; + + pPager = tdbEnvGetPager(pEnv, fname); + if (pPager == NULL) { + snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->rootDir, fname); + ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager); + if (ret < 0) { + return -1; + } + } + + ASSERT(pPager != NULL); + + // pDb->pBt + ret = tdbBtreeOpen(keyLen, valLen, pPager, keyCmprFn, &(pDb->pBt)); + if (ret < 0) { + return -1; + } + + *ppDb = pDb; + return 0; +} + +int tdbDbClose(STDb *pDb) { + // TODO + return 0; +} + +int tdbDbDrop(STDb *pDb) { + // TODO + return 0; +} + +int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) { + SBtCursor btc; + SBtCursor *pCur; + int ret; + + pCur = &btc; + ret = tdbBtreeCursor(pCur, pDb->pBt); + if (ret < 0) { + return -1; + } + + ret = tdbBtCursorInsert(pCur, pKey, keyLen, pVal, valLen); + if (ret < 0) { + return -1; + } + + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbEnv.c b/source/libs/tdb/src/db/tdbEnv.c index 30e8fab039..9a4dcdbcd5 100644 --- a/source/libs/tdb/src/db/tdbEnv.c +++ b/source/libs/tdb/src/db/tdbEnv.c @@ -15,155 +15,56 @@ #include "tdbInt.h" -struct STDbEnv { - char * rootDir; // root directory of the environment - char * jname; // journal file name - TdFilePtr jpFile; // journal file fd - pgsz_t pgSize; // page size - cachesz_t cacheSize; // total cache size - STDbList dbList; // TDB List - SPgFileList pgfList; // SPgFile List - SPgCache * pPgCache; // page cache - struct { -#define TDB_ENV_PGF_HASH_BUCKETS 17 - SPgFileList buckets[TDB_ENV_PGF_HASH_BUCKETS]; - } pgfht; // page file hash table; -}; - -#define TDB_ENV_PGF_HASH(fileid) (((uint8_t *)(fileid))[0] + ((uint8_t *)(fileid))[1] + ((uint8_t *)(fileid))[2]) - -static int tdbEnvDestroy(TENV *pEnv); - -int tdbEnvCreate(TENV **ppEnv, const char *rootDir) { - TENV * pEnv; - size_t slen; - size_t jlen; - - ASSERT(rootDir != NULL); +int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, STEnv **ppEnv) { + STEnv *pEnv; + int dsize; + int zsize; + u8 *pPtr; + int ret; *ppEnv = NULL; - slen = strlen(rootDir); - jlen = slen + strlen(TDB_JOURNAL_NAME) + 1; - pEnv = (TENV *)calloc(1, sizeof(*pEnv) + slen + 1 + jlen + 1); - if (pEnv == NULL) { + + dsize = strlen(rootDir); + zsize = sizeof(*pEnv) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3; + + pPtr = (uint8_t *)calloc(1, zsize); + if (pPtr == NULL) { return -1; } - pEnv->rootDir = (char *)(&pEnv[1]); - pEnv->jname = pEnv->rootDir + slen + 1; - pEnv->jpFile = NULL; - pEnv->pgSize = TDB_DEFAULT_PGSIZE; - pEnv->cacheSize = TDB_DEFAULT_CACHE_SIZE; + pEnv = (STEnv *)pPtr; + pPtr += sizeof(*pEnv); + // pEnv->rootDir + pEnv->rootDir = pPtr; + memcpy(pEnv->rootDir, rootDir, dsize); + pEnv->rootDir[dsize] = '\0'; + pPtr = pPtr + dsize + 1; + // pEnv->jfname + pEnv->jfname = pPtr; + memcpy(pEnv->jfname, rootDir, dsize); + pEnv->jfname[dsize] = '/'; + memcpy(pEnv->jfname + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME)); + pEnv->jfname[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0'; - memcpy(pEnv->rootDir, rootDir, slen); - pEnv->rootDir[slen] = '\0'; - sprintf(pEnv->jname, "%s/%s", rootDir, TDB_JOURNAL_NAME); + pEnv->jfd = -1; - TD_DLIST_INIT(&(pEnv->dbList)); - TD_DLIST_INIT(&(pEnv->pgfList)); + ret = tdbPCacheOpen(pageSize, cacheSize, &(pEnv->pCache)); + if (ret < 0) { + return -1; + } - /* TODO */ + mkdir(rootDir, 0755); *ppEnv = pEnv; return 0; } -int tdbEnvOpen(TENV *pEnv) { - SPgCache *pPgCache; - int ret; - - ASSERT(pEnv != NULL); - - /* TODO: here we do not need to create the root directory, more - * work should be done here - */ - mkdir(pEnv->rootDir, 0755); - - ret = pgCacheOpen(&pPgCache, pEnv); - if (ret != 0) { - goto _err; - } - - pEnv->pPgCache = pPgCache; - return 0; - -_err: - return -1; -} - -int tdbEnvClose(TENV *pEnv) { - if (pEnv == NULL) return 0; - pgCacheClose(pEnv->pPgCache); - tdbEnvDestroy(pEnv); - return 0; -} - -int tdbEnvSetCache(TENV *pEnv, pgsz_t pgSize, cachesz_t cacheSize) { - if (!TDB_IS_PGSIZE_VLD(pgSize) || cacheSize / pgSize < 10) { - return -1; - } - - /* TODO */ - - pEnv->pgSize = pgSize; - pEnv->cacheSize = cacheSize; - - return 0; -} - -pgsz_t tdbEnvGetPageSize(TENV *pEnv) { return pEnv->pgSize; } - -cachesz_t tdbEnvGetCacheSize(TENV *pEnv) { return pEnv->cacheSize; } - -SPgFile *tdbEnvGetPageFile(TENV *pEnv, const uint8_t fileid[]) { - SPgFileList *pBucket; - SPgFile * pPgFile; - - pBucket = pEnv->pgfht.buckets + (TDB_ENV_PGF_HASH(fileid) % TDB_ENV_PGF_HASH_BUCKETS); // TODO - for (pPgFile = TD_DLIST_HEAD(pBucket); pPgFile != NULL; pPgFile = TD_DLIST_NODE_NEXT_WITH_FIELD(pPgFile, envHash)) { - if (memcmp(fileid, pPgFile->fileid, TDB_FILE_ID_LEN) == 0) break; - }; - - return pPgFile; -} - -SPgCache *tdbEnvGetPgCache(TENV *pEnv) { return pEnv->pPgCache; } - -static int tdbEnvDestroy(TENV *pEnv) { +int tdbEnvClose(STEnv *pEnv) { // TODO return 0; } -int tdbEnvBeginTxn(TENV *pEnv) { - pEnv->jpFile = taosOpenFile(pEnv->jname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); - if (pEnv->jpFile == NULL) { - return -1; - } - - return 0; -} - -int tdbEnvCommit(TENV *pEnv) { - /* TODO */ - taosCloseFile(&pEnv->jpFile); - pEnv->jpFile = NULL; - return 0; -} - -const char *tdbEnvGetRootDir(TENV *pEnv) { return pEnv->rootDir; } - -int tdbEnvRgstPageFile(TENV *pEnv, SPgFile *pPgFile) { - SPgFileList *pBucket; - - TD_DLIST_APPEND_WITH_FIELD(&(pEnv->pgfList), pPgFile, envPgfList); - - pBucket = pEnv->pgfht.buckets + (TDB_ENV_PGF_HASH(pPgFile->fileid) % TDB_ENV_PGF_HASH_BUCKETS); // TODO - TD_DLIST_APPEND_WITH_FIELD(pBucket, pPgFile, envHash); - - return 0; -} - -int tdbEnvRgstDB(TENV *pEnv, TDB *pDb) { +SPager *tdbEnvGetPager(STEnv *pEnv, const char *fname) { // TODO - return 0; + return NULL; } \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c new file mode 100644 index 0000000000..9d7181c1da --- /dev/null +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -0,0 +1,309 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "tdbInt.h" + +struct SPCache { + int pageSize; + int cacheSize; + TdThreadMutex mutex; + int nFree; + SPage *pFree; + int nPage; + int nHash; + SPage **pgHash; + int nRecyclable; + SPage lru; +}; + +#define PCACHE_PAGE_HASH(pPgid) \ + ({ \ + u32 *t = (u32 *)((pPgid)->fileid); \ + t[0] + t[1] + t[2] + t[3] + t[4] + t[5] + (pPgid)->pgno; \ + }) +#define PAGE_IS_PINNED(pPage) ((pPage)->pLruNext == NULL) + +// For page ref +#define TDB_INIT_PAGE_REF(pPage) ((pPage)->nRef = 0) +#if 0 +#define TDB_REF_PAGE(pPage) (++(pPage)->nRef) +#define TDB_UNREF_PAGE(pPage) (--(pPage)->nRef) +#define TDB_GET_PAGE_REF(pPage) ((pPage)->nRef) +#else +#define TDB_REF_PAGE(pPage) atomic_add_fetch_32(&((pPage)->nRef), 1) +#define TDB_UNREF_PAGE(pPage) atomic_sub_fetch_32(&((pPage)->nRef), 1) +#define TDB_GET_PAGE_REF(pPage) atomic_load_32(&((pPage)->nRef)) +#endif + +static int tdbPCacheOpenImpl(SPCache *pCache); +static void tdbPCacheInitLock(SPCache *pCache); +static void tdbPCacheClearLock(SPCache *pCache); +static void tdbPCacheLock(SPCache *pCache); +static void tdbPCacheUnlock(SPCache *pCache); +static bool tdbPCacheLocked(SPCache *pCache); +static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNewPage); +static void tdbPCachePinPage(SPage *pPage); +static void tdbPCacheRemovePageFromHash(SPage *pPage); +static void tdbPCacheAddPageToHash(SPage *pPage); +static void tdbPCacheUnpinPage(SPage *pPage); +static void *tdbOsMalloc(void *arg, size_t size); +static void tdbOsFree(void *arg, void *ptr); + +int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) { + SPCache *pCache; + void *pPtr; + SPage *pPgHdr; + + pCache = (SPCache *)calloc(1, sizeof(*pCache)); + if (pCache == NULL) { + return -1; + } + + pCache->pageSize = pageSize; + pCache->cacheSize = cacheSize; + + if (tdbPCacheOpenImpl(pCache) < 0) { + free(pCache); + return -1; + } + + *ppCache = pCache; + return 0; +} + +int tdbPCacheClose(SPCache *pCache) { + /* TODO */ + return 0; +} + +SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage) { + SPage *pPage; + + tdbPCacheLock(pCache); + + pPage = tdbPCacheFetchImpl(pCache, pPgid, alcNewPage); + if (pPage) { + TDB_REF_PAGE(pPage); + } + + tdbPCacheUnlock(pCache); + + return pPage; +} + +void tdbPCacheRelease(SPage *pPage) { + i32 nRef; + + nRef = TDB_UNREF_PAGE(pPage); + ASSERT(nRef >= 0); + + if (nRef == 0) { + if (1 /*TODO: page still clean*/) { + tdbPCacheUnpinPage(pPage); + } else { + // TODO + ASSERT(0); + } + } +} + +static void tdbPCacheInitLock(SPCache *pCache) { taosThreadMutexInit(&(pCache->mutex), NULL); } + +static void tdbPCacheClearLock(SPCache *pCache) { taosThreadMutexDestroy(&(pCache->mutex)); } + +static void tdbPCacheLock(SPCache *pCache) { taosThreadMutexLock(&(pCache->mutex)); } + +static void tdbPCacheUnlock(SPCache *pCache) { taosThreadMutexUnlock(&(pCache->mutex)); } + +static bool tdbPCacheLocked(SPCache *pCache) { + assert(0); + // TODO + return true; +} + +static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNewPage) { + SPage *pPage; + + // 1. Search the hash table + pPage = pCache->pgHash[PCACHE_PAGE_HASH(pPgid) % pCache->nHash]; + while (pPage) { + if (TDB_IS_SAME_PAGE(&(pPage->pgid), pPgid)) break; + pPage = pPage->pHashNext; + } + + if (pPage || !alcNewPage) { + if (pPage) { + tdbPCachePinPage(pPage); + } + return pPage; + } + + // 2. Try to allocate a new page from the free list + if (pCache->pFree) { + pPage = pCache->pFree; + pCache->pFree = pPage->pFreeNext; + pCache->nFree--; + pPage->pLruNext = NULL; + } + + // 3. Try to Recycle a page + if (!pPage && !pCache->lru.pLruPrev->isAnchor) { + pPage = pCache->lru.pLruPrev; + tdbPCacheRemovePageFromHash(pPage); + tdbPCachePinPage(pPage); + } + + // 4. Try a stress allocation (TODO) + + // 5. Page here are just created from a free list + // or by recycling or allocated streesly, + // need to initialize it + if (pPage) { + memcpy(&(pPage->pgid), pPgid, sizeof(*pPgid)); + pPage->pLruNext = NULL; + pPage->pPager = NULL; + tdbPCacheAddPageToHash(pPage); + } + + return pPage; +} + +static void tdbPCachePinPage(SPage *pPage) { + SPCache *pCache; + + pCache = pPage->pCache; + if (!PAGE_IS_PINNED(pPage)) { + pPage->pLruPrev->pLruNext = pPage->pLruNext; + pPage->pLruNext->pLruPrev = pPage->pLruPrev; + pPage->pLruNext = NULL; + + pCache->nRecyclable--; + } +} + +static void tdbPCacheUnpinPage(SPage *pPage) { + SPCache *pCache; + i32 nRef; + + pCache = pPage->pCache; + + tdbPCacheLock(pCache); + + nRef = TDB_GET_PAGE_REF(pPage); + ASSERT(nRef >= 0); + if (nRef == 0) { + // Add the page to LRU list + ASSERT(pPage->pLruNext == NULL); + + pPage->pLruPrev = &(pCache->lru); + pPage->pLruNext = pCache->lru.pLruNext; + pCache->lru.pLruNext->pLruPrev = pPage; + pCache->lru.pLruNext = pPage; + } + + pCache->nRecyclable++; + + tdbPCacheUnlock(pCache); +} + +static void tdbPCacheRemovePageFromHash(SPage *pPage) { + SPCache *pCache; + SPage **ppPage; + int h; + + pCache = pPage->pCache; + h = PCACHE_PAGE_HASH(&(pPage->pgid)); + for (ppPage = &(pCache->pgHash[h % pCache->nHash]); *ppPage != pPage; ppPage = &((*ppPage)->pHashNext)) + ; + ASSERT(*ppPage == pPage); + *ppPage = pPage->pHashNext; + + pCache->nPage--; +} + +static void tdbPCacheAddPageToHash(SPage *pPage) { + SPCache *pCache; + int h; + + pCache = pPage->pCache; + h = PCACHE_PAGE_HASH(&(pPage->pgid)) % pCache->nHash; + + pPage->pHashNext = pCache->pgHash[h]; + pCache->pgHash[h] = pPage; + + pCache->nPage++; +} + +static int tdbPCacheOpenImpl(SPCache *pCache) { + SPage *pPage; + u8 *pPtr; + int tsize; + int ret; + + tdbPCacheInitLock(pCache); + + // Open the free list + pCache->nFree = 0; + pCache->pFree = NULL; + for (int i = 0; i < pCache->cacheSize; i++) { + ret = tdbPageCreate(pCache->pageSize, &pPage, tdbOsMalloc, NULL); + if (ret < 0) { + // TODO: handle error + return -1; + } + + // pPage->pgid = 0; + pPage->isAnchor = 0; + pPage->isLocalPage = 1; + pPage->pCache = pCache; + TDB_INIT_PAGE_REF(pPage); + pPage->pHashNext = NULL; + pPage->pLruNext = NULL; + pPage->pLruPrev = NULL; + pPage->pDirtyNext = NULL; + + pPage->pFreeNext = pCache->pFree; + pCache->pFree = pPage; + pCache->nFree++; + } + + // Open the hash table + pCache->nPage = 0; + pCache->nHash = pCache->cacheSize; + pCache->pgHash = (SPage **)calloc(pCache->nHash, sizeof(SPage *)); + if (pCache->pgHash == NULL) { + // TODO + return -1; + } + + // Open LRU list + pCache->nRecyclable = 0; + pCache->lru.isAnchor = 1; + pCache->lru.pLruNext = &(pCache->lru); + pCache->lru.pLruPrev = &(pCache->lru); + + return 0; +} + +int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->pageSize; } + +static void *tdbOsMalloc(void *arg, size_t size) { + void *ptr; + + ptr = malloc(size); + + return ptr; +} + +static void tdbOsFree(void *arg, void *ptr) { free(ptr); } \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c new file mode 100644 index 0000000000..df158de756 --- /dev/null +++ b/source/libs/tdb/src/db/tdbPage.c @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdbInt.h" + +typedef struct __attribute__((__packed__)) { + u8 szCell[2]; + u8 nxOffset[2]; +} SFreeCell; + +typedef struct __attribute__((__packed__)) { + u8 szCell[3]; + u8 nxOffset[3]; +} SFreeCellL; + +/* For small page */ +#define TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL) (((SFreeCell *)(PCELL))->szCell) +#define TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) (((SFreeCell *)(PCELL))->nxOffset) + +#define TDB_SPAGE_FREE_CELL_SIZE(PCELL) ((u16 *)TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL))[0] +#define TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL) ((u16 *)TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL))[0] + +#define TDB_SPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE) (TDB_SPAGE_FREE_CELL_SIZE(PCELL) = (SIZE)) +#define TDB_SPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET) (TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL) = (OFFSET)) + +/* For large page */ +#define TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL) (((SFreeCellL *)(PCELL))->szCell) +#define TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) (((SFreeCellL *)(PCELL))->nxOffset) + +#define TDB_LPAGE_FREE_CELL_SIZE(PCELL) TDB_GET_U24(TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL)) +#define TDB_LPAGE_FREE_CELL_NXOFFSET(PCELL) TDB_GET_U24(TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL)) + +#define TDB_LPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE) TDB_PUT_U24(TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL), SIZE) +#define TDB_LPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET) TDB_PUT_U24(TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL), OFFSET) + +/* For page */ +#define TDB_PAGE_FREE_CELL_SIZE_PTR(PPAGE, PCELL) \ + (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL) : TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL)) +#define TDB_PAGE_FREE_CELL_NXOFFSET_PTR(PPAGE, PCELL) \ + (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) : TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL)) + +#define TDB_PAGE_FREE_CELL_SIZE(PPAGE, PCELL) \ + (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_SIZE(PCELL) : TDB_SPAGE_FREE_CELL_SIZE(PCELL)) +#define TDB_PAGE_FREE_CELL_NXOFFSET(PPAGE, PCELL) \ + (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_NXOFFSET(PCELL) : TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL)) + +#define TDB_PAGE_FREE_CELL_SIZE_SET(PPAGE, PCELL, SIZE) \ + do { \ + if (TDB_IS_LARGE_PAGE(PPAGE)) { \ + TDB_LPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE); \ + } else { \ + TDB_SPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE); \ + } \ + } while (0) +#define TDB_PAGE_FREE_CELL_NXOFFSET_SET(PPAGE, PCELL, OFFSET) \ + do { \ + if (TDB_IS_LARGE_PAGE(PPAGE)) { \ + TDB_LPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET); \ + } else { \ + TDB_SPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET); \ + } \ + } while (0) + +static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell); +static int tdbPageDefragment(SPage *pPage); + +int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg) { + SPage *pPage; + u8 *ptr; + int size; + + ASSERT(TDB_IS_PGSIZE_VLD(pageSize)); + + *ppPage = NULL; + size = pageSize + sizeof(*pPage); + + ptr = (u8 *)((*xMalloc)(arg, size)); + if (pPage == NULL) { + return -1; + } + + memset(ptr, 0, size); + pPage = (SPage *)(ptr + pageSize); + + pPage->pData = ptr; + pPage->pageSize = pageSize; + if (pageSize < 65536) { + pPage->szOffset = 2; + pPage->szPageHdr = sizeof(SPageHdr); + pPage->szFreeCell = sizeof(SFreeCell); + } else { + pPage->szOffset = 3; + pPage->szPageHdr = sizeof(SPageHdrL); + pPage->szFreeCell = sizeof(SFreeCellL); + } + TDB_INIT_PAGE_LOCK(pPage); + + /* TODO */ + + *ppPage = pPage; + return 0; +} + +int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) { + u8 *ptr; + + ptr = pPage->pData; + (*xFree)(arg, ptr); + + return 0; +} + +int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { + int ret; + SCell *pTarget; + u8 *pTmp; + int j; + + if (pPage->nOverflow || szCell + pPage->szOffset > pPage->nFree) { + // TODO: need to figure out if pCell may be used by outside of this function + j = pPage->nOverflow++; + + pPage->apOvfl[j] = pCell; + pPage->aiOvfl[j] = idx; + } else { + ret = tdbPageAllocate(pPage, szCell, &pTarget); + if (ret < 0) { + return -1; + } + + memcpy(pTarget, pCell, szCell); + pTmp = pPage->pCellIdx + idx * pPage->szOffset; + memmove(pTmp + pPage->szOffset, pTmp, pPage->pFreeStart - pTmp - pPage->szOffset); + TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pTarget - pPage->pData); + TDB_PAGE_NCELLS_SET(pPage, TDB_PAGE_NCELLS(pPage) + 1); + } + + return 0; +} + +int tdbPageDropCell(SPage *pPage, int idx) { + // TODO + return 0; +} + +static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) { + SCell *pCell; + SFreeCell *pFreeCell; + u8 *pOffset; + int ret; + + ASSERT(pPage->nFree > size + pPage->szOffset); + + pCell = NULL; + *ppCell = NULL; + + // 1. Try to allocate from the free space area + if (pPage->pFreeEnd - pPage->pFreeStart > size + pPage->szOffset) { + pPage->pFreeEnd -= size; + pPage->pFreeStart += pPage->szOffset; + pCell = pPage->pFreeEnd; + } + + // 2. Try to allocate from the page free list + if ((pCell == NULL) && (pPage->pFreeEnd - pPage->pFreeStart >= pPage->szOffset) && TDB_PAGE_FCELL(pPage)) { + int szCell; + int nxOffset; + + pCell = pPage->pData + TDB_PAGE_FCELL(pPage); + pOffset = TDB_IS_LARGE_PAGE(pPage) ? ((SPageHdrL *)(pPage->pPageHdr))[0].fCell + : (u8 *)&(((SPageHdr *)(pPage->pPageHdr))[0].fCell); + szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); + nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); + + for (;;) { + // Find a cell + if (szCell >= size) { + if (szCell - size >= pPage->szFreeCell) { + SCell *pTmpCell = pCell + size; + + TDB_PAGE_FREE_CELL_SIZE_SET(pPage, pTmpCell, szCell - size); + TDB_PAGE_FREE_CELL_NXOFFSET_SET(pPage, pTmpCell, nxOffset); + // TODO: *pOffset = pTmpCell - pPage->pData; + } else { + TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_NFREE(pPage) + szCell - size); + // TODO: *pOffset = nxOffset; + } + break; + } + + // Not find a cell yet + if (nxOffset > 0) { + pCell = pPage->pData + nxOffset; + pOffset = TDB_PAGE_FREE_CELL_NXOFFSET_PTR(pPage, pCell); + szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); + nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); + continue; + } else { + pCell = NULL; + break; + } + } + + if (pCell) { + pPage->pFreeStart = pPage->pFreeStart + pPage->szOffset; + } + } + + // 3. Try to dfragment and allocate again + if (pCell == NULL) { + ret = tdbPageDefragment(pPage); + if (ret < 0) { + return -1; + } + + ASSERT(pPage->pFreeEnd - pPage->pFreeStart > size + pPage->szOffset); + ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart); + + // Allocate from the free space area again + pPage->pFreeEnd -= size; + pPage->pFreeStart += pPage->szOffset; + pCell = pPage->pFreeEnd; + } + + ASSERT(pCell != NULL); + + pPage->nFree = pPage->nFree - size - pPage->szOffset; + *ppCell = pCell; + return 0; +} + +static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int size) { + // TODO + return 0; +} + +static int tdbPageDefragment(SPage *pPage) { + // TODO + ASSERT(0); + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c new file mode 100644 index 0000000000..ac0bc15e1d --- /dev/null +++ b/source/libs/tdb/src/db/tdbPager.c @@ -0,0 +1,328 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdbInt.h" + +struct SPager { + char *dbFileName; + char *jFileName; + int pageSize; + uint8_t fid[TDB_FILE_ID_LEN]; + int fd; + int jfd; + SPCache *pCache; + SPgno dbFileSize; + SPgno dbOrigSize; + int nDirty; + SPage *pDirty; + SPage *pDirtyTail; + u8 inTran; +}; + +typedef struct __attribute__((__packed__)) { + u8 hdrString[16]; + u16 pageSize; + SPgno freePage; + u32 nFreePages; + u8 reserved[102]; +} SFileHdr; + +TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct"); + +#define TDB_PAGE_INITIALIZED(pPage) ((pPage)->pPager != NULL) + +static int tdbPagerReadPage(SPager *pPager, SPage *pPage); +static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno); +static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg); + +int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { + uint8_t *pPtr; + SPager *pPager; + int fsize; + int zsize; + int ret; + + *ppPager = NULL; + + fsize = strlen(fileName); + zsize = sizeof(*pPager) /* SPager */ + + fsize + 1 /* dbFileName */ + + fsize + 8 + 1; /* jFileName */ + pPtr = (uint8_t *)calloc(1, zsize); + if (pPtr == NULL) { + return -1; + } + + pPager = (SPager *)pPtr; + pPtr += sizeof(*pPager); + // pPager->dbFileName + pPager->dbFileName = (char *)pPtr; + memcpy(pPager->dbFileName, fileName, fsize); + pPager->dbFileName[fsize] = '\0'; + pPtr += fsize + 1; + // pPager->jFileName + pPager->jFileName = (char *)pPtr; + memcpy(pPager->jFileName, fileName, fsize); + memcpy(pPager->jFileName + fsize, "-journal", 8); + pPager->jFileName[fsize + 8] = '\0'; + // pPager->pCache + pPager->pCache = pCache; + + pPager->fd = open(pPager->dbFileName, O_RDWR | O_CREAT, 0755); + if (pPager->fd < 0) { + return -1; + } + + ret = tdbGnrtFileID(pPager->dbFileName, pPager->fid, false); + if (ret < 0) { + return -1; + } + + pPager->jfd = -1; + pPager->pageSize = tdbPCacheGetPageSize(pCache); + + *ppPager = pPager; + return 0; +} + +int tdbPagerClose(SPager *pPager) { + // TODO + return 0; +} + +int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate) { + SPgno pgno; + SPage *pPage; + int ret; + + { + // TODO: try to search the main DB to get the page number + pgno = 0; + } + + // if (pgno == 0 && toCreate) { + // ret = tdbPagerAllocPage(pPager, &pPage, &pgno); + // if (ret < 0) { + // return -1; + // } + + // // TODO: Need to zero the page + + // ret = tdbPagerWrite(pPager, pPage); + // if (ret < 0) { + // return -1; + // } + // } + + *ppgno = pgno; + return 0; +} + +int tdbPagerWrite(SPager *pPager, SPage *pPage) { + int ret; + + if (pPager->inTran == 0) { + ret = tdbPagerBegin(pPager); + if (ret < 0) { + return -1; + } + } + + if (pPage->isDirty == 0) { + pPage->isDirty = 1; + // TODO: add the page to the dirty list + + // TODO: write the page to the journal + if (1 /*actually load from the file*/) { + } + } + return 0; +} + +int tdbPagerBegin(SPager *pPager) { + if (pPager->inTran) { + return 0; + } + + // Open the journal + pPager->jfd = open(pPager->jFileName, O_RDWR | O_CREAT, 0755); + if (pPager->jfd < 0) { + return -1; + } + + // TODO: write the size of the file + + pPager->inTran = 1; + + return 0; +} + +int tdbPagerCommit(SPager *pPager) { + // TODO + return 0; +} + +static int tdbPagerReadPage(SPager *pPager, SPage *pPage) { + i64 offset; + int ret; + + ASSERT(memcmp(pPager->fid, pPage->pgid.fileid, TDB_FILE_ID_LEN) == 0); + + offset = (pPage->pgid.pgno - 1) * (i64)(pPager->pageSize); + ret = tdbPRead(pPager->fd, pPage->pData, pPager->pageSize, offset); + if (ret < 0) { + // TODO: handle error + return -1; + } + return 0; +} + +int tdbPagerGetPageSize(SPager *pPager) { return pPager->pageSize; } + +int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg) { + SPage *pPage; + SPgid pgid; + int ret; + + // Fetch a page container from the page cache + memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN); + pgid.pgno = pgno; + pPage = tdbPCacheFetch(pPager->pCache, &pgid, 1); + if (pPage == NULL) { + return -1; + } + + // Initialize the page if need + if (!TDB_PAGE_INITIALIZED(pPage)) { + ret = tdbPagerInitPage(pPager, pPage, initPage, arg); + if (ret < 0) { + return -1; + } + } + + ASSERT(TDB_PAGE_INITIALIZED(pPage)); + ASSERT(pPage->pPager == pPager); + + *ppPage = pPage; + return 0; +} + +int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg) { + int ret; + SPage *pPage; + SPgid pgid; + + // Allocate a page number + ret = tdbPagerAllocPage(pPager, ppgno); + if (ret < 0) { + return -1; + } + + ASSERT(*ppgno != 0); + + // Fetch a page container from the page cache + memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN); + pgid.pgno = *ppgno; + pPage = tdbPCacheFetch(pPager->pCache, &pgid, 1); + if (pPage == NULL) { + return -1; + } + + ASSERT(!TDB_PAGE_INITIALIZED(pPage)); + + // Initialize the page if need + ret = tdbPagerInitPage(pPager, pPage, initPage, arg); + if (ret < 0) { + return -1; + } + + ASSERT(TDB_PAGE_INITIALIZED(pPage)); + ASSERT(pPage->pPager == pPager); + + *ppPage = pPage; + return 0; +} + +static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno) { + // TODO: Allocate a page from the free list + return 0; +} + +static int tdbPagerAllocNewPage(SPager *pPager, SPgno *ppgno) { + *ppgno = ++pPager->dbFileSize; + return 0; +} + +static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno) { + int ret; + + *ppgno = 0; + + // Try to allocate from the free list of the pager + ret = tdbPagerAllocFreePage(pPager, ppgno); + if (ret < 0) { + return -1; + } + + if (*ppgno != 0) return 0; + + // Allocate the page by extending the pager + ret = tdbPagerAllocNewPage(pPager, ppgno); + if (ret < 0) { + return -1; + } + + ASSERT(*ppgno != 0); + + return 0; +} + +static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg) { + int ret; + int lcode; + int nLoops; + + lcode = TDB_TRY_LOCK_PAGE(pPage); + if (lcode == P_LOCK_SUCC) { + if (TDB_PAGE_INITIALIZED(pPage)) { + TDB_UNLOCK_PAGE(pPage); + return 0; + } + + ret = (*initPage)(pPage, arg); + if (ret < 0) { + TDB_UNLOCK_PAGE(pPage); + return -1; + } + + pPage->pPager = pPager; + + TDB_UNLOCK_PAGE(pPage); + } else if (lcode == P_LOCK_BUSY) { + nLoops = 0; + for (;;) { + if (TDB_PAGE_INITIALIZED(pPage)) break; + nLoops++; + if (nLoops > 1000) { + sched_yield(); + nLoops = 0; + } + } + } else { + return -1; + } + + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPgCache.c b/source/libs/tdb/src/db/tdbPgCache.c deleted file mode 100644 index ef3f79410a..0000000000 --- a/source/libs/tdb/src/db/tdbPgCache.c +++ /dev/null @@ -1,576 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "tdbInt.h" - -typedef TD_DLIST(SPage) SPgList; -struct SPgCache { - TENV * pEnv; // TENV containing this page cache - pgsz_t pgsize; - int32_t npage; - SPage **pages; - SPgList freeList; - SPgList lru; - struct { - int32_t nbucket; - SPgList *buckets; - } pght; // page hash table -}; - -static void pgCachePinPage(SPage *pPage); -static void pgCacheUnpinPage(SPage *pPage); - -int pgCacheOpen(SPgCache **ppPgCache, TENV *pEnv) { - SPgCache *pPgCache; - SPage * pPage; - void * pData; - pgsz_t pgSize; - cachesz_t cacheSize; - int32_t npage; - int32_t nbucket; - size_t msize; - - *ppPgCache = NULL; - pgSize = tdbEnvGetPageSize(pEnv); - cacheSize = tdbEnvGetCacheSize(pEnv); - npage = cacheSize / pgSize; - nbucket = npage; - msize = sizeof(*pPgCache) + sizeof(SPage *) * npage + sizeof(SPgList) * nbucket; - - // Allocate the handle - pPgCache = (SPgCache *)calloc(1, msize); - if (pPgCache == NULL) { - return -1; - } - - // Init the handle - pPgCache->pEnv = pEnv; - pPgCache->pgsize = pgSize; - pPgCache->npage = npage; - pPgCache->pages = (SPage **)(&pPgCache[1]); - pPgCache->pght.nbucket = nbucket; - pPgCache->pght.buckets = (SPgList *)(&(pPgCache->pages[npage])); - - TD_DLIST_INIT(&(pPgCache->freeList)); - - for (int32_t i = 0; i < npage; i++) { - pData = malloc(pgSize + sizeof(SPage)); - if (pData == NULL) { - return -1; - // TODO: handle error - } - - pPage = POINTER_SHIFT(pData, pgSize); - - pPage->pgid = TDB_IVLD_PGID; - pPage->frameid = i; - pPage->pData = pData; - - // add current page to the page cache - pPgCache->pages[i] = pPage; - TD_DLIST_APPEND_WITH_FIELD(&(pPgCache->freeList), pPage, freeNode); - } - -#if 0 - for (int32_t i = 0; i < nbucket; i++) { - TD_DLIST_INIT(pPgCache->pght.buckets + i); - } -#endif - - *ppPgCache = pPgCache; - return 0; -} - -int pgCacheClose(SPgCache *pPgCache) { - SPage *pPage; - if (pPgCache) { - for (int32_t i = 0; i < pPgCache->npage; i++) { - pPage = pPgCache->pages[i]; - tfree(pPage->pData); - } - - free(pPgCache); - } - - return 0; -} - -#define PG_CACHE_HASH(fileid, pgno) (((uint64_t *)(fileid))[0] + ((uint64_t *)(fileid))[1] + ((uint64_t *)(fileid))[2] + (pgno)) - -SPage *pgCacheFetch(SPgCache *pPgCache, pgid_t pgid) { - SPage * pPage; - SPgFile *pPgFile; - SPgList *pBucket; - - // 1. Search the page hash table SPgCache.pght - pBucket = pPgCache->pght.buckets + (PG_CACHE_HASH(pgid.fileid, pgid.pgno) % pPgCache->pght.nbucket); - pPage = TD_DLIST_HEAD(pBucket); - while (pPage && tdbCmprPgId(&(pPage->pgid), &pgid)) { - pPage = TD_DLIST_NODE_NEXT_WITH_FIELD(pPage, pghtNode); - } - - if (pPage) { - // Page is found, pin the page and return the page - pgCachePinPage(pPage); - return pPage; - } - - // 2. Check the free list - pPage = TD_DLIST_HEAD(&(pPgCache->freeList)); - if (pPage) { - TD_DLIST_POP_WITH_FIELD(&(pPgCache->freeList), pPage, freeNode); - pgCachePinPage(pPage); - return pPage; - } - - // 3. Try to recycle a page from the LRU list - pPage = TD_DLIST_HEAD(&(pPgCache->lru)); - if (pPage) { - TD_DLIST_POP_WITH_FIELD(&(pPgCache->lru), pPage, lruNode); - // TODO: remove from the hash table - pgCachePinPage(pPage); - return pPage; - } - - // 4. If a memory allocator is set, try to allocate from the allocator (TODO) - - return NULL; -} - -int pgCacheRelease(SPage *pPage) { - // TODO - return 0; -} - -static void pgCachePinPage(SPage *pPage) { - // TODO -} - -static void pgCacheUnpinPage(SPage *pPage) { - // TODO -} - -#if 0 -// Exposed handle -typedef struct TDB_MPOOL TDB_MPOOL; -typedef struct TDB_MPFILE TDB_MPFILE; - -typedef TD_DLIST_NODE(pg_t) pg_free_dlist_node_t, pg_hash_dlist_node_t; -typedef struct pg_t { - SRWLatch rwLatch; - frame_id_t frameid; - pgid_t pgid; - uint8_t dirty; - uint8_t rbit; - int32_t pinRef; - pg_free_dlist_node_t free; - pg_hash_dlist_node_t hash; - void * p; -} pg_t; - -typedef TD_DLIST(pg_t) pg_list_t; -typedef struct { - SRWLatch latch; - TD_DLIST(TDB_MPFILE); -} mpf_bucket_t; -struct TDB_MPOOL { - int64_t cachesize; - pgsz_t pgsize; - int32_t npages; - pg_t * pages; - pg_list_t freeList; - frame_id_t clockHand; - struct { - int32_t nbucket; - pg_list_t *hashtab; - } pgtab; // page table, hash - struct { -#define MPF_HASH_BUCKETS 16 - mpf_bucket_t buckets[MPF_HASH_BUCKETS]; - } mpfht; // MPF hash table. MPFs using this MP will be put in this hash table -}; - -#define MP_PAGE_AT(mp, idx) (mp)->pages[idx] - -typedef TD_DLIST_NODE(TDB_MPFILE) td_mpf_dlist_node_t; -struct TDB_MPFILE { - char * fname; // file name - int fd; // fd - uint8_t fileid[TDB_FILE_ID_LEN]; // file ID - TDB_MPOOL * mp; // underlying memory pool - td_mpf_dlist_node_t node; -}; - -/*=================================================== Exposed apis ==================================================*/ -// TDB_MPOOL -int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsz_t pgsize); -int tdbMPoolClose(TDB_MPOOL *mp); -int tdbMPoolSync(TDB_MPOOL *mp); - -// TDB_MPFILE -int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); -int tdbMPoolFileClose(TDB_MPFILE *mpf); -int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr); -int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr); -int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); -int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); -int tdbMPoolFileSync(TDB_MPFILE *mpf); - -static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); -static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); -static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid); -static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p); -static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p); -static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp); - -int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsz_t pgsize) { - TDB_MPOOL *mp = NULL; - size_t tsize; - pg_t * pagep; - - // check parameters - if (!TDB_IS_PGSIZE_VLD(pgsize)) { - tdbError("invalid page size"); - return -1; - } - - // allocate handle - mp = (TDB_MPOOL *)calloc(1, sizeof(*mp)); - if (mp == NULL) { - tdbError("failed to malloc memory pool handle"); - goto _err; - } - - // initialize the handle - mp->cachesize = cachesize; - mp->pgsize = pgsize; - mp->npages = cachesize / pgsize; - mp->clockHand = 0; - - TD_DLIST_INIT(&mp->freeList); - - mp->pages = (pg_t *)calloc(mp->npages, sizeof(pg_t)); - if (mp->pages == NULL) { - tdbError("failed to malloc memory pool pages"); - goto _err; - } - - for (frame_id_t i = 0; i < mp->npages; i++) { - mp->pages[i].p = malloc(pgsize); - if (mp->pages[i].p == NULL) { - goto _err; - } - - taosInitRWLatch(&mp->pages[i].rwLatch); - mp->pages[i].frameid = i; - mp->pages[i].pgid = TDB_IVLD_PGID; - - // add new page to the free list - TD_DLIST_APPEND_WITH_FIELD(&(mp->freeList), &(mp->pages[i]), free); - } - -#define PGTAB_FACTOR 1.0 - mp->pgtab.nbucket = mp->npages / PGTAB_FACTOR; - mp->pgtab.hashtab = (pg_list_t *)calloc(mp->pgtab.nbucket, sizeof(pg_list_t)); - if (mp->pgtab.hashtab == NULL) { - tdbError("failed to malloc memory pool hash table"); - goto _err; - } - - // return - *mpp = mp; - return 0; - -_err: - tdbMPoolClose(mp); - *mpp = NULL; - return -1; -} - -int tdbMPoolClose(TDB_MPOOL *mp) { - if (mp) { - tfree(mp->pgtab.hashtab); - if (mp->pages) { - for (int i = 0; i < mp->npages; i++) { - tfree(mp->pages[i].p); - } - - free(mp->pages); - } - - free(mp); - } - return 0; -} - -int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) { - TDB_MPFILE *mpf; - - if ((mpf = (TDB_MPFILE *)calloc(1, sizeof(*mpf))) == NULL) { - return -1; - } - - mpf->fd = -1; - - if ((mpf->fname = strdup(fname)) == NULL) { - goto _err; - } - - if ((mpf->fd = open(fname, O_CREAT | O_RDWR, 0755)) < 0) { - goto _err; - } - - if (tdbGnrtFileID(fname, mpf->fileid, false) < 0) { - goto _err; - } - - // Register current MPF to MP - tdbMPoolRegFile(mp, mpf); - - *mpfp = mpf; - return 0; - -_err: - tdbMPoolFileClose(mpf); - *mpfp = NULL; - return -1; -} - -int tdbMPoolFileClose(TDB_MPFILE *mpf) { - if (mpf) { - if (mpf->fd > 0) { - close(mpf->fd); - } - tfree(mpf->fname); - free(mpf); - } - return 0; -} - -#define MPF_GET_PAGE_BUCKETID(fileid, pgno, nbuckets) \ - ({ \ - uint64_t *tmp = (uint64_t *)fileid; \ - (tmp[0] + tmp[1] + tmp[2] + (pgno)) % (nbuckets); \ - }) - -int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr) { - // TODO - return 0; -} - -int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr) { - // TODO - return 0; -} - -int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { - pg_t * pagep; - TDB_MPOOL *mp; - pg_list_t *pglist; - - mp = mpf->mp; - - // check if the page already in pool - pglist = mp->pgtab.hashtab + MPF_GET_PAGE_BUCKETID(mpf->fileid, pgno, mp->pgtab.nbucket); - pagep = TD_DLIST_HEAD(pglist); - while (pagep) { - if (memcmp(mpf->fileid, pagep->pgid.fileid, TDB_FILE_ID_LEN) == 0 && pgno == pagep->pgid.pgno) { - break; - } - - pagep = TD_DLIST_NODE_NEXT_WITH_FIELD(pagep, hash); - } - - if (pagep) { - // page is found - // todo: pin the page and return - *(void **)addr = pagep->p; - return 0; - } - - // page not found - pagep = TD_DLIST_HEAD(&mp->freeList); - if (pagep) { - // has free page - TD_DLIST_POP_WITH_FIELD(&(mp->freeList), pagep, free); - } else { - // no free page available - tdbMPoolClockEvictPage(mp, &pagep); - if (pagep) { - if (pagep->dirty) { - // TODO: Handle dirty page eviction - } - } - } - - if (pagep == NULL) { - // no available container page - return -1; - } - - // load page from the disk if a container page is available - // TODO: load the page from the disk - if (tdbMPoolFileReadPage(mpf, pgno, pagep->p) < 0) { - return -1; - } - - memcpy(pagep->pgid.fileid, mpf->fileid, TDB_FILE_ID_LEN); - pagep->pgid.pgno = pgno; - pagep->dirty = 0; - pagep->pinRef = 1; - - // add current page to page table - TD_DLIST_APPEND_WITH_FIELD(pglist, pagep, hash); - - return 0; -} - -int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { - // TODO - return 0; -} - -#define MPF_GET_BUCKETID(fileid) \ - ({ \ - uint64_t *tmp = (uint64_t *)fileid; \ - (tmp[0] + tmp[1] + tmp[2]) % MPF_HASH_BUCKETS; \ - }) - -static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { - mpf_bucket_t *bktp; - - bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid); - - taosWLockLatch(&(bktp->latch)); - - TD_DLIST_APPEND_WITH_FIELD(bktp, mpf, node); - - taosWUnLockLatch(&(bktp->latch)); - - mpf->mp = mp; -} - -static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid) { - TDB_MPFILE * mpf = NULL; - mpf_bucket_t *bktp; - - bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(fileid); - - taosRLockLatch(&(bktp->latch)); - - mpf = TD_DLIST_HEAD(bktp); - while (mpf) { - if (memcmp(fileid, mpf->fileid, TDB_FILE_ID_LEN) == 0) { - break; - } - - mpf = TD_DLIST_NODE_NEXT_WITH_FIELD(mpf, node); - } - - taosRUnLockLatch(&(bktp->latch)); - - return mpf; -} - -static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { - mpf_bucket_t *bktp; - TDB_MPFILE * tmpf; - - if (mpf->mp == NULL) return; - - ASSERT(mpf->mp == mp); - - bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid); - - taosWLockLatch(&(bktp->latch)); - - tmpf = TD_DLIST_HEAD(bktp); - - while (tmpf) { - if (memcmp(mpf->fileid, tmpf->fileid, TDB_FILE_ID_LEN) == 0) { - TD_DLIST_POP_WITH_FIELD(bktp, tmpf, node); - break; - } - - tmpf = TD_DLIST_NODE_NEXT_WITH_FIELD(tmpf, node); - } - - taosWUnLockLatch(&(bktp->latch)); - - ASSERT(tmpf == mpf); -} - -static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p) { - pgsz_t pgsize; - TDB_MPOOL *mp; - off_t offset; - size_t rsize; - - mp = mpf->mp; - pgsize = mp->pgsize; - offset = pgno * pgsize; - - // TODO: use loop to read all data - rsize = pread(mpf->fd, p, pgsize, offset); - // TODO: error handle - - return 0; -} - -static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p) { - pgsz_t pgsize; - TDB_MPOOL *mp; - off_t offset; - - mp = mpf->mp; - pgsize = mp->pgsize; - offset = pgno * pgsize; - - lseek(mpf->fd, offset, SEEK_SET); - // TODO: handle error - - write(mpf->fd, p, pgsize); - // TODO: handle error - - return 0; -} - -static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp) { - pg_t * pagep; - frame_id_t och; - - *pagepp = NULL; - och = mp->clockHand; - - do { - pagep = mp->pages + mp->clockHand; - mp->clockHand = (mp->clockHand + 1) % mp->npages; - - if (pagep->pinRef == 0) { - if (pagep->rbit == 1) { - pagep->rbit = 0; - } else { - break; - } - } - - if (mp->clockHand == och) { - return; - } - } while (1); - - *pagepp = pagep; -} - -#endif \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPgFile.c b/source/libs/tdb/src/db/tdbPgFile.c deleted file mode 100644 index 12f062ebf7..0000000000 --- a/source/libs/tdb/src/db/tdbPgFile.c +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdbInt.h" - -typedef struct SPage1 { - char magic[64]; - pgno_t mdbRootPgno; // master DB root page number - pgno_t freePgno; // free list page number - uint32_t nFree; // number of free pages -} SPage1; - -typedef struct SFreePage { - /* TODO */ -} SFreePage; - -TDB_STATIC_ASSERT(sizeof(SPage1) <= TDB_MIN_PGSIZE, "TDB Page1 definition too large"); - -static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData); - -int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) { - SPgFile * pPgFile; - SPgCache *pPgCache; - size_t fnameLen; - pgno_t fsize; - - *ppPgFile = NULL; - - // create the handle - fnameLen = strlen(fname); - pPgFile = (SPgFile *)calloc(1, sizeof(*pPgFile) + fnameLen + 1); - if (pPgFile == NULL) { - return -1; - } - - ASSERT(pEnv != NULL); - - // init the handle - pPgFile->fname = (char *)(&(pPgFile[1])); - memcpy(pPgFile->fname, fname, fnameLen); - pPgFile->fname[fnameLen] = '\0'; - pPgFile->pFile = NULL; - - pPgFile->pFile = taosOpenFile(fname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); - if (pPgFile->pFile == NULL) { - // TODO: handle error - return -1; - } - - tdbGnrtFileID(fname, pPgFile->fileid, false); - tdbGetFileSize(fname, tdbEnvGetPageSize(pEnv), &fsize); - - pPgFile->fsize = fsize; - pPgFile->lsize = fsize; - - if (pPgFile->fsize == 0) { - // A created file - pgno_t pgno; - pgid_t pgid; - - pgFileAllocatePage(pPgFile, &pgno); - - ASSERT(pgno == 1); - - memcpy(pgid.fileid, pPgFile->fileid, TDB_FILE_ID_LEN); - pgid.pgno = pgno; - - pgCacheFetch(pPgCache, pgid); - // Need to allocate the first page as a description page - } else { - // An existing file - } - - /* TODO: other open operations */ - - // add the page file to the environment - tdbEnvRgstPageFile(pEnv, pPgFile); - pPgFile->pEnv = pEnv; - - *ppPgFile = pPgFile; - return 0; -} - -int pgFileClose(SPgFile *pPgFile) { - if (pPgFile) { - if (pPgFile->pFile != NULL) { - taosCloseFile(&pPgFile->pFile); - } - - tfree(pPgFile->fname); - free(pPgFile); - } - - return 0; -} - -SPage *pgFileFetch(SPgFile *pPgFile, pgno_t pgno) { - SPgCache *pPgCache; - SPage * pPage; - pgid_t pgid; - - // 1. Fetch from the page cache - // pgCacheFetch(pPgCache, pgid); - - // 2. If only get a page frame, no content, maybe - // need to load from the file - if (1 /*page not initialized*/) { - if (pgno < pPgFile->fsize) { - // load the page content from the disk - // ?? How about the freed pages ?? - } else { - // zero the page, make the page as a empty - // page with zero records. - } - } - -#if 0 - pPgCache = pPgFile->pPgCache; - pPage = NULL; - memcpy(pgid.fileid, pPgFile->fileid, TDB_FILE_ID_LEN); - pgid.pgno = pgno; - - if (pgno > pPgFile->pgFileSize) { - // TODO - } else { - pPage = pgCacheFetch(pPgCache, pgid); - if (1 /*Page is cached, no need to load from file*/) { - return pPage; - } else { - // TODO: handle error - if (pgFileRead(pPgFile, pgno, (void *)pPage) < 0) { - // todoerr - } - return pPage; - } - } -#endif - - return pPage; -} - -int pgFileRelease(SPage *pPage) { - pgCacheRelease(pPage); - return 0; -} - -int pgFileWrite(SPage *pPage) { - // TODO - return 0; -} - -int pgFileAllocatePage(SPgFile *pPgFile, pgno_t *pPgno) { - pgno_t pgno; - SPage1 * pPage1; - SPgCache *pPgCache; - pgid_t pgid; - SPage * pPage; - - if (pPgFile->lsize == 0) { - pgno = ++(pPgFile->lsize); - } else { - if (0) { - // TODO: allocate from the free list - pPage = pgCacheFetch(pPgCache, pgid); - - if (pPage1->nFree > 0) { - // TODO - } else { - pgno = ++(pPgFile->lsize); - } - } else { - pgno = ++(pPgFile->lsize); - } - } - - *pPgno = pgno; - return 0; -} - -static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData) { - pgsz_t pgSize; - ssize_t rsize; - uint8_t *pTData; - size_t szToRead; - -#if 0 - - // pgSize = ; (TODO) - pTData = pData; - szToRead = pgSize; - for (; szToRead > 0;) { - rsize = pread(pPgFile->pFile, pTData, szToRead, pgno * pgSize); - if (rsize < 0) { - if (errno == EINTR) { - continue; - } else { - return -1; - } - } else if (rsize == 0) { - return -1; - } - - szToRead -= rsize; - pTData += rsize; - } -#endif - - return 0; -} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbUtil.c b/source/libs/tdb/src/db/tdbUtil.c index 237a39e47d..b5373be9dd 100644 --- a/source/libs/tdb/src/db/tdbUtil.c +++ b/source/libs/tdb/src/db/tdbUtil.c @@ -51,7 +51,8 @@ int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) { // return access(pathname, flags); // } -int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize) { +int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize) { + struct stat st; int ret; int64_t file_size = 0; ret = taosStatFile(fname, &file_size, NULL); @@ -63,4 +64,29 @@ int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize) { *pSize = file_size / pgSize; return 0; +} + +int tdbPRead(int fd, void *pData, int count, i64 offset) { + void *pBuf; + int nbytes; + i64 ioffset; + int iread; + + pBuf = pData; + nbytes = count; + ioffset = offset; + while (nbytes > 0) { + iread = pread(fd, pBuf, nbytes, ioffset); + if (iread < 0) { + /* TODO */ + } else if (iread == 0) { + return (count - iread); + } + + nbytes = nbytes - iread; + pBuf = (void *)((u8 *)pBuf + iread); + ioffset += iread; + } + + return count; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h index 94af3331ba..c1fe77c22e 100644 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -23,20 +23,21 @@ extern "C" { typedef struct SBTree SBTree; typedef struct SBtCursor SBtCursor; -// SBTree -int btreeOpen(SBTree **ppBt, SPgFile *pPgFile); -int btreeClose(SBTree *pBt); - -// SBtCursor -int btreeCursorOpen(SBtCursor *pBtCur, SBTree *pBt); -int btreeCursorClose(SBtCursor *pBtCur); -int btreeCursorMoveTo(SBtCursor *pBtCur, int kLen, const void *pKey); -int btreeCursorNext(SBtCursor *pBtCur); - -struct SBTree { - pgno_t root; +struct SBtCursor { + SBTree *pBt; + i8 iPage; + SPage *pPage; + int idx; + int idxStack[BTREE_MAX_DEPTH + 1]; + SPage *pgStack[BTREE_MAX_DEPTH + 1]; + void *pBuf; }; +int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SBTree **ppBt); +int tdbBtreeClose(SBTree *pBt); +int tdbBtreeCursor(SBtCursor *pCur, SBTree *pBt); +int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *pVal, int vLen); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/impl/inc/dndWorker.h b/source/libs/tdb/src/inc/tdbDb.h similarity index 63% rename from source/dnode/mgmt/impl/inc/dndWorker.h rename to source/libs/tdb/src/inc/tdbDb.h index 9c037d91c7..06ea74a83e 100644 --- a/source/dnode/mgmt/impl/inc/dndWorker.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -13,21 +13,22 @@ * along with this program. If not, see . */ -#ifndef _TD_DND_WORKER_H_ -#define _TD_DND_WORKER_H_ +#ifndef _TD_TDB_DB_H_ +#define _TD_TDB_DB_H_ #ifdef __cplusplus extern "C" { #endif -#include "dndEnv.h" -int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum, - int32_t maxNum, void *queueFp); -void dndCleanupWorker(SDnodeWorker *pWorker); -int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pCont, int32_t contLen); +typedef struct STDb STDb; + +int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDb **ppDb); +int tdbDbClose(STDb *pDb); +int tdbDbDrop(STDb *pDb); +int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); #ifdef __cplusplus } #endif -#endif /*_TD_DND_WORKER_H_*/ \ No newline at end of file +#endif /*_TD_TDB_DB_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbEnv.h b/source/libs/tdb/src/inc/tdbEnv.h index 6cb5c7a2cd..959b963a07 100644 --- a/source/libs/tdb/src/inc/tdbEnv.h +++ b/source/libs/tdb/src/inc/tdbEnv.h @@ -20,11 +20,17 @@ extern "C" { #endif -const char* tdbEnvGetRootDir(TENV* pEnv); -SPgFile* tdbEnvGetPageFile(TENV* pEnv, const uint8_t fileid[]); -SPgCache* tdbEnvGetPgCache(TENV* pEnv); -int tdbEnvRgstPageFile(TENV* pEnv, SPgFile* pPgFile); -int tdbEnvRgstDB(TENV* pEnv, TDB* pDb); +typedef struct STEnv { + char * rootDir; + char * jfname; + int jfd; + SPCache *pCache; +} STEnv; + +int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, STEnv **ppEnv); +int tdbEnvClose(STEnv *pEnv); + +SPager *tdbEnvGetPager(STEnv *pEnv, const char *fname); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index ac42e15002..5902a6a716 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -19,16 +19,33 @@ #include "tlist.h" #include "tlockfree.h" -#include "tdb.h" +// #include "tdb.h" #ifdef __cplusplus extern "C" { #endif -typedef struct SPgFile SPgFile; +typedef int8_t i8; +typedef int16_t i16; +typedef int32_t i32; +typedef int64_t i64; +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; -// pgno_t -typedef int32_t pgno_t; +// p must be u8 * +#define TDB_GET_U24(p) ((p)[0] * 65536 + *(u16 *)((p) + 1)) +#define TDB_PUT_U24(p, v) \ + do { \ + int tv = (v); \ + (p)[2] = tv & 0xff; \ + (p)[1] = (tv >> 8) & 0xff; \ + (p)[0] = (tv >> 16) & 0xff; \ + } while (0) + +// SPgno +typedef u32 SPgno; #define TDB_IVLD_PGNO ((pgno_t)0) // fileid @@ -37,8 +54,8 @@ typedef int32_t pgno_t; // pgid_t typedef struct { uint8_t fileid[TDB_FILE_ID_LEN]; - pgno_t pgno; -} pgid_t; + SPgno pgno; +} pgid_t, SPgid; #define TDB_IVLD_PGID (pgid_t){0, TDB_IVLD_PGNO}; @@ -61,18 +78,14 @@ static FORCE_INLINE int tdbCmprPgId(const void *p1, const void *p2) { } } -// framd_id_t -typedef int32_t frame_id_t; +#define TDB_IS_SAME_PAGE(pPgid1, pPgid2) (tdbCmprPgId(pPgid1, pPgid2) == 0) // pgsz_t -#define TDB_MIN_PGSIZE 512 -#define TDB_MAX_PGSIZE 65536 -#define TDB_DEFAULT_PGSIZE 4096 +#define TDB_MIN_PGSIZE 512 // 512B +#define TDB_MAX_PGSIZE 16777216 // 16M +#define TDB_DEFAULT_PGSIZE 4096 #define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE)) -// pgoff_t -typedef pgsz_t pgoff_t; - // cache #define TDB_DEFAULT_CACHE_SIZE (256 * 4096) // 1M @@ -100,7 +113,7 @@ typedef TD_DLIST_NODE(SPgFile) SPgFileListNode; } \ } while (0) -#define TDB_VARIANT_LEN (int)-1 +#define TDB_VARIANT_LEN ((int)-1) // page payload format // + + [key] + [value] @@ -115,18 +128,40 @@ typedef TD_DLIST_NODE(SPgFile) SPgFileListNode; /* TODO */ \ } while (0) +typedef int (*FKeyComparator)(const void *pKey1, int kLen1, const void *pKey2, int kLen2); + #define TDB_JOURNAL_NAME "tdb.journal" +#define TDB_FILENAME_LEN 128 + +#define TDB_DEFAULT_FANOUT 6 + +#define BTREE_MAX_DEPTH 20 + +#define TDB_FLAG_IS(flags, flag) ((flags) == (flag)) +#define TDB_FLAG_HAS(flags, flag) (((flags) & (flag)) != 0) +#define TDB_FLAG_NO(flags, flag) ((flags) & (flag) == 0) +#define TDB_FLAG_ADD(flags, flag) ((flags) |= (flag)) +#define TDB_FLAG_REMOVE(flags, flag) ((flags) &= (~(flag))) + +typedef struct SPager SPager; +typedef struct SPCache SPCache; +typedef struct SPage SPage; + #include "tdbUtil.h" +#include "tdbPCache.h" + +#include "tdbPager.h" + #include "tdbBtree.h" -#include "tdbPgCache.h" - -#include "tdbPgFile.h" - #include "tdbEnv.h" +#include "tdbDb.h" + +#include "tdbPage.h" + #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/src/inc/tdbPgCache.h b/source/libs/tdb/src/inc/tdbPCache.h similarity index 56% rename from source/libs/tdb/src/inc/tdbPgCache.h rename to source/libs/tdb/src/inc/tdbPCache.h index c25ef27c10..ff4f1acbb6 100644 --- a/source/libs/tdb/src/inc/tdbPgCache.h +++ b/source/libs/tdb/src/inc/tdbPCache.h @@ -20,26 +20,25 @@ extern "C" { #endif -typedef struct SPgCache SPgCache; -typedef struct SPage SPage; +#define TDB_PCACHE_PAGE \ + u8 isAnchor; \ + u8 isLocalPage; \ + u8 isDirty; \ + i32 nRef; \ + SPCache *pCache; \ + SPage *pFreeNext; \ + SPage *pHashNext; \ + SPage *pLruNext; \ + SPage *pLruPrev; \ + SPage *pDirtyNext; \ + SPager *pPager; \ + SPgid pgid; -// SPgCache -int pgCacheOpen(SPgCache **ppPgCache, TENV *pEnv); -int pgCacheClose(SPgCache *pPgCache); - -SPage *pgCacheFetch(SPgCache *pPgCache, pgid_t pgid); -int pgCacheRelease(SPage *pPage); - -// SPage -typedef TD_DLIST_NODE(SPage) SPgListNode; -struct SPage { - pgid_t pgid; // page id - frame_id_t frameid; // frame id - uint8_t * pData; // real data - SPgListNode freeNode; // for SPgCache.freeList - SPgListNode pghtNode; // for pght - SPgListNode lruNode; // for LRU -}; +int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache); +int tdbPCacheClose(SPCache *pCache); +SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage); +void tdbPCacheRelease(SPage *pPage); +int tdbPCacheGetPageSize(SPCache *pCache); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h new file mode 100644 index 0000000000..8a51c331b6 --- /dev/null +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TDB_PAGE_H_ +#define _TDB_PAGE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef u8 SCell; + +// PAGE APIS implemented +typedef struct { + int szOffset; + int szPageHdr; + int szFreeCell; + // flags + u16 (*getFlags)(SPage *); + void (*setFlags)(SPage *, u16); + // cell number + int (*getCellNum)(SPage *); + void (*setCellNum)(SPage *, int); + // cell content offset + int (*getCellBody)(SPage *); + void (*setCellBody)(SPage *, int); + // first free cell offset (0 means no free cells) + int (*getCellFree)(SPage *); + void (*setCellFree)(SPage *, int); + // total free bytes + int (*getFreeBytes)(SPage *); + void (*setFreeBytes)(SPage *, int); + // cell offset at idx + int (*getCellOffset)(SPage *, int); + void (*setCellOffset)(SPage *, int, int); +} SPageMethods; + +// Page footer +typedef struct __attribute__((__packed__)) { + u8 cksm[4]; +} SPageFtr; + +struct SPage { + TdThreadSpinlock lock; + u8 *pData; + int pageSize; + SPageMethods *pPageMethods; + // Fields below used by pager and am + u8 szAmHdr; + u8 *pPageHdr; + u8 *pAmHdr; + u8 *pCellIdx; + u8 *pFreeStart; + u8 *pFreeEnd; + SPageFtr *pPageFtr; + int kLen; // key length of the page, -1 for unknown + int vLen; // value length of the page, -1 for unknown + int nFree; + int maxLocal; + int minLocal; + int nOverflow; + SCell *apOvfl[4]; + int aiOvfl[4]; + // Fields used by SPCache + TDB_PCACHE_PAGE +}; + +/* For page */ +#define TDB_PAGE_FLAGS(pPage) (*(pPage)->pPageMethods->getFlags)(pPage) +#define TDB_PAGE_NCELLS(pPage) (*(pPage)->pPageMethods->getCellNum)(pPage) +#define TDB_PAGE_CCELLS(pPage) (*(pPage)->pPageMethods->getCellBody)(pPage) +#define TDB_PAGE_FCELL(pPage) (*(pPage)->pPageMethods->getCellFree)(pPage) +#define TDB_PAGE_NFREE(pPage) (*(pPage)->pPageMethods->getFreeBytes)(pPage) +#define TDB_PAGE_CELL_OFFSET_AT(pPage, idx) (*(pPage)->pPageMethods->getCellOffset)(pPage, idx) + +#define TDB_PAGE_FLAGS_SET(pPage, FLAGS) (*(pPage)->pPageMethods->setFlags)(pPage, FLAGS) +#define TDB_PAGE_NCELLS_SET(pPage, NCELLS) (*(pPage)->pPageMethods->setCellNum)(pPage, NCELLS) +#define TDB_PAGE_CCELLS_SET(pPage, CCELLS) (*(pPage)->pPageMethods->setCellBody)(pPage, CCELLS) +#define TDB_PAGE_FCELL_SET(pPage, FCELL) (*(pPage)->pPageMethods->setCellFree)(pPage, FCELL) +#define TDB_PAGE_NFREE_SET(pPage, NFREE) (*(pPage)->pPageMethods->setFreeBytes)(pPage, NFREE) +#define TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, OFFSET) (*(pPage)->pPageMethods->setCellOffset)(pPage, idx, OFFSET) + +#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) + +#define TDB_PAGE_CELL_AT(pPage, idx) ((pPage)->pData + TDB_PAGE_CELL_OFFSET_AT(pPage, idx)) + +// For page lock +#define P_LOCK_SUCC 0 +#define P_LOCK_BUSY 1 +#define P_LOCK_FAIL -1 + +#define TDB_INIT_PAGE_LOCK(pPage) taosThreadSpinInit(&((pPage)->lock), 0) +#define TDB_DESTROY_PAGE_LOCK(pPage) taosThreadSpinDestroy(&((pPage)->lock)) +#define TDB_LOCK_PAGE(pPage) taosThreadSpinLock(&((pPage)->lock)) +#define TDB_UNLOCK_PAGE(pPage) taosThreadSpinUnlock(&((pPage)->lock)) +#define TDB_TRY_LOCK_PAGE(pPage) \ + ({ \ + int ret; \ + if (pthread_spin_trylock(&((pPage)->lock)) == 0) { \ + ret = P_LOCK_SUCC; \ + } else if (errno == EBUSY) { \ + ret = P_LOCK_BUSY; \ + } else { \ + ret = P_LOCK_FAIL; \ + } \ + ret; \ + }) + +// APIs +int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); +int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); +int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); +int tdbPageDropCell(SPage *pPage, int idx); + +#ifdef __cplusplus +} +#endif + +#endif /*_TDB_PAGE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbPager.h b/source/libs/tdb/src/inc/tdbPager.h new file mode 100644 index 0000000000..e4ed8552fd --- /dev/null +++ b/source/libs/tdb/src/inc/tdbPager.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TDB_PAGER_H_ +#define _TDB_PAGER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager); +int tdbPagerClose(SPager *pPager); +int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate); +int tdbPagerWrite(SPager *pPager, SPage *pPage); +int tdbPagerBegin(SPager *pPager); +int tdbPagerCommit(SPager *pPager); +int tdbPagerGetPageSize(SPager *pPager); +int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg); +int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TDB_PAGER_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbPgFile.h b/source/libs/tdb/src/inc/tdbPgFile.h deleted file mode 100644 index 0a8c277479..0000000000 --- a/source/libs/tdb/src/inc/tdbPgFile.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_PAGE_FILE_H_ -#define _TD_PAGE_FILE_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#pragma pack (push,1) -typedef struct { - char hdrInfo[16]; // info string - pgsz_t szPage; // page size of current file - int32_t cno; // commit number counter - pgno_t freePgno; // freelist page number - uint8_t resv[100]; // reserved space -} SPgFileHdr; -#pragma pack(pop) - -#define TDB_PG_FILE_HDR_SIZE 128 - -TDB_STATIC_ASSERT(sizeof(SPgFileHdr) == TDB_PG_FILE_HDR_SIZE, "Page file header size if not 128"); - -struct SPgFile { - TENV * pEnv; // env containing this page file - char * fname; // backend file name - uint8_t fileid[TDB_FILE_ID_LEN]; // file id - pgno_t lsize; // page file logical size (for count) - pgno_t fsize; // real file size on disk (for rollback) - TdFilePtr pFile; - SPgFileListNode envHash; - SPgFileListNode envPgfList; -}; - -int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv); -int pgFileClose(SPgFile *pPgFile); - -SPage *pgFileFetch(SPgFile *pPgFile, pgno_t pgno); -int pgFileRelease(SPage *pPage); - -int pgFileWrite(SPage *pPage); -int pgFileAllocatePage(SPgFile *pPgFile, pgno_t *pPgno); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_PAGE_FILE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index ca05790f77..8aaded933a 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -35,7 +35,48 @@ int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique); // #define TDB_W_OK 0x4 // int tdbCheckFileAccess(const char *pathname, int mode); -int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize); +int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize); + +int tdbPRead(int fd, void *pData, int count, i64 offset); + +static inline int tdbPutVarInt(u8 *p, int v) { + int n = 0; + + for (;;) { + if (v <= 0x7f) { + p[n++] = v; + break; + } + + p[n++] = (v & 0x7f) | 0x80; + v >>= 7; + } + + ASSERT(n < 6); + + return n; +} + +static inline int tdbGetVarInt(const u8 *p, int *v) { + int n = 0; + int tv = 0; + + for (;;) { + if (p[n] <= 0x7f) { + tv = (tv << 7) | p[n]; + n++; + break; + } + + tv = (tv << 7) | (p[n] & 0x7f); + n++; + } + + ASSERT(n < 6); + + *v = tv; + return n; +} #ifdef __cplusplus } diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c new file mode 100644 index 0000000000..4ec3a895e7 --- /dev/null +++ b/source/libs/tdb/src/page/tdbPage.c @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdbInt.h" + +extern SPageMethods pageMethods; +extern SPageMethods pageLargeMethods; + +typedef struct __attribute__((__packed__)) { + u16 szCell; + u16 nxOffset; +} SFreeCell; + +static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell); +static int tdbPageDefragment(SPage *pPage); + +int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg) { + SPage *pPage; + u8 *ptr; + int size; + + ASSERT(TDB_IS_PGSIZE_VLD(pageSize)); + + *ppPage = NULL; + size = pageSize + sizeof(*pPage); + + ptr = (u8 *)((*xMalloc)(arg, size)); + if (pPage == NULL) { + return -1; + } + + memset(ptr, 0, size); + pPage = (SPage *)(ptr + pageSize); + + pPage->pData = ptr; + pPage->pageSize = pageSize; + if (pageSize < 65536) { + pPage->pPageMethods = &pageMethods; + } else { + pPage->pPageMethods = &pageLargeMethods; + } + TDB_INIT_PAGE_LOCK(pPage); + + /* TODO */ + + *ppPage = pPage; + return 0; +} + +int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) { + u8 *ptr; + + ptr = pPage->pData; + (*xFree)(arg, ptr); + + return 0; +} + +int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { + int ret; + SCell *pTarget; + u8 *pTmp; + int j; + + if (pPage->nOverflow || szCell + TDB_PAGE_OFFSET_SIZE(pPage) > pPage->nFree) { + // TODO: need to figure out if pCell may be used by outside of this function + j = pPage->nOverflow++; + + pPage->apOvfl[j] = pCell; + pPage->aiOvfl[j] = idx; + } else { + ret = tdbPageAllocate(pPage, szCell, &pTarget); + if (ret < 0) { + return -1; + } + + memcpy(pTarget, pCell, szCell); + pTmp = pPage->pCellIdx + idx * TDB_PAGE_OFFSET_SIZE(pPage); + memmove(pTmp + TDB_PAGE_OFFSET_SIZE(pPage), pTmp, pPage->pFreeStart - pTmp - TDB_PAGE_OFFSET_SIZE(pPage)); + TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pTarget - pPage->pData); + TDB_PAGE_NCELLS_SET(pPage, TDB_PAGE_NCELLS(pPage) + 1); + } + + return 0; +} + +int tdbPageDropCell(SPage *pPage, int idx) { + // TODO + return 0; +} + +static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) { + SCell *pCell; + SFreeCell *pFreeCell; + u8 *pOffset; + int ret; + + ASSERT(pPage->nFree > size + TDB_PAGE_OFFSET_SIZE(pPage)); + + pCell = NULL; + *ppCell = NULL; + + // 1. Try to allocate from the free space area + if (pPage->pFreeEnd - pPage->pFreeStart > size + TDB_PAGE_OFFSET_SIZE(pPage)) { + pPage->pFreeEnd -= size; + pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); + pCell = pPage->pFreeEnd; + } + + // 2. Try to allocate from the page free list + if ((pCell == NULL) && (pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) && + TDB_PAGE_FCELL(pPage)) { +#if 0 + int szCell; + int nxOffset; + + pCell = pPage->pData + TDB_PAGE_FCELL(pPage); + pOffset = TDB_IS_LARGE_PAGE(pPage) ? ((SPageHdrL *)(pPage->pPageHdr))[0].fCell + : (u8 *)&(((SPageHdr *)(pPage->pPageHdr))[0].fCell); + szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); + nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); + + for (;;) { + // Find a cell + if (szCell >= size) { + if (szCell - size >= pPage->szFreeCell) { + SCell *pTmpCell = pCell + size; + + TDB_PAGE_FREE_CELL_SIZE_SET(pPage, pTmpCell, szCell - size); + TDB_PAGE_FREE_CELL_NXOFFSET_SET(pPage, pTmpCell, nxOffset); + // TODO: *pOffset = pTmpCell - pPage->pData; + } else { + TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_NFREE(pPage) + szCell - size); + // TODO: *pOffset = nxOffset; + } + break; + } + + // Not find a cell yet + if (nxOffset > 0) { + pCell = pPage->pData + nxOffset; + pOffset = TDB_PAGE_FREE_CELL_NXOFFSET_PTR(pPage, pCell); + szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); + nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); + continue; + } else { + pCell = NULL; + break; + } + } + + if (pCell) { + pPage->pFreeStart = pPage->pFreeStart + pPage->szOffset; + } +#endif + } + + // 3. Try to dfragment and allocate again + if (pCell == NULL) { + ret = tdbPageDefragment(pPage); + if (ret < 0) { + return -1; + } + + ASSERT(pPage->pFreeEnd - pPage->pFreeStart > size + TDB_PAGE_OFFSET_SIZE(pPage)); + ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart); + + // Allocate from the free space area again + pPage->pFreeEnd -= size; + pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); + pCell = pPage->pFreeEnd; + } + + ASSERT(pCell != NULL); + + pPage->nFree = pPage->nFree - size - TDB_PAGE_OFFSET_SIZE(pPage); + *ppCell = pCell; + return 0; +} + +static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int size) { + // TODO + return 0; +} + +static int tdbPageDefragment(SPage *pPage) { + // TODO + ASSERT(0); + return 0; +} + +/* ---------------------------------------------------------------------------------------------------------- */ +typedef struct __attribute__((__packed__)) { + u16 flags; + u16 cellNum; + u16 cellBody; + u16 cellFree; + u16 nFree; +} SPageHdr; + +// flags +static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].flags; } +static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdr *)(pPage->pPageHdr))[0].flags = flags; } + +// cellNum +static inline int getPageCellNum(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellNum; } +static inline void setPageCellNum(SPage *pPage, int cellNum) { + ASSERT(cellNum < 65536); + ((SPageHdr *)(pPage->pPageHdr))[0].cellNum = (u16)cellNum; +} + +// cellBody +static inline int getPageCellBody(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellBody; } +static inline void setPageCellBody(SPage *pPage, int cellBody) { + ASSERT(cellBody < 65536); + ((SPageHdr *)(pPage->pPageHdr))[0].cellBody = (u16)cellBody; +} + +// cellFree +static inline int getPageCellFree(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellFree; } +static inline void setPageCellFree(SPage *pPage, int cellFree) { + ASSERT(cellFree < 65536); + ((SPageHdr *)(pPage->pPageHdr))[0].cellFree = (u16)cellFree; +} + +// nFree +static inline int getPageNFree(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].nFree; } +static inline void setPageNFree(SPage *pPage, int nFree) { + ASSERT(nFree < 65536); + ((SPageHdr *)(pPage->pPageHdr))[0].nFree = (u16)nFree; +} + +// cell offset +static inline int getPageCellOffset(SPage *pPage, int idx) { + ASSERT(idx >= 0 && idx < getPageCellNum(pPage)); + return ((u16 *)pPage->pCellIdx)[idx]; +} + +static inline void setPageCellOffset(SPage *pPage, int idx, int offset) { + ASSERT(offset < 65536); + ((u16 *)pPage->pCellIdx)[idx] = (u16)offset; +} + +SPageMethods pageMethods = { + 2, // szOffset + sizeof(SPageHdr), // szPageHdr + sizeof(SFreeCell), // szFreeCell + getPageFlags, // getPageFlags + setPageFlags, // setFlagsp + getPageCellNum, // getCellNum + setPageCellNum, // setCellNum + getPageCellBody, // getCellBody + setPageCellBody, // setCellBody + getPageCellFree, // getCellFree + setPageCellFree, // setCellFree + getPageNFree, // getFreeBytes + setPageNFree, // setFreeBytes + getPageCellOffset, // getCellOffset + setPageCellOffset // setCellOffset +}; \ No newline at end of file diff --git a/source/libs/tdb/src/page/tdbPageL.c b/source/libs/tdb/src/page/tdbPageL.c new file mode 100644 index 0000000000..e7c60118d2 --- /dev/null +++ b/source/libs/tdb/src/page/tdbPageL.c @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdbInt.h" + +typedef struct __attribute__((__packed__)) { + u16 flags; + u8 cellNum[3]; + u8 cellBody[3]; + u8 cellFree[3]; + u8 nFree[3]; +} SPageHdrL; + +typedef struct __attribute__((__packed__)) { + u8 szCell[3]; + u8 nxOffset[3]; +} SFreeCellL; + +// flags +static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdrL *)(pPage->pPageHdr))[0].flags; } +static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdrL *)(pPage->pPageHdr))[0].flags = flags; } + +// cellNum +static inline int getPageCellNum(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum); } +static inline void setPageCellNum(SPage *pPage, int cellNum) { + TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum, cellNum); +} + +// cellBody +static inline int getPageCellBody(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellBody); } +static inline void setPageCellBody(SPage *pPage, int cellBody) { + TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellBody, cellBody); +} + +// cellFree +static inline int getPageCellFree(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellFree); } +static inline void setPageCellFree(SPage *pPage, int cellFree) { + TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellFree, cellFree); +} + +// nFree +static inline int getPageNFree(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].nFree); } +static inline void setPageNFree(SPage *pPage, int nFree) { + TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].nFree, nFree); +} + +// cell offset +static inline int getPageCellOffset(SPage *pPage, int idx) { + ASSERT(idx >= 0 && idx < getPageCellNum(pPage)); + return TDB_GET_U24(pPage->pCellIdx + 3 * idx); +} + +static inline void setPageCellOffset(SPage *pPage, int idx, int offset) { + TDB_PUT_U24(pPage->pCellIdx + 3 * idx, offset); +} + +SPageMethods pageLargeMethods = { + 3, // szOffset + sizeof(SPageHdrL), // szPageHdr + sizeof(SFreeCellL), // szFreeCell + getPageFlags, // getPageFlags + setPageFlags, // setFlagsp + getPageCellNum, // getCellNum + setPageCellNum, // setCellNum + getPageCellBody, // getCellBody + setPageCellBody, // setCellBody + getPageCellFree, // getCellFree + setPageCellFree, // setCellFree + getPageNFree, // getFreeBytes + setPageNFree, // setFreeBytes + getPageCellOffset, // getCellOffset + setPageCellOffset // setCellOffset +}; \ No newline at end of file diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index ad550c7804..c3cc922f32 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -1,68 +1,39 @@ #include -#include "tdb.h" +#include "tdbInt.h" TEST(tdb_test, simple_test) { - TENV * pEnv; - TDB * pDb1, *pDb2, *pDb3; - pgsz_t pgSize = 1024; - cachesz_t cacheSize = 10240; + int ret; + STEnv *pEnv; + STDb *pDb; - // ENV - GTEST_ASSERT_EQ(tdbEnvCreate(&pEnv, "./testtdb"), 0); + // Open Env + ret = tdbEnvOpen("tdb", 1024, 20, &pEnv); + GTEST_ASSERT_EQ(ret, 0); - GTEST_ASSERT_EQ(tdbEnvSetCache(pEnv, pgSize, cacheSize), 0); + // Create a database + ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, NULL, pEnv, &pDb); + GTEST_ASSERT_EQ(ret, 0); - GTEST_ASSERT_EQ(tdbEnvGetCacheSize(pEnv), cacheSize); + { // Insert some data + char key[64]; + char val[64]; - GTEST_ASSERT_EQ(tdbEnvGetPageSize(pEnv), pgSize); + for (int i = 1; i <= 1000; i++) { + sprintf(key, "key%d", i); + sprintf(val, "value%d", i); + ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val)); + GTEST_ASSERT_EQ(ret, 0); + } + } - GTEST_ASSERT_EQ(tdbEnvOpen(pEnv), 0); + ret = tdbDbDrop(pDb); + GTEST_ASSERT_EQ(ret, 0); -#if 1 - // DB - GTEST_ASSERT_EQ(tdbCreate(&pDb1), 0); + // Close a database + tdbDbClose(pDb); - // GTEST_ASSERT_EQ(tdbSetKeyLen(pDb1, 8), 0); - - // GTEST_ASSERT_EQ(tdbGetKeyLen(pDb1), 8); - - // GTEST_ASSERT_EQ(tdbSetValLen(pDb1, 3), 0); - - // GTEST_ASSERT_EQ(tdbGetValLen(pDb1), 3); - - // GTEST_ASSERT_EQ(tdbSetDup(pDb1, 1), 0); - - // GTEST_ASSERT_EQ(tdbGetDup(pDb1), 1); - - // GTEST_ASSERT_EQ(tdbSetCmprFunc(pDb1, NULL), 0); - - tdbEnvBeginTxn(pEnv); - - GTEST_ASSERT_EQ(tdbOpen(pDb1, "db.db", "db1", pEnv), 0); - - // char *key = "key1"; - // char *val = "value1"; - // tdbInsert(pDb1, (void *)key, strlen(key), (void *)val, strlen(val)); - - tdbEnvCommit(pEnv); - -#if 0 - // Insert - - // Query - - // Delete - - // Query -#endif - - // GTEST_ASSERT_EQ(tdbOpen(&pDb2, "db.db", "db2", pEnv), 0); - // GTEST_ASSERT_EQ(tdbOpen(&pDb3, "index.db", NULL, pEnv), 0); - // tdbClose(pDb3); - // tdbClose(pDb2); - tdbClose(pDb1); -#endif - - tdbEnvClose(pEnv); + // Close Env + ret = tdbEnvClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); } \ No newline at end of file diff --git a/source/libs/tfs/inc/tfsInt.h b/source/libs/tfs/inc/tfsInt.h index f16d0445c6..2a508cf676 100644 --- a/source/libs/tfs/inc/tfsInt.h +++ b/source/libs/tfs/inc/tfsInt.h @@ -41,7 +41,7 @@ typedef struct { } STfsDisk; typedef struct { - pthread_spinlock_t lock; + TdThreadSpinlock lock; int32_t level; int32_t nextid; // next disk id to allocate int32_t ndisk; // # of disks mounted to this tier @@ -64,7 +64,7 @@ typedef struct STfsDir { } STfsDir; typedef struct STfs { - pthread_spinlock_t lock; + TdThreadSpinlock lock; SDiskSize size; int32_t nlevel; STfsTier tiers[TFS_MAX_TIERS]; @@ -82,11 +82,11 @@ void tfsUpdateTierSize(STfsTier *pTier); int32_t tfsAllocDiskOnTier(STfsTier *pTier); void tfsPosNextId(STfsTier *pTier); -#define tfsLockTier(pTier) pthread_spin_lock(&(pTier)->lock) -#define tfsUnLockTier(pTier) pthread_spin_unlock(&(pTier)->lock) +#define tfsLockTier(pTier) taosThreadSpinLock(&(pTier)->lock) +#define tfsUnLockTier(pTier) taosThreadSpinUnlock(&(pTier)->lock) -#define tfsLock(pTfs) pthread_spin_lock(&(pTfs)->lock) -#define tfsUnLock(pTfs) pthread_spin_unlock(&(pTfs)->lock) +#define tfsLock(pTfs) taosThreadSpinLock(&(pTfs)->lock) +#define tfsUnLock(pTfs) taosThreadSpinUnlock(&(pTfs)->lock) #define TFS_TIER_AT(pTfs, level) (&(pTfs)->tiers[level]) #define TFS_DISK_AT(pTfs, did) ((pTfs)->tiers[(did).level].disks[(did).id]) diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index aee7376491..bfe7904a63 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -36,7 +36,7 @@ STfs *tfsOpen(SDiskCfg *pCfg, int32_t ndisk) { return NULL; } - if (pthread_spin_init(&pTfs->lock, 0) != 0) { + if (taosThreadSpinInit(&pTfs->lock, 0) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); tfsClose(pTfs); return NULL; @@ -85,7 +85,7 @@ void tfsClose(STfs *pTfs) { } taosHashCleanup(pTfs->hash); - pthread_spin_destroy(&pTfs->lock); + taosThreadSpinDestroy(&pTfs->lock); free(pTfs); } @@ -395,7 +395,7 @@ static int32_t tfsMount(STfs *pTfs, SDiskCfg *pCfg) { } static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { - char dirName[TSDB_FILENAME_LEN] = "\0"; + char dirName[TSDB_FILENAME_LEN] = "\0"; if (pCfg->level < 0 || pCfg->level >= TFS_MAX_TIERS) { fError("failed to mount %s to FS since invalid level %d", pCfg->dir, pCfg->level); diff --git a/source/libs/tfs/src/tfsTier.c b/source/libs/tfs/src/tfsTier.c index e4390d13d1..2ad021cbb5 100644 --- a/source/libs/tfs/src/tfsTier.c +++ b/source/libs/tfs/src/tfsTier.c @@ -19,7 +19,7 @@ int32_t tfsInitTier(STfsTier *pTier, int32_t level) { memset(pTier, 0, sizeof(STfsTier)); - if (pthread_spin_init(&pTier->lock, 0) != 0) { + if (taosThreadSpinInit(&pTier->lock, 0) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -34,7 +34,7 @@ void tfsDestroyTier(STfsTier *pTier) { } pTier->ndisk = 0; - pthread_spin_destroy(&pTier->lock); + taosThreadSpinDestroy(&pTier->lock); } STfsDisk *tfsMountDiskToTier(STfsTier *pTier, SDiskCfg *pCfg) { diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 8ea65b193d..8cfde8267d 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -14,6 +14,10 @@ */ #ifdef USE_UV +#ifdef __cplusplus +extern "C" { +#endif + #include #include "lz4.h" #include "os.h" @@ -121,24 +125,21 @@ typedef struct { } SRpcReqContext; typedef SRpcMsg STransMsg; +typedef SRpcCtx STransCtx; +typedef SRpcCtxVal STransCtxVal; typedef SRpcInfo STrans; typedef SRpcConnInfo STransHandleInfo; typedef struct { - SEpSet epSet; // ip list provided by app - void* ahandle; // handle provided by app - tmsg_t msgType; // message type - uint8_t* pCont; // content provided by app - int32_t contLen; // content length - // int32_t code; // error code - // int16_t numOfTry; // number of try for different servers - // int8_t oldInUse; // server EP inUse passed by app - // int8_t redirect; // flag to indicate redirect + SEpSet epSet; // ip list provided by app + void* ahandle; // handle provided by app + tmsg_t msgType; // message type int8_t connType; // connection type cli/srv int64_t rid; // refId returned by taosAddRef - STransMsg* pRsp; // for synchronous API - tsem_t* pSem; // for synchronous API + STransCtx appCtx; // + STransMsg* pRsp; // for synchronous API + tsem_t* pSem; // for synchronous API int hThrdIdx; char* ip; @@ -150,11 +151,12 @@ typedef struct { typedef struct { char version : 4; // RPC version - char comp : 4; // compression algorithm, 0:no compression 1:lz4 - char resflag : 2; // reserved bits - char spi : 1; // security parameter index + char comp : 2; // compression algorithm, 0:no compression 1:lz4 + char noResp : 2; // noResp bits, 0: resp, 1: resp + char persist : 2; // persist handle,0: no persit, 1: persist handle + char release : 2; char secured : 2; - char encrypt : 3; // encrypt algorithm, 0: no encryption + char spi : 2; uint32_t code; // del later uint32_t msgType; @@ -179,6 +181,9 @@ typedef struct { #pragma pack(pop) +typedef enum { Normal, Quit, Release, Register } STransMsgType; +typedef enum { ConnNormal, ConnAcquire, ConnRelease, ConnBroken } ConnStatus; + #define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) #define RPC_RESERVE_SIZE (sizeof(STranConnCtx)) @@ -223,9 +228,9 @@ typedef struct SConnBuffer { typedef void (*AsyncCB)(uv_async_t* handle); typedef struct { - void* pThrd; - queue qmsg; - pthread_mutex_t mtx; // protect qmsg; + void* pThrd; + queue qmsg; + TdThreadMutex mtx; // protect qmsg; } SAsyncItem; typedef struct { @@ -255,9 +260,10 @@ void transUnrefCliHandle(void* handle); void transReleaseCliHandle(void* handle); void transReleaseSrvHandle(void* handle); -void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg); +void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg, STransCtx* pCtx); void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg, STransMsg* pRsp); -void transSendResponse(const STransMsg* pMsg); +void transSendResponse(const STransMsg* msg); +void transRegisterMsg(const STransMsg* msg); int transGetConnInfo(void* thandle, STransHandleInfo* pInfo); void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); @@ -266,4 +272,55 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void transCloseClient(void* arg); void transCloseServer(void* arg); +void transCtxInit(STransCtx* ctx); +void transCtxCleanup(STransCtx* ctx); +void transCtxClear(STransCtx* ctx); +void transCtxMerge(STransCtx* dst, STransCtx* src); +void* transCtxDumpVal(STransCtx* ctx, int32_t key); + +// queue sending msgs +typedef struct { + SArray* q; + void (*free)(void* arg); +} STransQueue; + +/* + * init queue + * note: queue'size is small, default 1 + */ +void transQueueInit(STransQueue* queue, void (*free)(void* arg)); + +/* + * put arg into queue + * if queue'size > 1, return false; else return true + */ +bool transQueuePush(STransQueue* queue, void* arg); +/* + * pop head from queue + */ + +void* transQueuePop(STransQueue* queue); +/* + * get head from queue + */ +void* transQueueGet(STransQueue* queue); + +/* + * queue empty or not + */ + +bool transQueueEmpty(STransQueue* queue); +/* + * clear queue + */ +void transQueueClear(STransQueue* queue); +/* + * destroy queue + */ +void transQueueDestroy(STransQueue* queue); + +#ifdef __cplusplus +} +#endif + #endif diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index e739380467..ad50948a02 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -63,9 +63,6 @@ typedef struct { void (*cfp)(void* parent, SRpcMsg*, SEpSet*); int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); - bool (*pfp)(void* parent, tmsg_t msgType); - void* (*mfp)(void* parent, tmsg_t msgType); - bool (*efp)(void* parent, tmsg_t msgType); int32_t refCount; void* parent; @@ -73,7 +70,7 @@ typedef struct { void* tmrCtrl; // handle to timer SHashObj* hash; // handle returned by hash utility void* tcphandle; // returned handle from TCP initialization - pthread_mutex_t mutex; + TdThreadMutex mutex; } SRpcInfo; #endif // USE_LIBUV diff --git a/source/libs/transport/src/rpcCache.c b/source/libs/transport/src/rpcCache.c index 1db2808126..d629f6dba4 100644 --- a/source/libs/transport/src/rpcCache.c +++ b/source/libs/transport/src/rpcCache.c @@ -39,7 +39,7 @@ typedef struct { int total; int * count; int64_t keepTimer; - pthread_mutex_t mutex; + TdThreadMutex mutex; void (*cleanFp)(void *); void * tmrCtrl; void * pTimer; @@ -85,7 +85,7 @@ void *rpcOpenConnCache(int maxSessions, void (*cleanFp)(void *), void *tmrCtrl, pCache->lockedBy = calloc(sizeof(int64_t), maxSessions); taosTmrReset(rpcCleanConnCache, (int32_t)(pCache->keepTimer * 2), pCache, pCache->tmrCtrl, &pCache->pTimer); - pthread_mutex_init(&pCache->mutex, NULL); + taosThreadMutexInit(&pCache->mutex, NULL); return pCache; } @@ -96,7 +96,7 @@ void rpcCloseConnCache(void *handle) { pCache = (SConnCache *)handle; if (pCache == NULL || pCache->maxSessions == 0) return; - pthread_mutex_lock(&pCache->mutex); + taosThreadMutexLock(&pCache->mutex); taosTmrStopA(&(pCache->pTimer)); @@ -106,9 +106,9 @@ void rpcCloseConnCache(void *handle) { tfree(pCache->count); tfree(pCache->lockedBy); - pthread_mutex_unlock(&pCache->mutex); + taosThreadMutexUnlock(&pCache->mutex); - pthread_mutex_destroy(&pCache->mutex); + taosThreadMutexDestroy(&pCache->mutex); memset(pCache, 0, sizeof(SConnCache)); free(pCache); @@ -220,7 +220,7 @@ static void rpcCleanConnCache(void *handle, void *tmrId) { if (pCache == NULL || pCache->maxSessions == 0) return; if (pCache->pTimer != tmrId) return; - pthread_mutex_lock(&pCache->mutex); + taosThreadMutexLock(&pCache->mutex); uint64_t time = taosGetTimestampMs(); for (hash = 0; hash < pCache->maxSessions; ++hash) { @@ -232,7 +232,7 @@ static void rpcCleanConnCache(void *handle, void *tmrId) { // tTrace("timer, total connections in cache:%d", pCache->total); taosTmrReset(rpcCleanConnCache, (int32_t)(pCache->keepTimer * 2), pCache, pCache->tmrCtrl, &pCache->pTimer); - pthread_mutex_unlock(&pCache->mutex); + taosThreadMutexUnlock(&pCache->mutex); } static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash, uint64_t time) { diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 86dd17bb0c..98b24ac3c7 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -14,6 +14,7 @@ */ #include "lz4.h" +#include "transportInt.h" #include "os.h" #include "rpcCache.h" #include "rpcHead.h" @@ -27,13 +28,12 @@ #include "tmd5.h" #include "tmempool.h" #include "tmsg.h" -#include "transportInt.h" #include "tref.h" #include "trpc.h" #include "ttimer.h" #include "tutil.h" -static pthread_once_t tsRpcInitOnce = PTHREAD_ONCE_INIT; +static TdThreadOnce tsRpcInitOnce = PTHREAD_ONCE_INIT; int tsRpcMaxUdpSize = 15000; // bytes int tsProgressTimer = 100; @@ -72,7 +72,7 @@ typedef struct { void * tcphandle; // returned handle from TCP initialization void * udphandle; // returned handle from UDP initialization void * pCache; // connection cache - pthread_mutex_t mutex; + TdThreadMutex mutex; struct SRpcConn *connList; // connection list } SRpcInfo; @@ -143,7 +143,7 @@ typedef struct SRpcConn { static int tsRpcRefId = -1; static int32_t tsRpcNum = 0; -// static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT; +// static TdThreadOnce tsRpcInit = PTHREAD_ONCE_INIT; // server:0 client:1 tcp:2 udp:0 #define RPC_CONN_UDPS 0 @@ -223,7 +223,7 @@ static void rpcInitImp(void) { } int32_t rpcInit() { - pthread_once(&tsRpcInitOnce, rpcInitImp); + taosThreadOnce(&tsRpcInitOnce, rpcInitImp); return 0; } @@ -238,7 +238,7 @@ void rpcCleanup(void) { void *rpcOpen(const SRpcInit *pInit) { SRpcInfo *pRpc; - // pthread_once(&tsRpcInit, rpcInit); + // taosThreadOnce(&tsRpcInit, rpcInit); pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo)); if (pRpc == NULL) return NULL; @@ -307,7 +307,7 @@ void *rpcOpen(const SRpcInit *pInit) { } } - pthread_mutex_init(&pRpc->mutex, NULL); + taosThreadMutexInit(&pRpc->mutex, NULL); pRpc->tcphandle = (*taosInitConn[pRpc->connType | RPC_CONN_TCP])(0, pRpc->localPort, pRpc->label, pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc); @@ -1672,7 +1672,7 @@ static void rpcDecRef(SRpcInfo *pRpc) { taosIdPoolCleanUp(pRpc->idPool); tfree(pRpc->connList); - pthread_mutex_destroy(&pRpc->mutex); + taosThreadMutexDestroy(&pRpc->mutex); tDebug("%s rpc resources are released", pRpc->label); tfree(pRpc); diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index aac38b21e8..4b1700a10f 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -35,9 +35,9 @@ typedef struct SFdObj { } SFdObj; typedef struct SThreadObj { - pthread_t thread; + TdThread thread; SFdObj * pHead; - pthread_mutex_t mutex; + TdThreadMutex mutex; uint32_t ip; bool stop; TdEpollPtr pEpoll; @@ -65,7 +65,7 @@ typedef struct { int numOfThreads; void * shandle; SThreadObj **pThreadObj; - pthread_t thread; + TdThread thread; } SServerObj; static void * taosProcessTcpData(void *param); @@ -101,9 +101,9 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread } int code = 0; - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); // initialize parameters in case it may encounter error later for (int i = 0; i < numOfThreads; ++i) { @@ -129,7 +129,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread // initialize mutex, thread, fd which may fail for (int i = 0; i < numOfThreads; ++i) { pThreadObj = pServerObj->pThreadObj[i]; - code = pthread_mutex_init(&(pThreadObj->mutex), NULL); + code = taosThreadMutexInit(&(pThreadObj->mutex), NULL); if (code < 0) { tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno)); break; @@ -142,7 +142,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread break; } - code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); + code = taosThreadCreate(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); if (code != 0) { tError("%s failed to create TCP process data thread(%s)", label, strerror(errno)); break; @@ -155,7 +155,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread if (pServerObj->pSocketServer == NULL) code = -1; if (code == 0) { - code = pthread_create(&pServerObj->thread, &thattr, taosAcceptTcpConnection, (void *)pServerObj); + code = taosThreadCreate(&pServerObj->thread, &thattr, taosAcceptTcpConnection, (void *)pServerObj); if (code != 0) { tError("%s failed to create TCP accept thread(%s)", label, strerror(code)); } @@ -169,7 +169,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread tDebug("%s TCP server is initialized, ip:0x%x port:%hu numOfThreads:%d", label, ip, port, numOfThreads); } - pthread_attr_destroy(&thattr); + taosThreadAttrDestroy(&thattr); return (void *)pServerObj; } @@ -178,16 +178,16 @@ static void taosStopTcpThread(SThreadObj *pThreadObj) { return; } // save thread into local variable and signal thread to stop - pthread_t thread = pThreadObj->thread; + TdThread thread = pThreadObj->thread; if (!taosCheckPthreadValid(thread)) { return; } pThreadObj->stop = true; - if (taosComparePthread(thread, pthread_self())) { - pthread_detach(pthread_self()); + if (taosComparePthread(thread, taosThreadSelf())) { + pthread_detach(taosThreadSelf()); return; } - pthread_join(thread, NULL); + taosThreadJoin(thread, NULL); } void taosStopTcpServer(void *handle) { @@ -200,10 +200,10 @@ void taosStopTcpServer(void *handle) { taosShutDownSocketServerRD(pServerObj->pSocketServer); } if (taosCheckPthreadValid(pServerObj->thread)) { - if (taosComparePthread(pServerObj->thread, pthread_self())) { - pthread_detach(pthread_self()); + if (taosComparePthread(pServerObj->thread, taosThreadSelf())) { + pthread_detach(taosThreadSelf()); } else { - pthread_join(pServerObj->thread, NULL); + taosThreadJoin(pServerObj->thread, NULL); } } @@ -307,9 +307,9 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread } int code = 0; - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); for (int i = 0; i < numOfThreads; ++i) { SThreadObj *pThreadObj = (SThreadObj *)calloc(1, sizeof(SThreadObj)); @@ -318,7 +318,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread terrno = TAOS_SYSTEM_ERROR(errno); for (int j = 0; j < i; ++j) free(pClientObj->pThreadObj[j]); free(pClientObj); - pthread_attr_destroy(&thattr); + taosThreadAttrDestroy(&thattr); return NULL; } pClientObj->pThreadObj[i] = pThreadObj; @@ -333,7 +333,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread // initialize mutex, thread, fd which may fail for (int i = 0; i < numOfThreads; ++i) { SThreadObj *pThreadObj = pClientObj->pThreadObj[i]; - code = pthread_mutex_init(&(pThreadObj->mutex), NULL); + code = taosThreadMutexInit(&(pThreadObj->mutex), NULL); if (code < 0) { tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno)); break; @@ -346,7 +346,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread break; } - code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); + code = taosThreadCreate(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); if (code != 0) { tError("%s failed to create TCP process data thread(%s)", label, strerror(errno)); break; @@ -578,7 +578,7 @@ static void *taosProcessTcpData(void *param) { taosReportBrokenLink(pFdObj); } - pthread_mutex_destroy(&(pThreadObj->mutex)); + taosThreadMutexDestroy(&(pThreadObj->mutex)); tDebug("%s TCP thread exits ...", pThreadObj->label); tfree(pThreadObj); @@ -607,12 +607,12 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, TdSocketPtr pSocket) { } // notify the data process, add into the FdObj list - pthread_mutex_lock(&(pThreadObj->mutex)); + taosThreadMutexLock(&(pThreadObj->mutex)); pFdObj->next = pThreadObj->pHead; if (pThreadObj->pHead) (pThreadObj->pHead)->prev = pFdObj; pThreadObj->pHead = pFdObj; pThreadObj->numOfFds++; - pthread_mutex_unlock(&(pThreadObj->mutex)); + taosThreadMutexUnlock(&(pThreadObj->mutex)); return pFdObj; } @@ -622,10 +622,10 @@ static void taosFreeFdObj(SFdObj *pFdObj) { if (pFdObj->signature != pFdObj) return; SThreadObj *pThreadObj = pFdObj->pThreadObj; - pthread_mutex_lock(&pThreadObj->mutex); + taosThreadMutexLock(&pThreadObj->mutex); if (pFdObj->signature == NULL) { - pthread_mutex_unlock(&pThreadObj->mutex); + taosThreadMutexUnlock(&pThreadObj->mutex); return; } @@ -648,7 +648,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) { (pFdObj->next)->prev = pFdObj->prev; } - pthread_mutex_unlock(&pThreadObj->mutex); + taosThreadMutexUnlock(&pThreadObj->mutex); tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds); diff --git a/source/libs/transport/src/rpcUdp.c b/source/libs/transport/src/rpcUdp.c index 81c8d0af76..af26761603 100644 --- a/source/libs/transport/src/rpcUdp.c +++ b/source/libs/transport/src/rpcUdp.c @@ -35,7 +35,7 @@ typedef struct { uint16_t port; // peer port uint16_t localPort; // local port char label[TSDB_LABEL_LEN]; // copy from udpConnSet; - pthread_t thread; + TdThread thread; void *hash; void *shandle; // handle passed by upper layer during server initialization void *pSet; @@ -77,9 +77,9 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads pSet->threads = threads; tstrncpy(pSet->label, label, sizeof(pSet->label)); - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); int i; uint16_t ownPort; @@ -111,14 +111,14 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads pConn->index = i; pConn->pSet = pSet; - int code = pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn); + int code = taosThreadCreate(&pConn->thread, &thAttr, taosRecvUdpData, pConn); if (code != 0) { tError("%s failed to create thread to process UDP data(%s)", label, strerror(errno)); break; } } - pthread_attr_destroy(&thAttr); + taosThreadAttrDestroy(&thAttr); if (i != threads) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -146,7 +146,7 @@ void taosStopUdpConnection(void *handle) { for (int i = 0; i < pSet->threads; ++i) { pConn = pSet->udpConn + i; if (taosCheckPthreadValid(pConn->thread)) { - pthread_join(pConn->thread, NULL); + taosThreadJoin(pConn->thread, NULL); } tfree(pConn->buffer); // tTrace("%s UDP thread is closed, index:%d", pConn->label, i); diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 6d1c691b9b..4e2cb83527 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -14,10 +14,13 @@ */ #define _DEFAULT_SOURCE +#ifdef USE_UV +#include +#endif +#include "zlib.h" #include "thttp.h" #include "taoserror.h" #include "tlog.h" -#include "zlib.h" static int32_t taosBuildHttpHeader(const char* server, int32_t contLen, char* pHead, int32_t headLen, EHttpCompFlag flag) { @@ -111,7 +114,6 @@ _OVER: } #ifdef USE_UV -#include static void clientConnCb(uv_connect_t* req, int32_t status) { if (status < 0) { terrno = TAOS_SYSTEM_ERROR(status); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 2cab03f133..9d0fba4885 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -39,9 +39,6 @@ void* rpcOpen(const SRpcInit* pInit) { // register callback handle pRpc->cfp = pInit->cfp; pRpc->afp = pInit->afp; - pRpc->pfp = pInit->pfp; - pRpc->mfp = pInit->mfp; - pRpc->efp = pInit->efp; if (pInit->connType == TAOS_CONN_SERVER) { pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; @@ -121,7 +118,12 @@ void rpcCancelRequest(int64_t rid) { return; } void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); uint32_t port = pEpSet->eps[pEpSet->inUse].port; - transSendRequest(shandle, ip, port, pMsg); + transSendRequest(shandle, ip, port, pMsg, NULL); +} +void rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) { + char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); + uint32_t port = pEpSet->eps[pEpSet->inUse].port; + transSendRequest(shandle, ip, port, pMsg, pCtx); } void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) { char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); @@ -142,6 +144,7 @@ void rpcUnrefHandle(void* handle, int8_t type) { (*taosUnRefHandle[type])(handle); } +void rpcRegisterBrokenLinkArg(SRpcMsg* msg) { transRegisterMsg(msg); } void rpcReleaseHandle(void* handle, int8_t type) { assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT); (*transReleaseHandle[type])(handle); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index df35b8d67b..c0ee9b9ca5 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -17,11 +17,6 @@ #include "transComm.h" -// Normal(default): send/recv msg -// Quit: quit rpc inst -// Release: release handle to rpc inst -typedef enum { Normal, Quit, Release } SCliMsgType; - typedef struct SCliConn { T_REF_DECLARE() uv_connect_t connReq; @@ -30,13 +25,16 @@ typedef struct SCliConn { void* hostThrd; SConnBuffer readBuf; void* data; - SArray* cliMsgs; - queue conn; - uint64_t expireTime; - int hThrdIdx; - bool broken; // link broken or not + // SArray* cliMsgs; + STransQueue cliMsgs; + queue conn; + uint64_t expireTime; + int hThrdIdx; + STransCtx ctx; - int persist; // + bool broken; // link broken or not + ConnStatus status; // + int release; // 1: release // spi configure char spi; char secured; @@ -55,19 +53,19 @@ typedef struct SCliMsg { STransMsg msg; queue q; uint64_t st; - SCliMsgType type; + STransMsgType type; } SCliMsg; typedef struct SCliThrdObj { - pthread_t thread; + TdThread thread; uv_loop_t* loop; SAsyncPool* asyncPool; uv_timer_t timer; void* pool; // conn pool // msg queue - queue msg; - pthread_mutex_t msgMtx; + queue msg; + TdThreadMutex msgMtx; uint64_t nextTimeout; // next timeout void* pTransInst; // @@ -113,10 +111,12 @@ static void cliSend(SCliConn* pConn); static void cliHandleResp(SCliConn* conn); // handle except about conn static void cliHandleExcept(SCliConn* conn); + // handle req from app static void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd); static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd); +static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrdObj* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease}; static void cliSendQuit(SCliThrdObj* thrd); static void destroyUserdata(STransMsg* userdata); @@ -133,42 +133,60 @@ static void destroyThrdObj(SCliThrdObj* pThrd); #define CONN_PERSIST_TIME(para) (para * 1000 * 10) #define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) #define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label) -#define CONN_HANDLE_THREAD_QUIT(conn, thrd) \ - do { \ - if (thrd->quit) { \ - cliHandleExcept(conn); \ - goto _RETURE; \ - } \ +#define CONN_SHOULD_RELEASE(conn, head) \ + do { \ + if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ + conn->status = ConnRelease; \ + transClearBuffer(&conn->readBuf); \ + transFreeMsg(transContFromHead((char*)head)); \ + tDebug("cli conn %p receive release request, ref: %d", conn, T_REF_VAL_GET(conn)); \ + while (T_REF_VAL_GET(conn) > 1) { \ + transUnrefCliHandle(conn); \ + } \ + if (T_REF_VAL_GET(conn) == 1) { \ + SCliThrdObj* thrd = conn->hostThrd; \ + addConnToPool(thrd->pool, conn); \ + } \ + return; \ + } \ + } while (0) + +#define CONN_HANDLE_THREAD_QUIT(thrd) \ + do { \ + if (thrd->quit) { \ + return; \ + } \ } while (0) #define CONN_HANDLE_BROKEN(conn) \ do { \ if (conn->broken) { \ cliHandleExcept(conn); \ - goto _RETURE; \ + return; \ } \ - } while (0); + } while (0) #define CONN_SET_PERSIST_BY_APP(conn) \ do { \ - if (conn->persist == false) { \ - conn->persist = true; \ + if (conn->status == ConnNormal) { \ + conn->status = ConnAcquire; \ transRefCliHandle(conn); \ } \ } while (0) -#define CONN_NO_PERSIST_BY_APP(conn) ((conn)->persist == false) +#define CONN_NO_PERSIST_BY_APP(conn) ((conn)->status == ConnNormal && T_REF_VAL_GET(conn) == 1) #define REQUEST_NO_RESP(msg) ((msg)->noResp == 1) +#define REQUEST_PERSIS_HANDLE(msg) ((msg)->persistHandle == 1) +#define REQUEST_RELEASE_HANDLE(cmsg) ((cmsg)->type == Release) static void* cliWorkThread(void* arg); bool cliMaySendCachedMsg(SCliConn* conn) { - if (taosArrayGetSize(conn->cliMsgs) > 0) { + if (!transQueueEmpty(&conn->cliMsgs)) { cliSend(conn); return true; - } else { - return false; } + return false; } void cliHandleResp(SCliConn* conn) { SCliThrdObj* pThrd = conn->hostThrd; @@ -185,24 +203,21 @@ void cliHandleResp(SCliConn* conn) { transMsg.msgType = pHead->msgType; transMsg.ahandle = NULL; - SCliMsg* pMsg = NULL; - if (taosArrayGetSize(conn->cliMsgs) > 0) { - pMsg = taosArrayGetP(conn->cliMsgs, 0); - taosArrayRemove(conn->cliMsgs, 0); - } + CONN_SHOULD_RELEASE(conn, pHead); + + SCliMsg* pMsg = transQueuePop(&conn->cliMsgs); STransConnCtx* pCtx = pMsg ? pMsg->ctx : NULL; if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(conn)) { - transMsg.ahandle = pTransInst->mfp ? (*pTransInst->mfp)(pTransInst->parent, transMsg.msgType) : NULL; + transMsg.ahandle = transCtxDumpVal(&conn->ctx, transMsg.msgType); } else { transMsg.ahandle = pCtx ? pCtx->ahandle : NULL; } // buf's mem alread translated to transMsg.pCont transClearBuffer(&conn->readBuf); - if (pTransInst->pfp != NULL && (*pTransInst->pfp)(pTransInst->parent, transMsg.msgType)) { + if (!CONN_NO_PERSIST_BY_APP(conn)) { transMsg.handle = conn; - CONN_SET_PERSIST_BY_APP(conn); tDebug("%s cli conn %p ref by app", CONN_GET_INST_LABEL(conn), conn); } @@ -241,10 +256,12 @@ void cliHandleResp(SCliConn* conn) { if (!uv_is_active((uv_handle_t*)&pThrd->timer) && pTransInst->idleTime > 0) { // uv_timer_start((uv_timer_t*)&pThrd->timer, cliTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); } +_RETURN: + return; } void cliHandleExcept(SCliConn* pConn) { - if (taosArrayGetSize(pConn->cliMsgs) == 0) { + if (transQueueEmpty(&pConn->cliMsgs)) { if (pConn->broken == true || CONN_NO_PERSIST_BY_APP(pConn)) { transUnrefCliHandle(pConn); return; @@ -254,11 +271,7 @@ void cliHandleExcept(SCliConn* pConn) { STrans* pTransInst = pThrd->pTransInst; do { - SCliMsg* pMsg = NULL; - if (taosArrayGetSize(pConn->cliMsgs) > 0) { - pMsg = taosArrayGetP(pConn->cliMsgs, 0); - taosArrayRemove(pConn->cliMsgs, 0); - } + SCliMsg* pMsg = transQueuePop(&pConn->cliMsgs); STransConnCtx* pCtx = pMsg ? pMsg->ctx : NULL; @@ -268,7 +281,7 @@ void cliHandleExcept(SCliConn* pConn) { transMsg.ahandle = NULL; if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) { - transMsg.ahandle = pTransInst->mfp ? (*pTransInst->mfp)(pTransInst->parent, transMsg.msgType) : NULL; + transMsg.ahandle = transCtxDumpVal(&pConn->ctx, transMsg.msgType); } else { transMsg.ahandle = pCtx ? pCtx->ahandle : NULL; } @@ -283,7 +296,7 @@ void cliHandleExcept(SCliConn* pConn) { } destroyCmsg(pMsg); tTrace("%s cli conn %p start to destroy", CONN_GET_INST_LABEL(pConn), pConn); - } while (taosArrayGetSize(pConn->cliMsgs) > 0); + } while (!transQueueEmpty(&pConn->cliMsgs)); transUnrefCliHandle(pConn); } @@ -357,25 +370,29 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { return conn; } static void addConnToPool(void* pool, SCliConn* conn) { - char key[128] = {0}; + SCliThrdObj* thrd = conn->hostThrd; + CONN_HANDLE_THREAD_QUIT(thrd); + STrans* pTransInst = ((SCliThrdObj*)conn->hostThrd)->pTransInst; + conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); + transCtxCleanup(&conn->ctx); + transQueueClear(&conn->cliMsgs); + conn->status = ConnNormal; + + char key[128] = {0}; tstrncpy(key, conn->ip, strlen(conn->ip)); tstrncpy(key + strlen(key), (char*)(&conn->port), sizeof(conn->port)); tTrace("cli conn %p added to conn pool, read buf cap: %d", conn, conn->readBuf.cap); - STrans* pTransInst = ((SCliThrdObj*)conn->hostThrd)->pTransInst; - - conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key)); // list already create before assert(plist != NULL); QUEUE_PUSH(&plist->conn, &conn->conn); + assert(!QUEUE_IS_EMPTY(&plist->conn)); } static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SCliConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; - // avoid conn - QUEUE_REMOVE(&conn->conn); transAllocBuffer(pBuf, buf); } static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { @@ -401,6 +418,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { // ref http://docs.libuv.org/en/v1.x/stream.html?highlight=uv_read_start#c.uv_read_cb // nread might be 0, which does not indicate an error or EOF. This is equivalent to EAGAIN or EWOULDBLOCK under // read(2). + tTrace("%s cli conn %p read empty", CONN_GET_INST_LABEL(conn), conn); return; } if (nread < 0) { @@ -419,17 +437,18 @@ static SCliConn* cliCreateConn(SCliThrdObj* pThrd) { conn->writeReq.data = conn; conn->connReq.data = conn; - conn->cliMsgs = taosArrayInit(2, sizeof(void*)); + transQueueInit(&conn->cliMsgs, NULL); QUEUE_INIT(&conn->conn); conn->hostThrd = pThrd; - conn->persist = false; - conn->broken = false; + conn->status = ConnNormal; + conn->broken = 0; transRefCliHandle(conn); return conn; } static void cliDestroyConn(SCliConn* conn, bool clear) { tTrace("%s cli conn %p remove from conn pool", CONN_GET_INST_LABEL(conn), conn); + QUEUE_REMOVE(&conn->conn); if (clear) { uv_close((uv_handle_t*)conn->stream, cliDestroy); @@ -439,17 +458,18 @@ static void cliDestroy(uv_handle_t* handle) { SCliConn* conn = handle->data; free(conn->ip); free(conn->stream); - taosArrayDestroy(conn->cliMsgs); + transCtxCleanup(&conn->ctx); + transQueueDestroy(&conn->cliMsgs); tTrace("%s cli conn %p destroy successfully", CONN_GET_INST_LABEL(conn), conn); free(conn); } static bool cliHandleNoResp(SCliConn* conn) { - bool res = false; - SArray* msgs = conn->cliMsgs; - if (taosArrayGetSize(msgs) > 0) { - SCliMsg* pMsg = taosArrayGetP(msgs, 0); + bool res = false; + if (!transQueueEmpty(&conn->cliMsgs)) { + SCliMsg* pMsg = transQueueGet(&conn->cliMsgs); if (REQUEST_NO_RESP(&pMsg->msg)) { - taosArrayRemove(msgs, 0); + transQueuePop(&conn->cliMsgs); + // taosArrayRemove(msgs, 0); destroyCmsg(pMsg); res = true; } @@ -482,15 +502,19 @@ static void cliSendCb(uv_write_t* req, int status) { void cliSend(SCliConn* pConn) { CONN_HANDLE_BROKEN(pConn); - assert(taosArrayGetSize(pConn->cliMsgs) > 0); - SCliMsg* pCliMsg = taosArrayGetP(pConn->cliMsgs, 0); + // assert(taosArrayGetSize(pConn->cliMsgs) > 0); + assert(!transQueueEmpty(&pConn->cliMsgs)); + SCliMsg* pCliMsg = transQueueGet(&pConn->cliMsgs); STransConnCtx* pCtx = pCliMsg->ctx; SCliThrdObj* pThrd = pConn->hostThrd; STrans* pTransInst = pThrd->pTransInst; STransMsg* pMsg = (STransMsg*)(&pCliMsg->msg); - + if (pMsg->pCont == 0) { + pMsg->pCont = (void*)rpcMallocCont(0); + pMsg->contLen = 0; + } STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); int msgLen = transMsgLenFromCont(pMsg->contLen); @@ -513,20 +537,24 @@ void cliSend(SCliConn* pConn) { msgLen += sizeof(STransUserMsg); } - pHead->resflag = REQUEST_NO_RESP(pMsg) ? 1 : 0; + pHead->noResp = REQUEST_NO_RESP(pMsg) ? 1 : 0; + pHead->persist = REQUEST_PERSIS_HANDLE(pMsg) ? 1 : 0; pHead->msgType = pMsg->msgType; pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); + pHead->release = REQUEST_RELEASE_HANDLE(pCliMsg) ? 1 : 0; uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); tDebug("%s cli conn %p %s is send to %s:%d, local info %s:%d", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pHead->msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); + if (pHead->persist == 1) { + CONN_SET_PERSIST_BY_APP(pConn); + } + pConn->writeReq.data = pConn; uv_write(&pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb); - return; -_RETURE: return; } @@ -562,20 +590,16 @@ static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) { } static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd) { SCliConn* conn = pMsg->msg.handle; - tDebug("%s cli conn %p release to inst", CONN_GET_INST_LABEL(conn), conn); + tDebug("%s cli conn %p start to release to inst", CONN_GET_INST_LABEL(conn), conn); - while (taosArrayGetSize(conn->cliMsgs) > 0) { - SCliMsg* pMsg = taosArrayGetP(conn->cliMsgs, 0); - destroyCmsg(pMsg); - taosArrayRemove(conn->cliMsgs, 0); - } - - transDestroyBuffer(&conn->readBuf); - if (conn->persist && T_REF_VAL_GET(conn) >= 2) { - conn->persist = false; + if (T_REF_VAL_GET(conn) == 2) { transUnrefCliHandle(conn); - addConnToPool(pThrd->pool, conn); + if (!transQueuePush(&conn->cliMsgs, pMsg)) { + return; + } + cliSend(conn); } else { + // conn already broken down transUnrefCliHandle(conn); } } @@ -592,6 +616,8 @@ SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) { conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port); if (conn != NULL) { tTrace("%s cli conn %p get from conn pool", CONN_GET_INST_LABEL(conn), conn); + } else { + tTrace("not found conn in conn pool %p", pThrd->pool); } } return conn; @@ -609,16 +635,15 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { if (conn != NULL) { conn->hThrdIdx = pCtx->hThrdIdx; - if (taosArrayGetSize(conn->cliMsgs) > 0) { - taosArrayPush(conn->cliMsgs, &pMsg); + transCtxMerge(&conn->ctx, &pCtx->appCtx); + if (!transQueuePush(&conn->cliMsgs, pMsg)) { return; } - taosArrayPush(conn->cliMsgs, &pMsg); transDestroyBuffer(&conn->readBuf); cliSend(conn); } else { conn = cliCreateConn(pThrd); - taosArrayPush(conn->cliMsgs, &pMsg); + transQueuePush(&conn->cliMsgs, pMsg); conn->hThrdIdx = pCtx->hThrdIdx; conn->ip = strdup(pMsg->ctx->ip); @@ -642,9 +667,9 @@ static void cliAsyncCb(uv_async_t* handle) { // batch process to avoid to lock/unlock frequently queue wq; - pthread_mutex_lock(&item->mtx); + taosThreadMutexLock(&item->mtx); QUEUE_MOVE(&item->qmsg, &wq); - pthread_mutex_unlock(&item->mtx); + taosThreadMutexUnlock(&item->mtx); int count = 0; while (!QUEUE_IS_EMPTY(&wq)) { @@ -652,14 +677,10 @@ static void cliAsyncCb(uv_async_t* handle) { QUEUE_REMOVE(h); SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q); - - if (pMsg->type == Normal) { - cliHandleReq(pMsg, pThrd); - } else if (pMsg->type == Quit) { - cliHandleQuit(pMsg, pThrd); - } else if (pMsg->type == Release) { - cliHandleRelease(pMsg, pThrd); + if (pMsg == NULL) { + continue; } + (*cliAsyncHandle[pMsg->type])(pMsg, pThrd); count++; } if (count >= 2) { @@ -688,7 +709,7 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); pThrd->pTransInst = shandle; - int err = pthread_create(&pThrd->thread, NULL, cliWorkThread, (void*)(pThrd)); + int err = taosThreadCreate(&pThrd->thread, NULL, cliWorkThread, (void*)(pThrd)); if (err == 0) { tDebug("success to create tranport-cli thread %d", i); } @@ -717,7 +738,7 @@ static SCliThrdObj* createThrdObj() { SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj)); QUEUE_INIT(&pThrd->msg); - pthread_mutex_init(&pThrd->msgMtx, NULL); + taosThreadMutexInit(&pThrd->msgMtx, NULL); pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); uv_loop_init(pThrd->loop); @@ -737,8 +758,8 @@ static void destroyThrdObj(SCliThrdObj* pThrd) { return; } uv_stop(pThrd->loop); - pthread_join(pThrd->thread, NULL); - pthread_mutex_destroy(&pThrd->msgMtx); + taosThreadJoin(pThrd->thread, NULL); + taosThreadMutexDestroy(&pThrd->msgMtx); transDestroyAsyncPool(pThrd->asyncPool); uv_timer_stop(&pThrd->timer); @@ -802,23 +823,19 @@ void transReleaseCliHandle(void* handle) { STransMsg tmsg = {.handle = handle}; SCliMsg* cmsg = calloc(1, sizeof(SCliMsg)); - cmsg->type = Release; cmsg->msg = tmsg; + cmsg->type = Release; transSendAsync(thrd->asyncPool, &cmsg->q); } -void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg) { +void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg, STransCtx* ctx) { STrans* pTransInst = (STrans*)shandle; int index = CONN_HOST_THREAD_INDEX((SCliConn*)pMsg->handle); if (index == -1) { index = cliRBChoseIdx(pTransInst); } - int32_t flen = 0; - if (transCompressMsg(pMsg->pCont, pMsg->contLen, &flen)) { - // imp later - } - tDebug("send request at thread:%d %p", index, pMsg); + STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx)); pCtx->ahandle = pMsg->ahandle; pCtx->msgType = pMsg->msgType; @@ -826,17 +843,23 @@ void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* p pCtx->port = port; pCtx->hThrdIdx = index; + if (ctx != NULL) { + pCtx->appCtx = *ctx; + } assert(pTransInst->connType == TAOS_CONN_CLIENT); - // atomic or not SCliMsg* cliMsg = calloc(1, sizeof(SCliMsg)); cliMsg->ctx = pCtx; cliMsg->msg = *pMsg; cliMsg->st = taosGetTimestampUs(); + cliMsg->type = Normal; SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[index]; + + tDebug("send request at thread:%d %p, dst: %s:%d", index, pMsg, ip, port); transSendAsync(thrd->asyncPool, &(cliMsg->q)); } + void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pReq, STransMsg* pRsp) { STrans* pTransInst = (STrans*)shandle; int index = CONN_HOST_THREAD_INDEX(pReq->handle); @@ -858,6 +881,7 @@ void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pReq cliMsg->ctx = pCtx; cliMsg->msg = *pReq; cliMsg->st = taosGetTimestampUs(); + cliMsg->type = Normal; SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[index]; transSendAsync(thrd->asyncPool, &(cliMsg->q)); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 7123593a33..209475ca05 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -155,9 +155,9 @@ bool transReadComplete(SConnBuffer* connBuf) { } return false; } -int transPackMsg(STransMsgHead* msgHead, bool sercured, bool auth) {return 0;} +int transPackMsg(STransMsgHead* msgHead, bool sercured, bool auth) { return 0; } -int transUnpackMsg(STransMsgHead* msgHead) {return 0;} +int transUnpackMsg(STransMsgHead* msgHead) { return 0; } int transDestroyBuffer(SConnBuffer* buf) { if (buf->cap > 0) { tfree(buf->buf); @@ -186,7 +186,7 @@ SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb) SAsyncItem* item = calloc(1, sizeof(SAsyncItem)); item->pThrd = arg; QUEUE_INIT(&item->qmsg); - pthread_mutex_init(&item->mtx, NULL); + taosThreadMutexInit(&item->mtx, NULL); async->data = item; } @@ -197,7 +197,7 @@ void transDestroyAsyncPool(SAsyncPool* pool) { uv_async_t* async = &(pool->asyncs[i]); SAsyncItem* item = async->data; - pthread_mutex_destroy(&item->mtx); + taosThreadMutexDestroy(&item->mtx); free(item); } free(pool->asyncs); @@ -214,9 +214,9 @@ int transSendAsync(SAsyncPool* pool, queue* q) { SAsyncItem* item = async->data; int64_t st = taosGetTimestampUs(); - pthread_mutex_lock(&item->mtx); + taosThreadMutexLock(&item->mtx); QUEUE_PUSH(&item->qmsg, q); - pthread_mutex_unlock(&item->mtx); + taosThreadMutexUnlock(&item->mtx); int64_t el = taosGetTimestampUs() - st; if (el > 50) { // tInfo("lock and unlock cost: %d", (int)el); @@ -224,4 +224,101 @@ int transSendAsync(SAsyncPool* pool, queue* q) { return uv_async_send(async); } +void transCtxInit(STransCtx* ctx) { + // init transCtx + ctx->args = taosHashInit(2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UINT), true, HASH_NO_LOCK); +} +void transCtxCleanup(STransCtx* ctx) { + if (ctx->args == NULL) { + return; + } + + STransCtxVal* iter = taosHashIterate(ctx->args, NULL); + while (iter) { + iter->free(iter->val); + iter = taosHashIterate(ctx->args, iter); + } + taosHashCleanup(ctx->args); +} + +void transCtxMerge(STransCtx* dst, STransCtx* src) { + if (dst->args == NULL) { + dst->args = src->args; + src->args = NULL; + return; + } + void* key = NULL; + size_t klen = 0; + void* iter = taosHashIterate(src->args, NULL); + while (iter) { + STransCtxVal* sVal = (STransCtxVal*)iter; + key = taosHashGetKey(sVal, &klen); + + STransCtxVal* dVal = taosHashGet(dst->args, key, klen); + if (dVal) { + dVal->free(dVal->val); + } + taosHashPut(dst->args, key, klen, sVal, sizeof(*sVal)); + iter = taosHashIterate(src->args, iter); + } + taosHashCleanup(src->args); +} +void* transCtxDumpVal(STransCtx* ctx, int32_t key) { + if (ctx->args == NULL) { + return NULL; + } + STransCtxVal* cVal = taosHashGet(ctx->args, (const void*)&key, sizeof(key)); + if (cVal == NULL) { + return NULL; + } + char* ret = calloc(1, cVal->len); + memcpy(ret, (char*)cVal->val, cVal->len); + return (void*)ret; +} + +void transQueueInit(STransQueue* queue, void (*free)(void* arg)) { + queue->q = taosArrayInit(2, sizeof(void*)); + queue->free = free; +} +bool transQueuePush(STransQueue* queue, void* arg) { + taosArrayPush(queue->q, &arg); + if (taosArrayGetSize(queue->q) > 1) { + return false; + } + return true; +} +void* transQueuePop(STransQueue* queue) { + if (taosArrayGetSize(queue->q) == 0) { + return NULL; + } + void* ptr = taosArrayGetP(queue->q, 0); + taosArrayRemove(queue->q, 0); + return ptr; +} + +void* transQueueGet(STransQueue* queue) { + if (taosArrayGetSize(queue->q) == 0) { + return NULL; + } + void* ptr = taosArrayGetP(queue->q, 0); + return ptr; +} +bool transQueueEmpty(STransQueue* queue) { + // + return taosArrayGetSize(queue->q) == 0; +} +void transQueueClear(STransQueue* queue) { + if (queue->free != NULL) { + for (int i = 0; i < taosArrayGetSize(queue->q); i++) { + void* p = taosArrayGetP(queue->q, i); + queue->free(p); + } + } + taosArrayClear(queue->q); +} +void transQueueDestroy(STransQueue* queue) { + transQueueClear(queue); + taosArrayDestroy(queue->q); +} + #endif diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 73d836319f..c6032a9569 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -17,6 +17,12 @@ #include "transComm.h" +typedef struct { + int notifyCount; // + int init; // init or not + STransMsg msg; +} SSrvRegArg; + typedef struct SSrvConn { T_REF_DECLARE() uv_tcp_t* pTcp; @@ -31,10 +37,12 @@ typedef struct SSrvConn { void* pTransInst; // rpc init void* ahandle; // void* hostThrd; - SArray* srvMsgs; + STransQueue srvMsgs; - bool broken; // conn broken; + SSrvRegArg regArg; + bool broken; // conn broken; + ConnStatus status; struct sockaddr_in addr; struct sockaddr_in locaddr; @@ -47,20 +55,20 @@ typedef struct SSrvConn { } SSrvConn; typedef struct SSrvMsg { - SSrvConn* pConn; - STransMsg msg; - queue q; + SSrvConn* pConn; + STransMsg msg; + queue q; + STransMsgType type; } SSrvMsg; typedef struct SWorkThrdObj { - pthread_t thread; - uv_pipe_t* pipe; - uv_os_fd_t fd; - uv_loop_t* loop; - SAsyncPool* asyncPool; - - queue msg; - pthread_mutex_t msgMtx; + TdThread thread; + uv_pipe_t* pipe; + uv_os_fd_t fd; + uv_loop_t* loop; + SAsyncPool* asyncPool; + queue msg; + TdThreadMutex msgMtx; queue conn; void* pTransInst; @@ -68,7 +76,7 @@ typedef struct SWorkThrdObj { } SWorkThrdObj; typedef struct SServerObj { - pthread_t thread; + TdThread thread; uv_tcp_t server; uv_loop_t* loop; @@ -85,10 +93,26 @@ typedef struct SServerObj { static const char* notify = "a"; -// refactor later -static int transAddAuthPart(SSrvConn* pConn, char* msg, int msgLen); - -static int uvAuthMsg(SSrvConn* pConn, char* msg, int msgLen); +#define CONN_SHOULD_RELEASE(conn, head) \ + do { \ + if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ + conn->status = ConnRelease; \ + transClearBuffer(&conn->readBuf); \ + transFreeMsg(transContFromHead((char*)head)); \ + tTrace("server conn %p received release request", conn); \ + \ + STransMsg tmsg = {.handle = (void*)conn, .code = 0}; \ + SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); \ + srvMsg->msg = tmsg; \ + srvMsg->type = Release; \ + srvMsg->pConn = conn; \ + if (!transQueuePush(&conn->srvMsgs, srvMsg)) { \ + return; \ + } \ + uvStartSendRespInternal(srvMsg); \ + return; \ + } \ + } while (0) static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); static void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); @@ -113,6 +137,13 @@ static void destroySmsg(SSrvMsg* smsg); static SSrvConn* createConn(void* hThrd); static void destroyConn(SSrvConn* conn, bool clear /*clear handle or not*/); +static void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd); +static void uvHandleRelease(SSrvMsg* msg, SWorkThrdObj* thrd); +static void uvHandleResp(SSrvMsg* msg, SWorkThrdObj* thrd); +static void uvHandleRegister(SSrvMsg* msg, SWorkThrdObj* thrd); +static void (*transAsyncHandle[])(SSrvMsg* msg, SWorkThrdObj* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease, + uvHandleRegister}; + static void uvDestroyConn(uv_handle_t* handle); // server and worker thread @@ -129,59 +160,6 @@ void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* b transAllocBuffer(pBuf, buf); } -static int uvAuthMsg(SSrvConn* pConn, char* msg, int len) { - STransMsgHead* pHead = (STransMsgHead*)msg; - - int code = 0; - - if ((pConn->secured && pHead->spi == 0) || (pHead->spi == 0 && pConn->spi == 0)) { - // secured link, or no authentication - pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); - // tTrace("%s, secured link, no auth is required", pConn->info); - return 0; - } - - if (!rpcIsReq(pHead->msgType)) { - // for response, if code is auth failure, it shall bypass the auth process - code = htonl(pHead->code); - if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE || - code == TSDB_CODE_RPC_INVALID_VERSION || code == TSDB_CODE_RPC_AUTH_REQUIRED || - code == TSDB_CODE_MND_USER_NOT_EXIST || code == TSDB_CODE_RPC_NOT_READY) { - pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); - // tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code); - return 0; - } - } - - code = 0; - if (pHead->spi == pConn->spi) { - // authentication - SRpcDigest* pDigest = (SRpcDigest*)((char*)pHead + len - sizeof(SRpcDigest)); - - int32_t delta; - delta = (int32_t)htonl(pDigest->timeStamp); - delta -= (int32_t)taosGetTimestampSec(); - if (abs(delta) > 900) { - tWarn("%s, time diff:%d is too big, msg discarded", pConn->info, delta); - code = TSDB_CODE_RPC_INVALID_TIME_STAMP; - } else { - if (transAuthenticateMsg(pHead, len - TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) { - // tDebug("%s, authentication failed, msg discarded", pConn->info); - code = TSDB_CODE_RPC_AUTH_FAILURE; - } else { - pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen) - sizeof(SRpcDigest); - if (!rpcIsReq(pHead->msgType)) pConn->secured = 1; // link is secured for client - // tTrace("%s, message is authenticated", pConn->info); - } - } - } else { - tDebug("%s, auth spi:%d not matched with received:%d", pConn->info, pConn->spi, pHead->spi); - code = pHead->spi ? TSDB_CODE_RPC_AUTH_FAILURE : TSDB_CODE_RPC_AUTH_REQUIRED; - } - - return code; -} - // refers specifically to query or insert timeout static void uvHandleActivityTimeout(uv_timer_t* handle) { SSrvConn* conn = handle->data; @@ -189,37 +167,24 @@ static void uvHandleActivityTimeout(uv_timer_t* handle) { } static void uvHandleReq(SSrvConn* pConn) { - SRecvInfo info; - SRecvInfo* p = &info; SConnBuffer* pBuf = &pConn->readBuf; - p->msg = pBuf->buf; - p->msgLen = pBuf->len; - p->ip = 0; - p->port = 0; - p->shandle = pConn->pTransInst; // - p->thandle = pConn; - p->chandle = NULL; + char* msg = pBuf->buf; + uint32_t msgLen = pBuf->len; - STransMsgHead* pHead = (STransMsgHead*)p->msg; + STransMsgHead* pHead = (STransMsgHead*)msg; if (pHead->secured == 1) { - STransUserMsg* uMsg = (STransUserMsg*)((char*)p->msg + p->msgLen - sizeof(STransUserMsg)); + STransUserMsg* uMsg = (STransUserMsg*)((char*)msg + msgLen - sizeof(STransUserMsg)); memcpy(pConn->user, uMsg->user, tListLen(uMsg->user)); memcpy(pConn->secret, uMsg->secret, tListLen(uMsg->secret)); } pHead->code = htonl(pHead->code); - - int32_t dlen = 0; - if (transDecompressMsg(NULL, 0, NULL)) { - // add compress later - // pHead = rpcDecompresSTransMsg(pHead); - } else { - pHead->msgLen = htonl(pHead->msgLen); - if (pHead->secured == 1) { - pHead->msgLen -= sizeof(STransUserMsg); - } - // + pHead->msgLen = htonl(pHead->msgLen); + if (pHead->secured == 1) { + pHead->msgLen -= sizeof(STransUserMsg); } + CONN_SHOULD_RELEASE(pConn, pHead); + STransMsg transMsg; transMsg.contLen = transContLenFromMsg(pHead->msgLen); transMsg.pCont = pHead->content; @@ -230,24 +195,31 @@ static void uvHandleReq(SSrvConn* pConn) { transClearBuffer(&pConn->readBuf); pConn->inType = pHead->msgType; - - if (pHead->resflag == 0) { + if (pConn->status == ConnNormal) { + if (pHead->persist == 1) { + pConn->status = ConnAcquire; + transRefSrvHandle(pConn); + } + } + if (pConn->status == ConnNormal && pHead->noResp == 0) { transRefSrvHandle(pConn); - transMsg.handle = pConn; tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pConn, TMSG_INFO(transMsg.msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port), transMsg.contLen); } else { - tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, no resp ", pConn, + tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, resp:%d ", pConn, TMSG_INFO(transMsg.msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), - taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port), transMsg.contLen); + taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port), transMsg.contLen, pHead->noResp); + // no ref here } - STrans* pTransInst = (STrans*)p->shandle; + if (pHead->noResp == 0) { + transMsg.handle = pConn; + } + + STrans* pTransInst = pConn->pTransInst; (*pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); // uv_timer_start(&pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); - // auth - // validate msg type } void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { @@ -272,11 +244,13 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { tError("server conn %p read error: %s", conn, uv_err_name(nread)); if (nread < 0) { conn->broken = true; - uvNotifyLinkBrokenToApp(conn); - - // STrans* pTransInst = conn->pTransInst; - // if (pTransInst->efp != NULL && (pTransInst->efp)(NULL, conn->inType)) { - //} + if (conn->status == ConnAcquire) { + if (conn->regArg.init) { + STrans* pTransInst = conn->pTransInst; + (*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); + memset(&conn->regArg, 0, sizeof(conn->regArg)); + } + } transUnrefSrvHandle(conn); } } @@ -296,23 +270,35 @@ void uvOnSendCb(uv_write_t* req, int status) { transClearBuffer(&conn->readBuf); if (status == 0) { tTrace("server conn %p data already was written on stream", conn); - if (conn->srvMsgs != NULL) { - assert(taosArrayGetSize(conn->srvMsgs) >= 1); - SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, 0); - tTrace("server conn %p sending msg size: %d", conn, (int)taosArrayGetSize(conn->srvMsgs)); - taosArrayRemove(conn->srvMsgs, 0); + if (!transQueueEmpty(&conn->srvMsgs)) { + SSrvMsg* msg = transQueuePop(&conn->srvMsgs); + if (msg->type == Release && conn->status != ConnNormal) { + conn->status = ConnNormal; + transUnrefSrvHandle(conn); + } destroySmsg(msg); - // send second data, just use for push - if (taosArrayGetSize(conn->srvMsgs) > 0) { - tTrace("resent server conn %p sending msg size: %d", conn, (int)taosArrayGetSize(conn->srvMsgs)); - msg = (SSrvMsg*)taosArrayGetP(conn->srvMsgs, 0); - uvStartSendRespInternal(msg); + if (!transQueueEmpty(&conn->srvMsgs)) { + msg = (SSrvMsg*)transQueueGet(&conn->srvMsgs); + if (msg->type == Register && conn->status == ConnAcquire) { + conn->regArg.notifyCount = 0; + conn->regArg.init = 1; + conn->regArg.msg = msg->msg; + if (conn->broken) { + STrans* pTransInst = conn->pTransInst; + (pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); + memset(&conn->regArg, 0, sizeof(conn->regArg)); + } + transQueuePop(&conn->srvMsgs); + free(msg); + } else { + uvStartSendRespInternal(msg); + } } } } else { tError("server conn %p failed to write data, %s", conn, uv_err_name(status)); - conn->broken = false; + conn->broken = true; transUnrefSrvHandle(conn); } } @@ -326,7 +312,6 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) { } static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { - // impl later; tTrace("server conn %p prepare to send resp", smsg->pConn); SSrvConn* pConn = smsg->pConn; @@ -337,20 +322,27 @@ static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { } STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); - pHead->secured = pMsg->code == 0 ? 1 : 0; // - pHead->msgType = smsg->pConn->inType + 1; + // pHead->secured = pMsg->code == 0 ? 1 : 0; // + if (!pConn->secured) { + pConn->secured = pMsg->code == 0 ? 1 : 0; + } + pHead->secured = pConn->secured; + + if (pConn->status == ConnNormal) { + pHead->msgType = pConn->inType + 1; + } else { + pHead->msgType = smsg->type == Release ? 0 : pMsg->msgType; + } + pHead->release = smsg->type == Release ? 1 : 0; pHead->code = htonl(pMsg->code); - // add more info + char* msg = (char*)pHead; int32_t len = transMsgLenFromCont(pMsg->contLen); - if (transCompressMsg(msg, len, NULL)) { - // impl later - } tDebug("server conn %p %s is sent to %s:%d, local info: %s:%d", pConn, TMSG_INFO(pHead->msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); - pHead->msgLen = htonl(len); + wb->base = msg; wb->len = len; } @@ -368,32 +360,21 @@ static void uvStartSendResp(SSrvMsg* smsg) { SSrvConn* pConn = smsg->pConn; if (pConn->broken == true) { + // persist by transUnrefSrvHandle(pConn); return; } - transUnrefSrvHandle(pConn); + if (pConn->status == ConnNormal) { + transUnrefSrvHandle(pConn); + } - if (taosArrayGetSize(pConn->srvMsgs) > 0) { - tDebug("server conn %p push data to client %s:%d, local info: %s:%d", pConn, taosInetNtoa(pConn->addr.sin_addr), - ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); - taosArrayPush(pConn->srvMsgs, &smsg); + if (!transQueuePush(&pConn->srvMsgs, smsg)) { return; } - taosArrayPush(pConn->srvMsgs, &smsg); uvStartSendRespInternal(smsg); return; } -static void uvNotifyLinkBrokenToApp(SSrvConn* conn) { - STrans* pTransInst = conn->pTransInst; - if (pTransInst->efp != NULL && (*pTransInst->efp)(NULL, conn->inType) && T_REF_VAL_GET(conn) >= 2) { - STransMsg transMsg = {0}; - transMsg.msgType = conn->inType; - transMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - // transRefSrvHandle(conn); - (*pTransInst->cfp)(pTransInst->parent, &transMsg, 0); - } -} static void destroySmsg(SSrvMsg* smsg) { if (smsg == NULL) { return; @@ -408,6 +389,9 @@ static void destroyAllConn(SWorkThrdObj* pThrd) { QUEUE_INIT(h); SSrvConn* c = QUEUE_DATA(h, SSrvConn, queue); + while (T_REF_VAL_GET(c) >= 2) { + transUnrefSrvHandle(c); + } transUnrefSrvHandle(c); } } @@ -418,9 +402,9 @@ void uvWorkerAsyncCb(uv_async_t* handle) { queue wq; // batch process to avoid to lock/unlock frequently - pthread_mutex_lock(&item->mtx); + taosThreadMutexLock(&item->mtx); QUEUE_MOVE(&item->qmsg, &wq); - pthread_mutex_unlock(&item->mtx); + taosThreadMutexUnlock(&item->mtx); while (!QUEUE_IS_EMPTY(&wq)) { queue* head = QUEUE_HEAD(&wq); @@ -431,20 +415,7 @@ void uvWorkerAsyncCb(uv_async_t* handle) { tError("unexcept occurred, continue"); continue; } - if (msg->pConn == NULL) { - free(msg); - bool noConn = QUEUE_IS_EMPTY(&pThrd->conn); - if (noConn == true) { - uv_loop_close(pThrd->loop); - uv_stop(pThrd->loop); - } else { - destroyAllConn(pThrd); - // uv_loop_close(pThrd->loop); - pThrd->quit = true; - } - } else { - uvStartSendResp(msg); - } + (*transAsyncHandle[msg->type])(msg, pThrd); } } static void uvAcceptAsyncCb(uv_async_t* async) { @@ -577,7 +548,7 @@ static bool addHandleToWorkloop(void* arg) { pThrd->pipe->data = pThrd; QUEUE_INIT(&pThrd->msg); - pthread_mutex_init(&pThrd->msgMtx, NULL); + taosThreadMutexInit(&pThrd->msgMtx, NULL); // conn set QUEUE_INIT(&pThrd->conn); @@ -629,12 +600,15 @@ static SSrvConn* createConn(void* hThrd) { QUEUE_INIT(&pConn->queue); QUEUE_PUSH(&pThrd->conn, &pConn->queue); - pConn->srvMsgs = taosArrayInit(2, sizeof(void*)); // - tTrace("conn %p created", pConn); + transQueueInit(&pConn->srvMsgs, NULL); + + memset(&pConn->regArg, 0, sizeof(pConn->regArg)); pConn->broken = false; + pConn->status = ConnNormal; transRefSrvHandle(pConn); + tTrace("server conn %p created", pConn); return pConn; } @@ -644,13 +618,9 @@ static void destroyConn(SSrvConn* conn, bool clear) { } transDestroyBuffer(&conn->readBuf); - for (int i = 0; i < taosArrayGetSize(conn->srvMsgs); i++) { - SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, i); - destroySmsg(msg); - } - conn->srvMsgs = taosArrayDestroy(conn->srvMsgs); + transQueueDestroy(&conn->srvMsgs); if (clear) { - tTrace("try to destroy conn %p", conn); + tTrace("server conn %p to be destroyed", conn); uv_shutdown_t* req = malloc(sizeof(uv_shutdown_t)); uv_shutdown(req, (uv_stream_t*)conn->pTcp, uvShutDownCb); } @@ -673,25 +643,6 @@ static void uvDestroyConn(uv_handle_t* handle) { uv_stop(thrd->loop); } } -static int transAddAuthPart(SSrvConn* pConn, char* msg, int msgLen) { - STransMsgHead* pHead = (STransMsgHead*)msg; - - if (pConn->spi && pConn->secured == 0) { - // add auth part - pHead->spi = pConn->spi; - STransDigestMsg* pDigest = (STransDigestMsg*)(msg + msgLen); - pDigest->timeStamp = htonl(taosGetTimestampSec()); - msgLen += sizeof(SRpcDigest); - pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); - // transBuildAuthHead(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret); - // transBuildAuthHead(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret); - } else { - pHead->spi = 0; - pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); - } - - return msgLen; -} void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { SServerObj* srv = calloc(1, sizeof(SServerObj)); @@ -724,7 +675,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, if (false == addHandleToWorkloop(thrd)) { goto End; } - int err = pthread_create(&(thrd->thread), NULL, workerThread, (void*)(thrd)); + int err = taosThreadCreate(&(thrd->thread), NULL, workerThread, (void*)(thrd)); if (err == 0) { tDebug("sucess to create worker-thread %d", i); // printf("thread %d create\n", i); @@ -736,7 +687,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, if (false == addHandleToAcceptloop(srv)) { goto End; } - int err = pthread_create(&srv->thread, NULL, acceptThread, (void*)srv); + int err = taosThreadCreate(&srv->thread, NULL, acceptThread, (void*)srv); if (err == 0) { tDebug("success to create accept-thread"); } else { @@ -748,21 +699,68 @@ End: transCloseServer(srv); return NULL; } +void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd) { + if (QUEUE_IS_EMPTY(&thrd->conn)) { + uv_loop_close(thrd->loop); + uv_stop(thrd->loop); + } else { + destroyAllConn(thrd); + thrd->quit = true; + } + free(msg); +} +void uvHandleRelease(SSrvMsg* msg, SWorkThrdObj* thrd) { + // release handle to rpc init + SSrvConn* conn = msg->pConn; + if (conn->status == ConnAcquire) { + if (!transQueuePush(&conn->srvMsgs, msg)) { + return; + } + uvStartSendRespInternal(msg); + return; + } else if (conn->status == ConnRelease || conn->status == ConnNormal) { + tDebug("server conn %p already released, ignore release-msg", conn); + } + destroySmsg(msg); +} +void uvHandleResp(SSrvMsg* msg, SWorkThrdObj* thrd) { + // send msg to client + tDebug("server conn %p start to send resp", msg->pConn); + uvStartSendResp(msg); +} +void uvHandleRegister(SSrvMsg* msg, SWorkThrdObj* thrd) { + SSrvConn* conn = msg->pConn; + tDebug("server conn %p register brokenlink callback", conn); + if (conn->status == ConnAcquire) { + if (!transQueuePush(&conn->srvMsgs, msg)) { + return; + } + conn->regArg.notifyCount = 0; + conn->regArg.init = 1; + conn->regArg.msg = msg->msg; + if (conn->broken) { + STrans* pTransInst = conn->pTransInst; + (*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); + memset(&conn->regArg, 0, sizeof(conn->regArg)); + } + free(msg); + } +} void destroyWorkThrd(SWorkThrdObj* pThrd) { if (pThrd == NULL) { return; } - pthread_join(pThrd->thread, NULL); + taosThreadJoin(pThrd->thread, NULL); free(pThrd->loop); transDestroyAsyncPool(pThrd->asyncPool); free(pThrd); } void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { - SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); + SSrvMsg* msg = calloc(1, sizeof(SSrvMsg)); + msg->type = Quit; tDebug("server send quit msg to work thread"); - - transSendAsync(pThrd->asyncPool, &srvMsg->q); + transSendAsync(pThrd->asyncPool, &msg->q); } void transCloseServer(void* arg) { @@ -775,7 +773,7 @@ void transCloseServer(void* arg) { tDebug("send quit msg to accept thread"); uv_async_send(srv->pAcceptAsync); - pthread_join(srv->thread, NULL); + taosThreadJoin(srv->thread, NULL); free(srv->pThreadObj); free(srv->pAcceptAsync); @@ -804,17 +802,28 @@ void transUnrefSrvHandle(void* handle) { return; } int ref = T_REF_DEC((SSrvConn*)handle); - tDebug("handle %p ref count: %d", handle, ref); - + tDebug("server conn %p ref count: %d", handle, ref); if (ref == 0) { destroyConn((SSrvConn*)handle, true); } - // unref srv handle } void transReleaseSrvHandle(void* handle) { - // do nothing currently - // + if (handle == NULL) { + return; + } + SSrvConn* pConn = handle; + SWorkThrdObj* pThrd = pConn->hostThrd; + + STransMsg tmsg = {.handle = handle, .code = 0}; + + SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); + srvMsg->msg = tmsg; + srvMsg->type = Release; + srvMsg->pConn = pConn; + + tTrace("server conn %p start to release", pConn); + transSendAsync(pThrd->asyncPool, &srvMsg->q); } void transSendResponse(const STransMsg* pMsg) { if (pMsg->handle == NULL) { @@ -826,9 +835,24 @@ void transSendResponse(const STransMsg* pMsg) { SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); srvMsg->pConn = pConn; srvMsg->msg = *pMsg; + srvMsg->type = Normal; tTrace("server conn %p start to send resp", pConn); transSendAsync(pThrd->asyncPool, &srvMsg->q); } +void transRegisterMsg(const STransMsg* msg) { + if (msg->handle == NULL) { + return; + } + SSrvConn* pConn = msg->handle; + SWorkThrdObj* pThrd = pConn->hostThrd; + + SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); + srvMsg->pConn = pConn; + srvMsg->msg = *msg; + srvMsg->type = Register; + tTrace("server conn %p start to register brokenlink callback", pConn); + transSendAsync(pThrd->asyncPool, &srvMsg->q); +} int transGetConnInfo(void* thandle, STransHandleInfo* pInfo) { SSrvConn* pConn = thandle; struct sockaddr_in addr = pConn->addr; diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index c6ff2480ef..4af316acae 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -30,7 +30,7 @@ typedef struct { int msgSize; tsem_t rspSem; tsem_t * pOverSem; - pthread_t thread; + TdThread thread; void * pRpc; } SInfo; static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { @@ -104,7 +104,7 @@ int main(int argc, char *argv[]) { char secret[20] = "mypassword"; struct timeval systemTime; int64_t startTime, endTime; - pthread_attr_t thattr; + TdThreadAttr thattr; // server info epSet.inUse = 0; @@ -186,8 +186,8 @@ int main(int argc, char *argv[]) { SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo) * appThreads); - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); for (int i = 0; i < appThreads; ++i) { pInfo->index = i; @@ -196,7 +196,7 @@ int main(int argc, char *argv[]) { pInfo->msgSize = msgSize; tsem_init(&pInfo->rspSem, 0, 0); pInfo->pRpc = pRpc; - pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo); + taosThreadCreate(&pInfo->thread, &thattr, sendRequest, pInfo); pInfo++; } diff --git a/source/libs/transport/test/rsclient.c b/source/libs/transport/test/rsclient.c index 1fe13a35b1..f3be4e3452 100644 --- a/source/libs/transport/test/rsclient.c +++ b/source/libs/transport/test/rsclient.c @@ -29,7 +29,7 @@ typedef struct { int msgSize; tsem_t rspSem; tsem_t *pOverSem; - pthread_t thread; + TdThread thread; void *pRpc; } SInfo; @@ -80,7 +80,7 @@ int main(int argc, char *argv[]) { char secret[TSDB_KEY_LEN] = "mypassword"; struct timeval systemTime; int64_t startTime, endTime; - pthread_attr_t thattr; + TdThreadAttr thattr; // server info epSet.numOfEps = 1; @@ -163,8 +163,8 @@ int main(int argc, char *argv[]) { SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo)*appThreads); - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); for (int i=0; iindex = i; @@ -173,7 +173,7 @@ int main(int argc, char *argv[]) { pInfo->msgSize = msgSize; tsem_init(&pInfo->rspSem, 0, 0); pInfo->pRpc = pRpc; - pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo); + taosThreadCreate(&pInfo->thread, &thattr, sendRequest, pInfo); pInfo++; } diff --git a/source/libs/transport/test/syncClient.c b/source/libs/transport/test/syncClient.c index b3c2f6ebdc..14f32df0b8 100644 --- a/source/libs/transport/test/syncClient.c +++ b/source/libs/transport/test/syncClient.c @@ -30,7 +30,7 @@ typedef struct { int msgSize; tsem_t rspSem; tsem_t * pOverSem; - pthread_t thread; + TdThread thread; void * pRpc; } SInfo; static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { @@ -105,7 +105,7 @@ int main(int argc, char *argv[]) { char secret[20] = "mypassword"; struct timeval systemTime; int64_t startTime, endTime; - pthread_attr_t thattr; + TdThreadAttr thattr; // server info epSet.inUse = 0; @@ -186,8 +186,8 @@ int main(int argc, char *argv[]) { SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo) * appThreads); - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); for (int i = 0; i < appThreads; ++i) { pInfo->index = i; @@ -196,7 +196,7 @@ int main(int argc, char *argv[]) { pInfo->msgSize = msgSize; tsem_init(&pInfo->rspSem, 0, 0); pInfo->pRpc = pRpc; - pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo); + taosThreadCreate(&pInfo->thread, &thattr, sendRequest, pInfo); pInfo++; } diff --git a/source/libs/transport/test/transUT.cc b/source/libs/transport/test/transUT.cc index ec89d695a2..0b1b1834df 100644 --- a/source/libs/transport/test/transUT.cc +++ b/source/libs/transport/test/transUT.cc @@ -30,32 +30,13 @@ const char *ckey = "ckey"; class Server; int port = 7000; // server process - -static bool cliPersistHandle(void *parent, tmsg_t msgType) { - // client persist handle - return msgType == 2 || msgType == 4; -} - -typedef struct CbArgs { - tmsg_t msgType; -} CbArgs; - -static void *ConstructArgForSpecificMsgType(void *parent, tmsg_t msgType) { - if (msgType == 1 || msgType == 2) { - CbArgs *args = (CbArgs *)calloc(1, sizeof(CbArgs)); - args->msgType = msgType; - return args; - } - return NULL; -} // server except -static bool handleExcept(void *parent, tmsg_t msgType) { - // - return msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP; -} + typedef void (*CB)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); static void processContinueSend(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void processReleaseHandleCb(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void processRegisterFailure(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); static void processReq(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); // client process; static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); @@ -91,23 +72,6 @@ class Client { rpcClose(this->transCli); this->transCli = NULL; } - void SetPersistFP(bool (*pfp)(void *parent, tmsg_t msgType)) { - rpcClose(this->transCli); - rpcInit_.pfp = pfp; - this->transCli = rpcOpen(&rpcInit_); - } - void SetConstructFP(void *(*mfp)(void *parent, tmsg_t msgType)) { - rpcClose(this->transCli); - rpcInit_.mfp = mfp; - this->transCli = rpcOpen(&rpcInit_); - } - void SetPAndMFp(bool (*pfp)(void *parent, tmsg_t msgType), void *(*mfp)(void *parent, tmsg_t msgType)) { - rpcClose(this->transCli); - - rpcInit_.pfp = pfp; - rpcInit_.mfp = mfp; - this->transCli = rpcOpen(&rpcInit_); - } void SendAndRecv(SRpcMsg *req, SRpcMsg *resp) { SEpSet epSet = {0}; @@ -126,7 +90,6 @@ class Client { SendAndRecv(req, resp); } - void SendWithHandle(SRpcMsg *req, SRpcMsg *resp) {} void SemWait() { tsem_wait(&this->sem); } void SemPost() { tsem_post(&this->sem); } void Reset() {} @@ -149,7 +112,6 @@ class Server { rpcInit_.label = (char *)label; rpcInit_.numOfThreads = 5; rpcInit_.cfp = processReq; - rpcInit_.efp = NULL; rpcInit_.user = (char *)user; rpcInit_.secret = (char *)secret; rpcInit_.ckey = (char *)ckey; @@ -160,17 +122,17 @@ class Server { this->transSrv = rpcOpen(&this->rpcInit_); taosMsleep(1000); } + void SetSrvContinueSend(CB cb) { + this->Stop(); + rpcInit_.cfp = cb; + this->Start(); + } void Stop() { if (this->transSrv == NULL) return; rpcClose(this->transSrv); this->transSrv = NULL; } - void SetExceptFp(bool (*efp)(void *parent, tmsg_t msgType)) { - this->Stop(); - rpcInit_.efp = efp; - this->Start(); - } - void SetSrvContinueSend(void (*cfp)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet)) { + void SetSrvSend(void (*cfp)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet)) { this->Stop(); rpcInit_.cfp = cfp; this->Start(); @@ -198,9 +160,6 @@ static void processReq(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { } static void processContinueSend(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { - for (int i = 0; i < 9; i++) { - rpcRefHandle(pMsg->handle, TAOS_CONN_SERVER); - } for (int i = 0; i < 10; i++) { SRpcMsg rpcMsg = {0}; rpcMsg.pCont = rpcMallocCont(100); @@ -210,6 +169,35 @@ static void processContinueSend(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { rpcSendResponse(&rpcMsg); } } +static void processReleaseHandleCb(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = rpcMallocCont(100); + rpcMsg.contLen = 100; + rpcMsg.handle = pMsg->handle; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); + + rpcReleaseHandle(pMsg->handle, TAOS_CONN_SERVER); +} +static void processRegisterFailure(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + void *handle = pMsg->handle; + { + SRpcMsg rpcMsg1 = {0}; + rpcMsg1.pCont = rpcMallocCont(100); + rpcMsg1.contLen = 100; + rpcMsg1.handle = handle; + rpcMsg1.code = 0; + rpcRegisterBrokenLinkArg(&rpcMsg1); + } + taosMsleep(10); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = rpcMallocCont(100); + rpcMsg.contLen = 100; + rpcMsg.handle = pMsg->handle; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); +} // client process; static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { Client *client = (Client *)parent; @@ -262,29 +250,13 @@ class TransObj { // srv->Stop(); } - void SetCliPersistFp(bool (*pfp)(void *parent, tmsg_t msgType)) { - // do nothing - cli->SetPersistFP(pfp); - } - void SetCliMFp(void *(*mfp)(void *parent, tmsg_t msgType)) { - // do nothing - cli->SetConstructFP(mfp); - } - void SetCliMAndPFp(bool (*pfp)(void *parent, tmsg_t msgType), void *(*mfp)(void *parent, tmsg_t msgType)) { - // do nothing - cli->SetPAndMFp(pfp, mfp); - } // call when link broken, and notify query or fetch stop - void SetSrvExceptFp(bool (*efp)(void *parent, tmsg_t msgType)) { - //////// - srv->SetExceptFp(efp); - } void SetSrvContinueSend(void (*cfp)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet)) { /////// srv->SetSrvContinueSend(cfp); } void RestartSrv() { srv->Restart(); } - void cliStop() { + void StopCli() { /////// cli->Stop(); } @@ -315,7 +287,7 @@ class TransEnv : public ::testing::Test { }; TEST_F(TransEnv, 01sendAndRec) { - for (int i = 0; i < 1; i++) { + for (int i = 0; i < 10; i++) { SRpcMsg req = {0}, resp = {0}; req.msgType = 0; req.pCont = rpcMallocCont(10); @@ -358,60 +330,65 @@ TEST_F(TransEnv, clientUserDefined) { } TEST_F(TransEnv, cliPersistHandle) { - tr->SetCliPersistFp(cliPersistHandle); SRpcMsg resp = {0}; + void * handle = NULL; for (int i = 0; i < 10; i++) { - SRpcMsg req = {.handle = resp.handle, .noResp = 0}; + SRpcMsg req = {.handle = resp.handle, .persistHandle = 1}; req.msgType = 1; req.pCont = rpcMallocCont(10); req.contLen = 10; tr->cliSendAndRecv(&req, &resp); - if (i == 5) { - std::cout << "stop server" << std::endl; - tr->StopSrv(); - } - if (i >= 6) { - EXPECT_TRUE(resp.code != 0); - } - } - ////////////////// -} - -TEST_F(TransEnv, cliReleaseHandle) { - tr->SetCliPersistFp(cliPersistHandle); - - SRpcMsg resp = {0}; - for (int i = 0; i < 10; i++) { - SRpcMsg req = {.handle = resp.handle}; - req.msgType = 1; - req.pCont = rpcMallocCont(10); - req.contLen = 10; - tr->cliSendAndRecvNoHandle(&req, &resp); // if (i == 5) { // std::cout << "stop server" << std::endl; // tr->StopSrv(); //} // if (i >= 6) { - EXPECT_TRUE(resp.code == 0); + // EXPECT_TRUE(resp.code != 0); //} + handle = resp.handle; + } + rpcReleaseHandle(handle, TAOS_CONN_CLIENT); + for (int i = 0; i < 10; i++) { + SRpcMsg req = {0}; + req.msgType = 1; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + } + + taosMsleep(1000); + ////////////////// +} + +TEST_F(TransEnv, srvReleaseHandle) { + SRpcMsg resp = {0}; + tr->SetSrvContinueSend(processReleaseHandleCb); + // tr->Restart(processReleaseHandleCb); + void *handle = NULL; + for (int i = 0; i < 1; i++) { + SRpcMsg req = {.handle = resp.handle, .persistHandle = 1}; + req.msgType = 1; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + // tr->cliSendAndRecvNoHandle(&req, &resp); + EXPECT_TRUE(resp.code == 0); } ////////////////// } TEST_F(TransEnv, cliReleaseHandleExcept) { - tr->SetCliPersistFp(cliPersistHandle); - SRpcMsg resp = {0}; - for (int i = 0; i < 10; i++) { - SRpcMsg req = {.handle = resp.handle}; + for (int i = 0; i < 3; i++) { + SRpcMsg req = {.handle = resp.handle, .persistHandle = 1}; req.msgType = 1; req.pCont = rpcMallocCont(10); req.contLen = 10; - tr->cliSendAndRecvNoHandle(&req, &resp); - if (i == 5) { + tr->cliSendAndRecv(&req, &resp); + if (i == 1) { std::cout << "stop server" << std::endl; tr->StopSrv(); } - if (i >= 6) { + if (i > 1) { EXPECT_TRUE(resp.code != 0); } } @@ -426,12 +403,12 @@ TEST_F(TransEnv, srvContinueSend) { req.contLen = 10; tr->cliSendAndRecv(&req, &resp); } - taosMsleep(2000); + taosMsleep(1000); } TEST_F(TransEnv, srvPersistHandleExcept) { tr->SetSrvContinueSend(processContinueSend); - tr->SetCliPersistFp(cliPersistHandle); + // tr->SetCliPersistFp(cliPersistHandle); SRpcMsg resp = {0}; for (int i = 0; i < 5; i++) { SRpcMsg req = {.handle = resp.handle}; @@ -440,7 +417,7 @@ TEST_F(TransEnv, srvPersistHandleExcept) { req.contLen = 10; tr->cliSendAndRecv(&req, &resp); if (i > 2) { - tr->cliStop(); + tr->StopCli(); break; } } @@ -450,7 +427,6 @@ TEST_F(TransEnv, srvPersistHandleExcept) { } TEST_F(TransEnv, cliPersistHandleExcept) { tr->SetSrvContinueSend(processContinueSend); - tr->SetCliPersistFp(cliPersistHandle); SRpcMsg resp = {0}; for (int i = 0; i < 5; i++) { SRpcMsg req = {.handle = resp.handle}; @@ -472,9 +448,21 @@ TEST_F(TransEnv, multiCliPersistHandleExcept) { // conn broken } TEST_F(TransEnv, queryExcept) { - tr->SetSrvExceptFp(handleExcept); - - // query and conn is broken + tr->SetSrvContinueSend(processRegisterFailure); + SRpcMsg resp = {0}; + for (int i = 0; i < 5; i++) { + SRpcMsg req = {.handle = resp.handle, .persistHandle = 1}; + req.msgType = 1; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + if (i == 2) { + rpcReleaseHandle(resp.handle, TAOS_CONN_CLIENT); + tr->StopCli(); + break; + } + } + taosMsleep(4 * 1000); } TEST_F(TransEnv, noResp) { SRpcMsg resp = {0}; diff --git a/source/libs/transport/test/transportTests.cc b/source/libs/transport/test/transportTests.cc index 53910aa30c..65d9302994 100644 --- a/source/libs/transport/test/transportTests.cc +++ b/source/libs/transport/test/transportTests.cc @@ -136,4 +136,98 @@ TEST_F(QueueEnv, testIter) { assert(result.size() == vals.size()); } +class TransCtxEnv : public ::testing::Test { + protected: + virtual void SetUp() { + ctx = (STransCtx *)calloc(1, sizeof(STransCtx)); + transCtxInit(ctx); + // TODO + } + virtual void TearDown() { + transCtxCleanup(ctx); + // formate + } + STransCtx *ctx; +}; + +TEST_F(TransCtxEnv, mergeTest) { + int key = 1; + { + STransCtx *src = (STransCtx *)calloc(1, sizeof(STransCtx)); + transCtxInit(src); + { + STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; + val1.val = malloc(12); + val1.len = 12; + + taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); + key++; + } + { + STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; + val1.val = malloc(12); + val1.len = 12; + taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); + key++; + } + transCtxMerge(ctx, src); + free(src); + } + EXPECT_EQ(2, taosHashGetSize(ctx->args)); + { + STransCtx *src = (STransCtx *)calloc(1, sizeof(STransCtx)); + transCtxInit(src); + { + STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; + val1.val = malloc(12); + val1.len = 12; + + taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); + key++; + } + { + STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; + val1.val = malloc(12); + val1.len = 12; + taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); + key++; + } + transCtxMerge(ctx, src); + free(src); + } + std::string val("Hello"); + EXPECT_EQ(4, taosHashGetSize(ctx->args)); + { + key = 1; + STransCtx *src = (STransCtx *)calloc(1, sizeof(STransCtx)); + transCtxInit(src); + { + STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; + val1.val = calloc(1, 11); + memcpy(val1.val, val.c_str(), val.size()); + val1.len = 11; + + taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); + key++; + } + { + STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; + val1.val = calloc(1, 11); + memcpy(val1.val, val.c_str(), val.size()); + val1.len = 11; + taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); + key++; + } + transCtxMerge(ctx, src); + free(src); + } + EXPECT_EQ(4, taosHashGetSize(ctx->args)); + + char *skey = (char *)transCtxDumpVal(ctx, 1); + EXPECT_EQ(0, strcmp(skey, val.c_str())); + free(skey); + + skey = (char *)transCtxDumpVal(ctx, 2); + EXPECT_EQ(0, strcmp(skey, val.c_str())); +} #endif diff --git a/source/libs/transport/test/uv.c b/source/libs/transport/test/uv.c index 4c7d30900b..5a19b7998c 100644 --- a/source/libs/transport/test/uv.c +++ b/source/libs/transport/test/uv.c @@ -1,17 +1,16 @@ #include -#include +#include #include #include #include #include "task.h" -#include #define NUM_OF_THREAD 1 #define TIMEOUT 10000 typedef struct SThreadObj { - pthread_t thread; + TdThread thread; uv_pipe_t *pipe; uv_loop_t *loop; uv_async_t *workerAsync; // @@ -183,7 +182,7 @@ int main() { server->pThreadObj[i]->fd = fds[0]; server->pThreadObj[i]->pipe = &(server->pipe[i][1]); // init read - int err = pthread_create(&(server->pThreadObj[i]->thread), NULL, + int err = taosThreadCreate(&(server->pThreadObj[i]->thread), NULL, worker_thread, (void *)(server->pThreadObj[i])); if (err == 0) { printf("thread %d create\n", i); diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index b1caeab3b1..2da8f4f8af 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -25,7 +25,7 @@ typedef struct { int8_t inited; uint32_t seq; int32_t refSetId; - pthread_t thread; + TdThread thread; } SWalMgmt; static SWalMgmt tsWal = {0, .seq = 1}; @@ -101,7 +101,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->writeHead.head.headVer = WAL_HEAD_VER; pWal->writeHead.magic = WAL_MAGIC; - if (pthread_mutex_init(&pWal->mutex, NULL) < 0) { + if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) { taosArrayDestroy(pWal->fileInfoSet); free(pWal); return NULL; @@ -109,7 +109,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->refId = taosAddRef(tsWal.refSetId, pWal); if (pWal->refId < 0) { - pthread_mutex_destroy(&pWal->mutex); + taosThreadMutexDestroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); free(pWal); return NULL; @@ -119,7 +119,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { if (walCheckAndRepairMeta(pWal) < 0) { taosRemoveRef(tsWal.refSetId, pWal->refId); - pthread_mutex_destroy(&pWal->mutex); + taosThreadMutexDestroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); free(pWal); return NULL; @@ -156,7 +156,7 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { } void walClose(SWal *pWal) { - pthread_mutex_lock(&pWal->mutex); + taosThreadMutexLock(&pWal->mutex); taosCloseFile(&pWal->pWriteLogTFile); pWal->pWriteLogTFile = NULL; taosCloseFile(&pWal->pWriteIdxTFile); @@ -164,7 +164,7 @@ void walClose(SWal *pWal) { walSaveMeta(pWal); taosArrayDestroy(pWal->fileInfoSet); pWal->fileInfoSet = NULL; - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); taosRemoveRef(tsWal.refSetId, pWal->refId); } @@ -173,7 +173,7 @@ static void walFreeObj(void *wal) { SWal *pWal = wal; wDebug("vgId:%d, wal:%p is freed", pWal->cfg.vgId, pWal); - pthread_mutex_destroy(&pWal->mutex); + taosThreadMutexDestroy(&pWal->mutex); tfree(pWal); } @@ -223,17 +223,17 @@ static void *walThreadFunc(void *param) { } static int32_t walCreateThread() { - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&tsWal.thread, &thAttr, walThreadFunc, NULL) != 0) { + if (taosThreadCreate(&tsWal.thread, &thAttr, walThreadFunc, NULL) != 0) { wError("failed to create wal thread since %s", strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - pthread_attr_destroy(&thAttr); + taosThreadAttrDestroy(&thAttr); wDebug("wal thread is launched, thread:0x%08" PRIx64, taosGetPthreadId(tsWal.thread)); return 0; @@ -243,7 +243,7 @@ static void walStopThread() { atomic_store_8(&tsWal.stop, 1); if (taosCheckPthreadValid(tsWal.thread)) { - pthread_join(tsWal.thread, NULL); + taosThreadJoin(tsWal.thread, NULL); } wDebug("wal thread is stopped"); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 0f6ac0b214..8392e66e58 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -38,7 +38,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { terrno = TSDB_CODE_WAL_INVALID_VER; return -1; } - pthread_mutex_lock(&pWal->mutex); + taosThreadMutexLock(&pWal->mutex); // find correct file if (ver < walGetLastFileFirstVer(pWal)) { @@ -65,20 +65,20 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // TODO:change to deserialize function if (pIdxTFile == NULL) { - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); return -1; } int64_t idxOff = walGetVerIdxOffset(pWal, ver); code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET); if (code < 0) { - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); return -1; } // read idx file and get log file pos // TODO:change to deserialize function SWalIdxEntry entry; if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); return -1; } ASSERT(entry.ver == ver); @@ -87,13 +87,13 @@ int32_t walRollback(SWal *pWal, int64_t ver) { TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ); if (pLogTFile == NULL) { // TODO - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); return -1; } code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (code < 0) { // TODO - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); return -1; } // validate offset @@ -127,7 +127,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; // unlock - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); return 0; } @@ -283,7 +283,7 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in ASSERT(pWal->writeCur >= 0); - pthread_mutex_lock(&pWal->mutex); + taosThreadMutexLock(&pWal->mutex); if (pWal->pWriteIdxTFile == NULL || pWal->pWriteLogTFile == NULL) { walSetWrite(pWal); @@ -327,7 +327,7 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in walGetCurFileInfo(pWal)->lastVer = index; walGetCurFileInfo(pWal)->fileSize += sizeof(SWalHead) + bodyLen; - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); return 0; } diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 63fa600217..61b2593bc6 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -101,6 +101,8 @@ void osUpdate() { } } +void osCleanup() {} + bool osLogSpaceAvailable() { return tsLogSpace.reserved <= tsLogSpace.size.avail; } void osSetTimezone(const char *timezone) { taosSetSystemTimezone(tsTimezone, tsTimezone, &tsDaylight); } diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 5e859de5d6..4dece8abef 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -57,7 +57,7 @@ typedef int32_t SocketFd; typedef int32_t FileFd; typedef struct TdFile { - pthread_rwlock_t rwlock; + TdThreadRwlock rwlock; int refId; FileFd fd; FILE *fp; @@ -275,7 +275,7 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { return NULL; } #if FILE_WITH_LOCK - pthread_rwlock_init(&(pFile->rwlock), NULL); + taosThreadRwlockInit(&(pFile->rwlock), NULL); #endif pFile->fd = fd; pFile->fp = fp; @@ -292,7 +292,7 @@ int64_t taosCloseFile(TdFilePtr *ppFile) { return 0; } #if FILE_WITH_LOCK - pthread_rwlock_wrlock(&((*ppFile)->rwlock)); + taosThreadRwlockWrlock(&((*ppFile)->rwlock)); #endif if (ppFile == NULL || *ppFile == NULL || (*ppFile)->fd == -1) { return 0; @@ -309,8 +309,8 @@ int64_t taosCloseFile(TdFilePtr *ppFile) { } (*ppFile)->refId = 0; #if FILE_WITH_LOCK - pthread_rwlock_unlock(&((*ppFile)->rwlock)); - pthread_rwlock_destroy(&((*ppFile)->rwlock)); + taosThreadRwlockUnlock(&((*ppFile)->rwlock)); + taosThreadRwlockDestroy(&((*ppFile)->rwlock)); #endif free(*ppFile); *ppFile = NULL; @@ -323,7 +323,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { return 0; } #if FILE_WITH_LOCK - pthread_rwlock_rdlock(&(pFile->rwlock)); + taosThreadRwlockRdlock(&(pFile->rwlock)); #endif assert(pFile->fd >= 0); int64_t leftbytes = count; @@ -337,13 +337,13 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { continue; } else { #if FILE_WITH_LOCK - pthread_rwlock_unlock(&(pFile->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return -1; } } else if (readbytes == 0) { #if FILE_WITH_LOCK - pthread_rwlock_unlock(&(pFile->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return (int64_t)(count - leftbytes); } @@ -353,7 +353,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { } #if FILE_WITH_LOCK - pthread_rwlock_unlock(&(pFile->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return count; } @@ -363,12 +363,12 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) return 0; } #if FILE_WITH_LOCK - pthread_rwlock_rdlock(&(pFile->rwlock)); + taosThreadRwlockRdlock(&(pFile->rwlock)); #endif assert(pFile->fd >= 0); int64_t ret = pread(pFile->fd, buf, count, offset); #if FILE_WITH_LOCK - pthread_rwlock_unlock(&(pFile->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return ret; } @@ -378,7 +378,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { return 0; } #if FILE_WITH_LOCK - pthread_rwlock_wrlock(&(pFile->rwlock)); + taosThreadRwlockWrlock(&(pFile->rwlock)); #endif assert(pFile->fd >= 0); @@ -393,7 +393,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { continue; } #if FILE_WITH_LOCK - pthread_rwlock_unlock(&(pFile->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return -1; } @@ -402,7 +402,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { } #if FILE_WITH_LOCK - pthread_rwlock_unlock(&(pFile->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return count; } @@ -412,12 +412,12 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { return 0; } #if FILE_WITH_LOCK - pthread_rwlock_rdlock(&(pFile->rwlock)); + taosThreadRwlockRdlock(&(pFile->rwlock)); #endif assert(pFile->fd >= 0); int64_t ret = lseek(pFile->fd, (long)offset, whence); #if FILE_WITH_LOCK - pthread_rwlock_unlock(&(pFile->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return ret; } diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index e5b506e8d8..b41ef898d4 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -24,11 +24,11 @@ #include -bool taosCheckPthreadValid(pthread_t thread) { return thread.p != NULL; } +bool taosCheckPthreadValid(TdThread thread) { return thread.p != NULL; } -void taosResetPthread(pthread_t* thread) { thread->p = 0; } +void taosResetPthread(TdThread* thread) { thread->p = 0; } -int64_t taosGetPthreadId(pthread_t thread) { +int64_t taosGetPthreadId(TdThread thread) { #ifdef PTW32_VERSION return pthread_getw32threadid_np(thread); #else @@ -38,7 +38,7 @@ int64_t taosGetPthreadId(pthread_t thread) { int64_t taosGetSelfPthreadId() { return GetCurrentThreadId(); } -bool taosComparePthread(pthread_t first, pthread_t second) { return first.p == second.p; } +bool taosComparePthread(TdThread first, TdThread second) { return first.p == second.p; } int32_t taosGetPId() { return GetCurrentProcessId(); } @@ -84,10 +84,9 @@ int32_t tsem_wait(tsem_t* sem) { #include #include #include -#include -static pthread_t sem_thread; -static pthread_once_t sem_once; +static TdThread sem_thread; +static TdThreadOnce sem_once; static task_t sem_port; static volatile int sem_inited = 0; static semaphore_t sem_exit; @@ -110,7 +109,7 @@ static void *sem_thread_routine(void *arg) { static void once_init(void) { int r = 0; - r = pthread_create(&sem_thread, NULL, sem_thread_routine, NULL); + r = taosThreadCreate(&sem_thread, NULL, sem_thread_routine, NULL); if (r) { fprintf(stderr, "==%s[%d]%s()==failed to create thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__); return; @@ -123,8 +122,8 @@ static void once_init(void) { struct tsem_s { #ifdef SEM_USE_PTHREAD - pthread_mutex_t lock; - pthread_cond_t cond; + TdThreadMutex lock; + TdThreadCond cond; volatile int64_t val; #elif defined(SEM_USE_POSIX) size_t id; @@ -151,12 +150,12 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) { } #ifdef SEM_USE_PTHREAD - int r = pthread_mutex_init(&p->lock, NULL); + int r = taosThreadMutexInit(&p->lock, NULL); do { if (r) break; - r = pthread_cond_init(&p->cond, NULL); + r = taosThreadCondInit(&p->cond, NULL); if (r) { - pthread_mutex_destroy(&p->lock); + taosThreadMutexDestroy(&p->lock); break; } p->val = value; @@ -186,7 +185,7 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) { abort(); } while (p->sem == SEM_FAILED); #elif defined(SEM_USE_SEM) - pthread_once(&sem_once, once_init); + taosThreadOnce(&sem_once, once_init); if (sem_inited != 1) { fprintf(stderr, "==%s[%d]%s():[%p]==internal resource init failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); errno = ENOMEM; @@ -224,18 +223,18 @@ int tsem_wait(tsem_t *sem) { abort(); } #ifdef SEM_USE_PTHREAD - if (pthread_mutex_lock(&p->lock)) { + if (taosThreadMutexLock(&p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } p->val -= 1; if (p->val < 0) { - if (pthread_cond_wait(&p->cond, &p->lock)) { + if (taosThreadCondWait(&p->cond, &p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } } - if (pthread_mutex_unlock(&p->lock)) { + if (taosThreadMutexUnlock(&p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } @@ -260,18 +259,18 @@ int tsem_post(tsem_t *sem) { abort(); } #ifdef SEM_USE_PTHREAD - if (pthread_mutex_lock(&p->lock)) { + if (taosThreadMutexLock(&p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } p->val += 1; if (p->val <= 0) { - if (pthread_cond_signal(&p->cond)) { + if (taosThreadCondSignal(&p->cond)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } } - if (pthread_mutex_unlock(&p->lock)) { + if (taosThreadMutexUnlock(&p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } @@ -299,20 +298,20 @@ int tsem_destroy(tsem_t *sem) { return 0; } #ifdef SEM_USE_PTHREAD - if (pthread_mutex_lock(&p->lock)) { + if (taosThreadMutexLock(&p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } p->valid = 0; - if (pthread_cond_destroy(&p->cond)) { + if (taosThreadCondDestroy(&p->cond)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } - if (pthread_mutex_unlock(&p->lock)) { + if (taosThreadMutexUnlock(&p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } - if (pthread_mutex_destroy(&p->lock)) { + if (taosThreadMutexDestroy(&p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } @@ -338,23 +337,23 @@ int tsem_destroy(tsem_t *sem) { return 0; } -bool taosCheckPthreadValid(pthread_t thread) { +bool taosCheckPthreadValid(TdThread thread) { uint64_t id = 0; - int r = pthread_threadid_np(thread, &id); + int r = TdThreadhreadid_np(thread, &id); return r ? false : true; } int64_t taosGetSelfPthreadId() { uint64_t id; - pthread_threadid_np(0, &id); + TdThreadhreadid_np(0, &id); return (int64_t)id; } -int64_t taosGetPthreadId(pthread_t thread) { return (int64_t)thread; } +int64_t taosGetPthreadId(TdThread thread) { return (int64_t)thread; } -void taosResetPthread(pthread_t *thread) { *thread = NULL; } +void taosResetPthread(TdThread *thread) { *thread = NULL; } -bool taosComparePthread(pthread_t first, pthread_t second) { return pthread_equal(first, second) ? true : false; } +bool taosComparePthread(TdThread first, TdThread second) { return taosThreadEqual(first, second) ? true : false; } int32_t taosGetPId() { return (int32_t)getpid(); } @@ -378,7 +377,7 @@ int32_t taosGetAppName(char *name, int32_t *len) { #include #include -bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; } +bool taosCheckPthreadValid(TdThread thread) { return thread != 0; } int64_t taosGetSelfPthreadId() { static __thread int id = 0; @@ -387,9 +386,9 @@ int64_t taosGetSelfPthreadId() { return id; } -int64_t taosGetPthreadId(pthread_t thread) { return (int64_t)thread; } -void taosResetPthread(pthread_t* thread) { *thread = 0; } -bool taosComparePthread(pthread_t first, pthread_t second) { return first == second; } +int64_t taosGetPthreadId(TdThread thread) { return (int64_t)thread; } +void taosResetPthread(TdThread* thread) { *thread = 0; } +bool taosComparePthread(TdThread first, TdThread second) { return first == second; } int32_t taosGetPId() { return getpid(); } int32_t taosGetAppName(char* name, int32_t* len) { diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index e3b1015d88..e5f83fa167 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -52,7 +52,7 @@ typedef SocketFd EpollFd; typedef struct TdSocketServer { #if SOCKET_WITH_LOCK - pthread_rwlock_t rwlock; + TdThreadRwlock rwlock; #endif int refId; SocketFd fd; @@ -60,7 +60,7 @@ typedef struct TdSocketServer { typedef struct TdSocket { #if SOCKET_WITH_LOCK - pthread_rwlock_t rwlock; + TdThreadRwlock rwlock; #endif int refId; SocketFd fd; @@ -68,7 +68,7 @@ typedef struct TdSocket { typedef struct TdEpoll { #if SOCKET_WITH_LOCK - pthread_rwlock_t rwlock; + TdThreadRwlock rwlock; #endif int refId; EpollFd fd; @@ -775,7 +775,7 @@ void taosBlockSIGPIPE() { sigset_t signal_mask; sigemptyset(&signal_mask); sigaddset(&signal_mask, SIGPIPE); - int32_t rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL); + int32_t rc = taosThreadSigmask(SIG_BLOCK, &signal_mask, NULL); if (rc != 0) { // printf("failed to block SIGPIPE"); } @@ -893,7 +893,7 @@ void taosSetMaskSIGPIPE() { sigset_t signal_mask; sigemptyset(&signal_mask); sigaddset(&signal_mask, SIGPIPE); - int32_t rc = pthread_sigmask(SIG_SETMASK, &signal_mask, NULL); + int32_t rc = taosThreadSigmask(SIG_SETMASK, &signal_mask, NULL); if (rc != 0) { // printf("failed to setmask SIGPIPE"); } diff --git a/source/os/src/osThread.c b/source/os/src/osThread.c new file mode 100644 index 0000000000..716080274b --- /dev/null +++ b/source/os/src/osThread.c @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define ALLOW_FORBID_FUNC +#include "os.h" + +// int32_t taosThreadSetnameNp(TdThread thread, const char *name) { +// return pthread_setname_np(thread,name); +// } + +int32_t taosThreadSpinInit(TdThreadSpinlock *lock, int pshared) { + return pthread_spin_init(lock, pshared); +} + +int32_t taosThreadMutexInit(TdThreadMutex *mutex, const TdThreadMutexAttr *attr) { + return pthread_mutex_init(mutex, attr); +} + +int32_t taosThreadSpinDestroy(TdThreadSpinlock *lock) { + return pthread_spin_destroy(lock); +} + +int32_t taosThreadMutexDestroy(TdThreadMutex * mutex) { + return pthread_mutex_destroy(mutex); +} + +int32_t taosThreadSpinLock(TdThreadSpinlock *lock) { + return pthread_spin_lock(lock); +} + +int32_t taosThreadMutexLock(TdThreadMutex *mutex) { + return pthread_mutex_lock(mutex); +} + +int32_t taosThreadSpinUnlock(TdThreadSpinlock *lock) { + return pthread_spin_unlock(lock); +} + +int32_t taosThreadMutexUnlock(TdThreadMutex *mutex) { + return pthread_mutex_unlock(mutex); +} + +int32_t taosThreadRwlockRdlock(TdThreadRwlock *rwlock) { + return pthread_rwlock_rdlock(rwlock); +} + +int32_t taosThreadRwlockWrlock(TdThreadRwlock *rwlock) { + return pthread_rwlock_wrlock(rwlock); +} + +int32_t taosThreadRwlockUnlock(TdThreadRwlock *rwlock) { + return pthread_rwlock_unlock(rwlock); +} + +void taosThreadTestCancel(void) { + return pthread_testcancel(); +} + +int32_t taosThreadAttrInit(TdThreadAttr *attr) { + return pthread_attr_init(attr); +} + +int32_t taosThreadCreate(TdThread *tid, const TdThreadAttr *attr, void*(*start)(void*), void *arg) { + return pthread_create(tid, attr, start, arg); +} + +int32_t taosThreadOnce(TdThreadOnce *onceControl, void(*initRoutine)(void)) { + return pthread_once(onceControl, initRoutine); +} + +int32_t taosThreadAttrSetDetachState(TdThreadAttr *attr, int32_t detachState) { + return pthread_attr_setdetachstate(attr, detachState); +} + +int32_t taosThreadAttrDestroy(TdThreadAttr *attr) { + return pthread_attr_destroy(attr); +} + +int32_t taosThreadJoin(TdThread thread, void **pValue) { + return pthread_join(thread, pValue); +} + +int32_t taosThreadRwlockInit(TdThreadRwlock *rwlock, const TdThreadRwlockAttr *attr) { + return pthread_rwlock_init(rwlock, attr); +} + +int32_t taosThreadRwlockDestroy(TdThreadRwlock *rwlock) { + return pthread_rwlock_destroy(rwlock); +} + +int32_t taosThreadCondSignal(TdThreadCond *cond) { + return pthread_cond_signal(cond); +} + +int32_t taosThreadCondInit(TdThreadCond *cond, const TdThreadCondAttr *attr) { + return pthread_cond_init(cond, attr); +} + +int32_t taosThreadCondBroadcast(TdThreadCond *cond) { + return pthread_cond_broadcast(cond); +} + +int32_t taosThreadCondDestroy(TdThreadCond *cond) { + return pthread_cond_destroy(cond); +} + +int32_t taosThreadCondWait(TdThreadCond *cond, TdThreadMutex *mutex) { + return pthread_cond_wait(cond, mutex); +} + +TdThread taosThreadSelf(void) { + return pthread_self(); +} + +// int32_t taosThreadGetW32ThreadIdNp(TdThread thread) { +// return pthread_getw32threadid_np(thread); +// } + +int32_t taosThreadEqual(TdThread t1, TdThread t2) { + return pthread_equal(t1, t2); +} + +int32_t taosThreadSigmask(int how, sigset_t const *set, sigset_t *oset) { + return pthread_sigmask(how, set, oset); +} + +int32_t taosThreadCancel(TdThread thread) { + return pthread_cancel(thread); +} + +int32_t taosThreadKill(TdThread thread, int sig) { + return pthread_kill(thread, sig); +} \ No newline at end of file diff --git a/source/os/src/osTimer.c b/source/os/src/osTimer.c index c95ca72bd5..e6e3ec7962 100644 --- a/source/os/src/osTimer.c +++ b/source/os/src/osTimer.c @@ -44,7 +44,7 @@ static MMRESULT timerId; static void (*timer_callback)(int); static int timer_ms = 0; -static pthread_t timer_thread; +static TdThread timer_thread; static int timer_kq = -1; static volatile int timer_stop = 0; @@ -83,7 +83,7 @@ static void taosDeleteTimer(void *tharg) { timer_delete(*pTimer); } -static pthread_t timerThread; +static TdThread timerThread; static timer_t timerId; static volatile bool stopTimer = false; static void * taosProcessAlarmSignal(void *tharg) { @@ -112,7 +112,7 @@ static void * taosProcessAlarmSignal(void *tharg) { // printf("Failed to create timer"); } - pthread_cleanup_push(taosDeleteTimer, &timerId); + taosThreadCleanupPush(taosDeleteTimer, &timerId); struct itimerspec ts; ts.it_value.tv_sec = 0; @@ -136,7 +136,7 @@ static void * taosProcessAlarmSignal(void *tharg) { callback(0); } - pthread_cleanup_pop(1); + taosThreadCleanupPop(1); return NULL; } @@ -165,7 +165,7 @@ int taosInitTimer(void (*callback)(int), int ms) { abort(); } - r = pthread_create(&timer_thread, NULL, timer_routine, NULL); + r = taosThreadCreate(&timer_thread, NULL, timer_routine, NULL); if (r) { fprintf(stderr, "==%s[%d]%s()==failed to create timer thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__); // since no caller of this func checks the return value for the moment @@ -174,10 +174,10 @@ int taosInitTimer(void (*callback)(int), int ms) { return 0; #else stopTimer = false; - pthread_attr_t tattr; - pthread_attr_init(&tattr); - int code = pthread_create(&timerThread, &tattr, taosProcessAlarmSignal, callback); - pthread_attr_destroy(&tattr); + TdThreadAttr tattr; + taosThreadAttrInit(&tattr); + int code = taosThreadCreate(&timerThread, &tattr, taosProcessAlarmSignal, callback); + taosThreadAttrDestroy(&tattr); if (code != 0) { // printf("failed to create timer thread"); return -1; @@ -195,7 +195,7 @@ void taosUninitTimer() { #elif defined(_TD_DARWIN_64) int r = 0; timer_stop = 1; - r = pthread_join(timer_thread, NULL); + r = taosThreadJoin(timer_thread, NULL); if (r) { fprintf(stderr, "==%s[%d]%s()==failed to join timer thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__); // since no caller of this func checks the return value for the moment @@ -207,7 +207,7 @@ void taosUninitTimer() { stopTimer = true; // printf("join timer thread:0x%08" PRIx64, taosGetPthreadId(timerThread)); - pthread_join(timerThread, NULL); + taosThreadJoin(timerThread, NULL); #endif } diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index 864a60b706..bdafa63d64 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE #include "os.h" @@ -48,16 +49,11 @@ void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight) { if (inTimezone == NULL || inTimezone[0] == 0) return; -#ifdef WINDOWS +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) char winStr[TD_LOCALE_LEN * 2]; sprintf(winStr, "TZ=%s", inTimezone); putenv(winStr); -#else - setenv("TZ", inTimezone, 1); -#endif tzset(); - - /* * get CURRENT time zone. * system current time zone is affected by daylight saving time(DST) * @@ -75,15 +71,34 @@ void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *ou int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); tz += daylight; - /* * format: * (CST, +0800) * (BST, +0100) */ + sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + *outDaylight = daylight; + +#elif defined(_TD_DARWIN_64) + + setenv("TZ", inTimezone, 1); + tzset(); + int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); + tz += daylight; sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); *outDaylight = daylight; + +#else + setenv("TZ", inTimezone, 1); + tzset(); + int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); + tz += daylight; + sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + *outDaylight = daylight; + +#endif + } void taosGetSystemTimezone(char *outTimezone) { diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index 08fa0a2409..7a47639e75 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -5,10 +5,6 @@ target_include_directories( util PUBLIC "${CMAKE_SOURCE_DIR}/include/util" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -IF(${TD_WINDOWS}) - PRIVATE "${CMAKE_SOURCE_DIR}/contrib/pthread-win32" - PRIVATE "${CMAKE_SOURCE_DIR}/contrib/gnuregex" -ENDIF () ) target_link_libraries( util diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 51ea2b1f13..9a1b213534 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -22,9 +22,9 @@ #define CACHE_MAX_CAPACITY 1024*1024*16 #define CACHE_DEFAULT_CAPACITY 1024*4 -static pthread_t cacheRefreshWorker = {0}; -static pthread_once_t cacheThreadInit = PTHREAD_ONCE_INIT; -static pthread_mutex_t guard = PTHREAD_MUTEX_INITIALIZER; +static TdThread cacheRefreshWorker = {0}; +static TdThreadOnce cacheThreadInit = PTHREAD_ONCE_INIT; +static TdThreadMutex guard = PTHREAD_MUTEX_INITIALIZER; static SArray *pCacheArrayList = NULL; static bool stopRefreshWorker = false; static bool refreshWorkerNormalStopped = false; @@ -90,13 +90,13 @@ struct SCacheObj { STrashElem *pTrash; uint8_t deleting; // set the deleting flag to stop refreshing ASAP. - pthread_t refreshWorker; + TdThread refreshWorker; bool extendLifespan; // auto extend life span when one item is accessed. int64_t checkTick; // tick used to record the check times of the refresh threads #if defined(LINUX) - pthread_rwlock_t lock; + TdThreadRwlock lock; #else - pthread_mutex_t lock; + TdThreadMutex lock; #endif }; @@ -109,33 +109,33 @@ typedef struct SCacheObjTravSup { static FORCE_INLINE void __trashcan_wr_lock(SCacheObj *pCacheObj) { #if defined(LINUX) - pthread_rwlock_wrlock(&pCacheObj->lock); + taosThreadRwlockWrlock(&pCacheObj->lock); #else - pthread_mutex_lock(&pCacheObj->lock); + taosThreadMutexLock(&pCacheObj->lock); #endif } static FORCE_INLINE void __trashcan_unlock(SCacheObj *pCacheObj) { #if defined(LINUX) - pthread_rwlock_unlock(&pCacheObj->lock); + taosThreadRwlockUnlock(&pCacheObj->lock); #else - pthread_mutex_unlock(&pCacheObj->lock); + taosThreadMutexUnlock(&pCacheObj->lock); #endif } static FORCE_INLINE int32_t __trashcan_lock_init(SCacheObj *pCacheObj) { #if defined(LINUX) - return pthread_rwlock_init(&pCacheObj->lock, NULL); + return taosThreadRwlockInit(&pCacheObj->lock, NULL); #else - return pthread_mutex_init(&pCacheObj->lock, NULL); + return taosThreadMutexInit(&pCacheObj->lock, NULL); #endif } static FORCE_INLINE void __trashcan_lock_destroy(SCacheObj *pCacheObj) { #if defined(LINUX) - pthread_rwlock_destroy(&pCacheObj->lock); + taosThreadRwlockDestroy(&pCacheObj->lock); #else - pthread_mutex_destroy(&pCacheObj->lock); + taosThreadMutexDestroy(&pCacheObj->lock); #endif } @@ -154,20 +154,20 @@ static void *taosCacheTimedRefresh(void *handle); static void doInitRefreshThread(void) { pCacheArrayList = taosArrayInit(4, POINTER_BYTES); - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - pthread_create(&cacheRefreshWorker, &thattr, taosCacheTimedRefresh, NULL); - pthread_attr_destroy(&thattr); + taosThreadCreate(&cacheRefreshWorker, &thattr, taosCacheTimedRefresh, NULL); + taosThreadAttrDestroy(&thattr); } -pthread_t doRegisterCacheObj(SCacheObj *pCacheObj) { - pthread_once(&cacheThreadInit, doInitRefreshThread); +TdThread doRegisterCacheObj(SCacheObj *pCacheObj) { + taosThreadOnce(&cacheThreadInit, doInitRefreshThread); - pthread_mutex_lock(&guard); + taosThreadMutexLock(&guard); taosArrayPush(pCacheArrayList, &pCacheObj); - pthread_mutex_unlock(&guard); + taosThreadMutexUnlock(&guard); return cacheRefreshWorker; } @@ -836,19 +836,19 @@ void *taosCacheTimedRefresh(void *handle) { goto _end; } - pthread_mutex_lock(&guard); + taosThreadMutexLock(&guard); size_t size = taosArrayGetSize(pCacheArrayList); - pthread_mutex_unlock(&guard); + taosThreadMutexUnlock(&guard); count += 1; for (int32_t i = 0; i < size; ++i) { - pthread_mutex_lock(&guard); + taosThreadMutexLock(&guard); SCacheObj *pCacheObj = taosArrayGetP(pCacheArrayList, i); if (pCacheObj == NULL) { uError("object is destroyed. ignore and try next"); - pthread_mutex_unlock(&guard); + taosThreadMutexUnlock(&guard); continue; } @@ -860,11 +860,11 @@ void *taosCacheTimedRefresh(void *handle) { uDebug("%s is destroying, remove it from refresh list, remain cache obj:%" PRIzu, pCacheObj->name, size); pCacheObj->deleting = 0; // reset the deleting flag to enable pCacheObj to continue releasing resources. - pthread_mutex_unlock(&guard); + taosThreadMutexUnlock(&guard); continue; } - pthread_mutex_unlock(&guard); + taosThreadMutexUnlock(&guard); if ((count % pCacheObj->checkTick) != 0) { continue; @@ -892,7 +892,7 @@ _end: taosArrayDestroy(pCacheArrayList); pCacheArrayList = NULL; - pthread_mutex_destroy(&guard); + taosThreadMutexDestroy(&guard); refreshWorkerNormalStopped = true; uDebug("cache refresh thread quits"); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 469b944bf9..8dd41a8fa8 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -84,6 +84,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PARA, "Invalid parameters") TAOS_DEFINE_ERROR(TSDB_CODE_REPEAT_INIT, "Repeat initialization") TAOS_DEFINE_ERROR(TSDB_CODE_CFG_NOT_FOUND, "Config not found") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CFG, "Invalid config option") +TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_SHM_MEM, "Out of Share memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") @@ -273,33 +274,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_UNSUPPORTED_TOPIC, "Topic with aggregatio TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_DND_OFFLINE, "Dnode is offline") TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, "Invalid message length") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_DNODE_READ_FILE_ERROR, "Read dnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_DNODE_WRITE_FILE_ERROR, "Write dnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED, "Mnode already deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_NOT_DEPLOYED, "Mnode not deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_INVALID_OPTION, "Mnode option invalid") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_READ_FILE_ERROR, "Read mnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR, "Write mnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED, "Qnode already deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_QNODE_NOT_DEPLOYED, "Qnode not deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_QNODE_INVALID_OPTION, "Qnode option invalid") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_QNODE_READ_FILE_ERROR, "Read qnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR, "Write qnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED, "Snode already deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_SNODE_NOT_DEPLOYED, "Snode not deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_SNODE_INVALID_OPTION, "Snode option invalid") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_SNODE_READ_FILE_ERROR, "Read snode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR, "Write snode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED, "Bnode already deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_NOT_DEPLOYED, "Bnode not deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_INVALID_OPTION, "Bnode option invalid") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_READ_FILE_ERROR, "Read bnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR, "Write bnode.json error") +TAOS_DEFINE_ERROR(TSDB_CODE_NODE_ALREADY_DEPLOYED, "Node already deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_NODE_NOT_DEPLOYED, "Node not deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_NODE_PARSE_FILE_ERROR, "Invalid json format") +TAOS_DEFINE_ERROR(TSDB_CODE_NODE_INVALID_OPTION, "Invalid node option") TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED, "Vnode already deployed") TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_NOT_DEPLOYED, "Vnode not deployed") TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_INVALID_OPTION, "Vnode option invalid") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_READ_FILE_ERROR, "Read vnodes.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR, "Write vnodes.json error") TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_TOO_MANY_VNODES, "Too many vnodes") // vnode @@ -455,14 +436,14 @@ static int32_t taosCompareTaosError(const void* a, const void* b) { return 0; } -static pthread_once_t tsErrorInit = PTHREAD_ONCE_INIT; +static TdThreadOnce tsErrorInit = PTHREAD_ONCE_INIT; static void tsSortError(void) { qsort(errors, sizeof(errors) / sizeof(errors[0]), sizeof(errors[0]), taosCompareTaosError); } const char* tstrerror(int32_t err) { - pthread_once(&tsErrorInit, tsSortError); + taosThreadOnce(&tsErrorInit, tsSortError); // this is a system errno if ((err & 0x00ff0000) == 0x00ff0000) { diff --git a/source/util/src/tidpool.c b/source/util/src/tidpool.c index 3ae537eae8..705cb0d2d3 100644 --- a/source/util/src/tidpool.c +++ b/source/util/src/tidpool.c @@ -31,7 +31,7 @@ void *taosInitIdPool(int32_t maxId) { pIdPool->numOfFree = maxId; pIdPool->freeSlot = 0; - pthread_mutex_init(&pIdPool->mutex, NULL); + taosThreadMutexInit(&pIdPool->mutex, NULL); uDebug("pool:%p is setup, maxId:%d", pIdPool, pIdPool->maxId); @@ -42,7 +42,7 @@ int32_t taosAllocateId(id_pool_t *pIdPool) { if (pIdPool == NULL) return -1; int32_t slot = -1; - pthread_mutex_lock(&pIdPool->mutex); + taosThreadMutexLock(&pIdPool->mutex); if (pIdPool->numOfFree > 0) { for (int32_t i = 0; i < pIdPool->maxId; ++i) { @@ -56,14 +56,14 @@ int32_t taosAllocateId(id_pool_t *pIdPool) { } } - pthread_mutex_unlock(&pIdPool->mutex); + taosThreadMutexUnlock(&pIdPool->mutex); return slot + 1; } void taosFreeId(id_pool_t *pIdPool, int32_t id) { if (pIdPool == NULL) return; - pthread_mutex_lock(&pIdPool->mutex); + taosThreadMutexLock(&pIdPool->mutex); int32_t slot = (id - 1) % pIdPool->maxId; if (pIdPool->freeList[slot]) { @@ -71,7 +71,7 @@ void taosFreeId(id_pool_t *pIdPool, int32_t id) { pIdPool->numOfFree++; } - pthread_mutex_unlock(&pIdPool->mutex); + taosThreadMutexUnlock(&pIdPool->mutex); } void taosIdPoolCleanUp(id_pool_t *pIdPool) { @@ -81,7 +81,7 @@ void taosIdPoolCleanUp(id_pool_t *pIdPool) { if (pIdPool->freeList) free(pIdPool->freeList); - pthread_mutex_destroy(&pIdPool->mutex); + taosThreadMutexDestroy(&pIdPool->mutex); memset(pIdPool, 0, sizeof(id_pool_t)); @@ -89,16 +89,16 @@ void taosIdPoolCleanUp(id_pool_t *pIdPool) { } int32_t taosIdPoolNumOfUsed(id_pool_t *pIdPool) { - pthread_mutex_lock(&pIdPool->mutex); + taosThreadMutexLock(&pIdPool->mutex); int32_t ret = pIdPool->maxId - pIdPool->numOfFree; - pthread_mutex_unlock(&pIdPool->mutex); + taosThreadMutexUnlock(&pIdPool->mutex); return ret; } bool taosIdPoolMarkStatus(id_pool_t *pIdPool, int32_t id) { bool ret = false; - pthread_mutex_lock(&pIdPool->mutex); + taosThreadMutexLock(&pIdPool->mutex); int32_t slot = (id - 1) % pIdPool->maxId; if (!pIdPool->freeList[slot]) { @@ -109,7 +109,7 @@ bool taosIdPoolMarkStatus(id_pool_t *pIdPool, int32_t id) { ret = false; } - pthread_mutex_unlock(&pIdPool->mutex); + taosThreadMutexUnlock(&pIdPool->mutex); return ret; } @@ -123,7 +123,7 @@ int32_t taosUpdateIdPool(id_pool_t *pIdPool, int32_t maxId) { return -1; } - pthread_mutex_lock(&pIdPool->mutex); + taosThreadMutexLock(&pIdPool->mutex); memcpy(idList, pIdPool->freeList, sizeof(bool) * pIdPool->maxId); pIdPool->numOfFree += (maxId - pIdPool->maxId); @@ -133,15 +133,15 @@ int32_t taosUpdateIdPool(id_pool_t *pIdPool, int32_t maxId) { pIdPool->freeList = idList; free(oldIdList); - pthread_mutex_unlock(&pIdPool->mutex); + taosThreadMutexUnlock(&pIdPool->mutex); return 0; } int32_t taosIdPoolMaxSize(id_pool_t *pIdPool) { - pthread_mutex_lock(&pIdPool->mutex); + taosThreadMutexLock(&pIdPool->mutex); int32_t ret = pIdPool->maxId; - pthread_mutex_unlock(&pIdPool->mutex); + taosThreadMutexUnlock(&pIdPool->mutex); return ret; } \ No newline at end of file diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 0a4a1a07a6..a85da8cbbf 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -202,9 +202,16 @@ int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pV return (errno == ERANGE||errno == EINVAL) ? TSDB_CODE_FAILED:TSDB_CODE_SUCCESS; } +int32_t tjsonGetUIntValue(const SJson* pJson, const char* pName, uint32_t* pVal) { + uint64_t val = 0; + int32_t code = tjsonGetUBigIntValue(pJson, pName, &val); + *pVal = val; + return code; +} + int32_t tjsonGetUTinyIntValue(const SJson* pJson, const char* pName, uint8_t* pVal) { uint64_t val = 0; - int32_t code = tjsonGetUBigIntValue(pJson, pName, &val); + int32_t code = tjsonGetUBigIntValue(pJson, pName, &val); *pVal = val; return code; } @@ -239,6 +246,22 @@ int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, voi return func(pJsonObj, pObj); } +int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, void** pObj, int32_t objSize) { + if (objSize <= 0) { + return TSDB_CODE_SUCCESS; + } + + SJson* pJsonObj = tjsonGetObjectItem(pJson, pName); + if (NULL == pJsonObj) { + return TSDB_CODE_FAILED; + } + *pObj = calloc(1, objSize); + if (NULL == *pObj) { + return TSDB_CODE_OUT_OF_MEMORY; + } + return func(pJsonObj, *pObj); +} + int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void* pArray, int32_t itemSize) { const cJSON* jArray = tjsonGetObjectItem(pJson, pName); int32_t size = (NULL == jArray ? 0 : tjsonGetArraySize(jArray)); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 6dedc3f740..85db7883cf 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -45,8 +45,8 @@ typedef struct { int32_t minBuffSize; TdFilePtr pFile; int32_t stop; - pthread_t asyncThread; - pthread_mutex_t buffMutex; + TdThread asyncThread; + TdThreadMutex buffMutex; tsem_t buffNotEmpty; } SLogBuff; @@ -59,7 +59,7 @@ typedef struct { pid_t pid; char logName[LOG_FILE_NAME_LEN]; SLogBuff *logHandle; - pthread_mutex_t logMutex; + TdThreadMutex logMutex; } SLogObj; static int8_t tsLogInited = 0; @@ -107,12 +107,12 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) static int32_t taosCompressFile(char *srcFileName, char *destFileName); static int32_t taosStartLog() { - pthread_attr_t threadAttr; - pthread_attr_init(&threadAttr); - if (pthread_create(&(tsLogObj.logHandle->asyncThread), &threadAttr, taosAsyncOutputLog, tsLogObj.logHandle) != 0) { + TdThreadAttr threadAttr; + taosThreadAttrInit(&threadAttr); + if (taosThreadCreate(&(tsLogObj.logHandle->asyncThread), &threadAttr, taosAsyncOutputLog, tsLogObj.logHandle) != 0) { return -1; } - pthread_attr_destroy(&threadAttr); + taosThreadAttrDestroy(&threadAttr); return 0; } @@ -139,8 +139,9 @@ static void taosStopLog() { void taosCloseLog() { taosStopLog(); if (taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) { - pthread_join(tsLogObj.logHandle->asyncThread, NULL); + taosThreadJoin(tsLogObj.logHandle->asyncThread, NULL); } + tsLogInited = 0; // In case that other threads still use log resources causing invalid write in valgrind // we comment two lines below. // taosLogBuffDestroy(tsLogObj.logHandle); @@ -223,22 +224,22 @@ static void *taosThreadToOpenNewFile(void *param) { } static int32_t taosOpenNewLogFile() { - pthread_mutex_lock(&tsLogObj.logMutex); + taosThreadMutexLock(&tsLogObj.logMutex); if (tsLogObj.lines > tsLogObj.maxLines && tsLogObj.openInProgress == 0) { tsLogObj.openInProgress = 1; uInfo("open new log file ......"); - pthread_t thread; - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + TdThread thread; + TdThreadAttr attr; + taosThreadAttrInit(&attr); + taosThreadAttrSetDetachState(&attr, PTHREAD_CREATE_DETACHED); - pthread_create(&thread, &attr, taosThreadToOpenNewFile, NULL); - pthread_attr_destroy(&attr); + taosThreadCreate(&thread, &attr, taosThreadToOpenNewFile, NULL); + taosThreadAttrDestroy(&attr); } - pthread_mutex_unlock(&tsLogObj.logMutex); + taosThreadMutexUnlock(&tsLogObj.logMutex); return 0; } @@ -343,7 +344,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { char fileName[LOG_FILE_NAME_LEN + 50] = "\0"; sprintf(fileName, "%s.%d", tsLogObj.logName, tsLogObj.flag); - pthread_mutex_init(&tsLogObj.logMutex, NULL); + taosThreadMutexInit(&tsLogObj.logMutex, NULL); taosUmaskFile(0); tsLogObj.logHandle->pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE); @@ -517,7 +518,7 @@ static SLogBuff *taosLogBuffNew(int32_t bufSize) { tLogBuff->minBuffSize = bufSize / 10; tLogBuff->stop = 0; - if (pthread_mutex_init(&LOG_BUF_MUTEX(tLogBuff), NULL) < 0) goto _err; + if (taosThreadMutexInit(&LOG_BUF_MUTEX(tLogBuff), NULL) < 0) goto _err; // tsem_init(&(tLogBuff->buffNotEmpty), 0, 0); return tLogBuff; @@ -552,7 +553,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, const char *msg, int32_t ms if (tLogBuff == NULL || tLogBuff->stop) return -1; - pthread_mutex_lock(&LOG_BUF_MUTEX(tLogBuff)); + taosThreadMutexLock(&LOG_BUF_MUTEX(tLogBuff)); start = LOG_BUF_START(tLogBuff); end = LOG_BUF_END(tLogBuff); @@ -566,7 +567,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, const char *msg, int32_t ms if (remainSize <= msgLen || ((lostLine > 0) && (remainSize <= (msgLen + tmpBufLen)))) { lostLine++; tsAsyncLogLostLines++; - pthread_mutex_unlock(&LOG_BUF_MUTEX(tLogBuff)); + taosThreadMutexUnlock(&LOG_BUF_MUTEX(tLogBuff)); return -1; } @@ -587,7 +588,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, const char *msg, int32_t ms } */ - pthread_mutex_unlock(&LOG_BUF_MUTEX(tLogBuff)); + taosThreadMutexUnlock(&LOG_BUF_MUTEX(tLogBuff)); return 0; } diff --git a/source/util/src/tmempool.c b/source/util/src/tmempool.c index 1fc9bfc7ab..d62e903977 100644 --- a/source/util/src/tmempool.c +++ b/source/util/src/tmempool.c @@ -25,7 +25,7 @@ typedef struct { int32_t blockSize; /* block size in bytes */ int32_t *freeList; /* the index list */ char *pool; /* the actual mem block */ - pthread_mutex_t mutex; + TdThreadMutex mutex; } pool_t; mpool_h taosMemPoolInit(int32_t numOfBlock, int32_t blockSize) { @@ -58,7 +58,7 @@ mpool_h taosMemPoolInit(int32_t numOfBlock, int32_t blockSize) { return NULL; } - pthread_mutex_init(&(pool_p->mutex), NULL); + taosThreadMutexInit(&(pool_p->mutex), NULL); memset(pool_p->pool, 0, (size_t)(blockSize * numOfBlock)); for (i = 0; i < pool_p->numOfBlock; ++i) pool_p->freeList[i] = i; @@ -73,7 +73,7 @@ char *taosMemPoolMalloc(mpool_h handle) { char *pos = NULL; pool_t *pool_p = (pool_t *)handle; - pthread_mutex_lock(&(pool_p->mutex)); + taosThreadMutexLock(&(pool_p->mutex)); if (pool_p->numOfFree > 0) { pos = pool_p->pool + pool_p->blockSize * (pool_p->freeList[pool_p->first]); @@ -82,7 +82,7 @@ char *taosMemPoolMalloc(mpool_h handle) { pool_p->numOfFree--; } - pthread_mutex_unlock(&(pool_p->mutex)); + taosThreadMutexUnlock(&(pool_p->mutex)); if (pos == NULL) uDebug("mempool: out of memory"); return pos; @@ -108,18 +108,18 @@ void taosMemPoolFree(mpool_h handle, char *pMem) { memset(pMem, 0, (size_t)pool_p->blockSize); - pthread_mutex_lock(&pool_p->mutex); + taosThreadMutexLock(&pool_p->mutex); pool_p->freeList[(pool_p->first + pool_p->numOfFree) % pool_p->numOfBlock] = index; pool_p->numOfFree++; - pthread_mutex_unlock(&pool_p->mutex); + taosThreadMutexUnlock(&pool_p->mutex); } void taosMemPoolCleanUp(mpool_h handle) { pool_t *pool_p = (pool_t *)handle; - pthread_mutex_destroy(&pool_p->mutex); + taosThreadMutexDestroy(&pool_p->mutex); if (pool_p->pool) free(pool_p->pool); if (pool_p->freeList) free(pool_p->freeList); memset(pool_p, 0, sizeof(*pool_p)); diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c new file mode 100644 index 0000000000..f19a17fdb6 --- /dev/null +++ b/source/util/src/tprocess.c @@ -0,0 +1,460 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "tprocess.h" +#include "taoserror.h" +#include "tlog.h" +#include "tqueue.h" + +// todo +#include +#include + +#define SHM_DEFAULT_SIZE (20 * 1024 * 1024) +#define CEIL8(n) (ceil((float)(n) / 8) * 8) +typedef void *(*ProcThreadFp)(void *param); + +typedef struct SProcQueue { + int32_t head; + int32_t tail; + int32_t total; + int32_t avail; + int32_t items; + char *pBuffer; + ProcMallocFp mallocHeadFp; + ProcFreeFp freeHeadFp; + ProcMallocFp mallocBodyFp; + ProcFreeFp freeBodyFp; + ProcConsumeFp consumeFp; + void *pParent; + tsem_t sem; + TdThreadMutex *mutex; + int32_t mutexShmid; + int32_t bufferShmid; + const char *name; +} SProcQueue; + +typedef struct SProcObj { + TdThread childThread; + SProcQueue *pChildQueue; + TdThread parentThread; + SProcQueue *pParentQueue; + const char *name; + int32_t pid; + bool isChild; + bool stopFlag; + bool testFlag; +} SProcObj; + +static int32_t taosProcInitMutex(TdThreadMutex **ppMutex, int32_t *pShmid) { + TdThreadMutex *pMutex = NULL; + TdThreadMutexAttr mattr = {0}; + int32_t shmid = -1; + int32_t code = -1; + + if (pthread_mutexattr_init(&mattr) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init mutex while init attr since %s", terrstr()); + goto _OVER; + } + + if (pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init mutex while set shared since %s", terrstr()); + goto _OVER; + } + + shmid = shmget(IPC_PRIVATE, sizeof(TdThreadMutex), 0600); + if (shmid <= 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init mutex while shmget since %s", terrstr()); + goto _OVER; + } + + pMutex = (TdThreadMutex *)shmat(shmid, NULL, 0); + if (pMutex == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init mutex while shmat since %s", terrstr()); + goto _OVER; + } + + if (taosThreadMutexInit(pMutex, &mattr) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init mutex since %s", terrstr()); + goto _OVER; + } + + code = 0; + +_OVER: + if (code != 0) { + taosThreadMutexDestroy(pMutex); + shmctl(shmid, IPC_RMID, NULL); + } else { + *ppMutex = pMutex; + *pShmid = shmid; + } + + pthread_mutexattr_destroy(&mattr); + return code; +} + +static void taosProcDestroyMutex(TdThreadMutex *pMutex, int32_t *pShmid) { + if (pMutex != NULL) { + taosThreadMutexDestroy(pMutex); + } + if (*pShmid > 0) { + shmctl(*pShmid, IPC_RMID, NULL); + } +} + +static int32_t taosProcInitBuffer(void **ppBuffer, int32_t size) { + int32_t shmid = shmget(IPC_PRIVATE, size, IPC_CREAT | 0600); + if (shmid <= 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init buffer while shmget since %s", terrstr()); + return -1; + } + + void *shmptr = shmat(shmid, NULL, 0); + if (shmptr == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init buffer while shmat since %s", terrstr()); + shmctl(shmid, IPC_RMID, NULL); + return -1; + } + + *ppBuffer = shmptr; + return shmid; +} + +static void taosProcDestroyBuffer(void *pBuffer, int32_t *pShmid) { + if (*pShmid > 0) { + shmctl(*pShmid, IPC_RMID, NULL); + } +} + +static SProcQueue *taosProcQueueInit(int32_t size) { + if (size <= 0) size = SHM_DEFAULT_SIZE; + + int32_t bufSize = CEIL8(size); + int32_t headSize = CEIL8(sizeof(SProcQueue)); + + SProcQueue *pQueue = NULL; + int32_t shmId = taosProcInitBuffer((void **)&pQueue, bufSize + headSize); + if (shmId <= 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pQueue->bufferShmid = shmId; + + if (taosProcInitMutex(&pQueue->mutex, &pQueue->mutexShmid) != 0) { + free(pQueue); + return NULL; + } + + if (tsem_init(&pQueue->sem, 1, 0) != 0) { + taosProcDestroyMutex(pQueue->mutex, &pQueue->mutexShmid); + free(pQueue); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + if (taosProcInitMutex(&pQueue->mutex, &pQueue->mutexShmid) != 0) { + taosProcDestroyMutex(pQueue->mutex, &pQueue->mutexShmid); + tsem_destroy(&pQueue->sem); + free(pQueue); + return NULL; + } + + pQueue->head = 0; + pQueue->tail = 0; + pQueue->total = bufSize; + pQueue->avail = bufSize; + pQueue->items = 0; + pQueue->pBuffer = (char *)pQueue + headSize; + return pQueue; +} + +static void taosProcQueueCleanup(SProcQueue *pQueue) { + if (pQueue != NULL) { + uDebug("proc:%s, queue:%p clean up", pQueue->name, pQueue); + taosProcDestroyMutex(pQueue->mutex, &pQueue->mutexShmid); + tsem_destroy(&pQueue->sem); + free(pQueue); + } +} + +static int32_t taosProcQueuePush(SProcQueue *pQueue, char *pHead, int32_t rawHeadLen, char *pBody, int32_t rawBodyLen) { + const int32_t headLen = CEIL8(rawHeadLen); + const int32_t bodyLen = CEIL8(rawBodyLen); + const int32_t fullLen = headLen + bodyLen + 8; + + taosThreadMutexLock(pQueue->mutex); + if (fullLen > pQueue->avail) { + taosThreadMutexUnlock(pQueue->mutex); + terrno = TSDB_CODE_OUT_OF_SHM_MEM; + return -1; + } + + if (pQueue->tail < pQueue->total) { + *(int32_t *)(pQueue->pBuffer + pQueue->head) = headLen; + *(int32_t *)(pQueue->pBuffer + pQueue->head + 4) = bodyLen; + } else { + *(int32_t *)(pQueue->pBuffer) = headLen; + *(int32_t *)(pQueue->pBuffer + 4) = bodyLen; + } + + if (pQueue->tail < pQueue->head) { + memcpy(pQueue->pBuffer + pQueue->tail + 8, pHead, rawHeadLen); + memcpy(pQueue->pBuffer + pQueue->tail + 8 + headLen, pBody, rawBodyLen); + pQueue->tail = pQueue->tail + 8 + headLen + bodyLen; + } else { + int32_t remain = pQueue->total - pQueue->tail; + if (remain == 0) { + memcpy(pQueue->pBuffer + 8, pHead, rawHeadLen); + memcpy(pQueue->pBuffer + 8 + headLen, pBody, rawBodyLen); + pQueue->tail = 8 + headLen + bodyLen; + } else if (remain == 8) { + memcpy(pQueue->pBuffer, pHead, rawHeadLen); + memcpy(pQueue->pBuffer + headLen, pBody, rawBodyLen); + pQueue->tail = headLen + bodyLen; + } else if (remain < 8 + headLen) { + memcpy(pQueue->pBuffer + pQueue->head + 8, pHead, remain - 8); + memcpy(pQueue->pBuffer, pHead + remain - 8, rawHeadLen - (remain - 8)); + memcpy(pQueue->pBuffer + headLen - (remain - 8), pBody, rawBodyLen); + pQueue->tail = headLen - (remain - 8) + bodyLen; + } else if (remain < 8 + bodyLen) { + memcpy(pQueue->pBuffer + pQueue->head + 8, pHead, rawHeadLen); + memcpy(pQueue->pBuffer + pQueue->head + 8 + headLen, pBody, remain - 8 - headLen); + memcpy(pQueue->pBuffer, pBody + remain - 8 - headLen, rawBodyLen - (remain - 8 - headLen)); + pQueue->tail = bodyLen - (remain - 8 - headLen); + } else { + memcpy(pQueue->pBuffer + pQueue->head + 8, pHead, rawHeadLen); + memcpy(pQueue->pBuffer + pQueue->head + headLen + 8, pBody, rawBodyLen); + pQueue->tail = pQueue->head + headLen + bodyLen + 8; + } + } + + pQueue->avail -= fullLen; + pQueue->items++; + taosThreadMutexUnlock(pQueue->mutex); + tsem_post(&pQueue->sem); + + uTrace("proc:%s, push msg:%p:%d cont:%p:%d to queue:%p", pQueue->name, pHead, rawHeadLen, pBody, rawBodyLen, pQueue); + return 0; +} + +static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int32_t *pHeadLen, void **ppBody, + int32_t *pBodyLen) { + tsem_wait(&pQueue->sem); + + taosThreadMutexLock(pQueue->mutex); + if (pQueue->total - pQueue->avail <= 0) { + taosThreadMutexUnlock(pQueue->mutex); + tsem_post(&pQueue->sem); + terrno = TSDB_CODE_OUT_OF_SHM_MEM; + return -1; + } + + int32_t headLen = 0; + int32_t bodyLen = 0; + if (pQueue->head < pQueue->total) { + headLen = *(int32_t *)(pQueue->pBuffer + pQueue->head); + bodyLen = *(int32_t *)(pQueue->pBuffer + pQueue->head + 4); + } else { + headLen = *(int32_t *)(pQueue->pBuffer); + bodyLen = *(int32_t *)(pQueue->pBuffer + 4); + } + + void *pHead = (*pQueue->mallocHeadFp)(headLen); + void *pBody = (*pQueue->mallocBodyFp)(bodyLen); + if (pHead == NULL || pBody == NULL) { + taosThreadMutexUnlock(pQueue->mutex); + tsem_post(&pQueue->sem); + (*pQueue->freeHeadFp)(pHead); + (*pQueue->freeBodyFp)(pBody); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (pQueue->head < pQueue->tail) { + memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, headLen); + memcpy(pBody, pQueue->pBuffer + pQueue->head + 8 + headLen, bodyLen); + pQueue->head = pQueue->head + 8 + headLen + bodyLen; + } else { + int32_t remain = pQueue->total - pQueue->head; + if (remain == 0) { + memcpy(pHead, pQueue->pBuffer + 8, headLen); + memcpy(pBody, pQueue->pBuffer + 8 + headLen, bodyLen); + pQueue->head = 8 + headLen + bodyLen; + } else if (remain == 8) { + memcpy(pHead, pQueue->pBuffer, headLen); + memcpy(pBody, pQueue->pBuffer + headLen, bodyLen); + pQueue->head = headLen + bodyLen; + } else if (remain < 8 + headLen) { + memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, remain - 8); + memcpy(pHead + remain - 8, pQueue->pBuffer, headLen - (remain - 8)); + memcpy(pBody, pQueue->pBuffer + headLen - (remain - 8), bodyLen); + pQueue->head = headLen - (remain - 8) + bodyLen; + } else if (remain < 8 + bodyLen) { + memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, headLen); + memcpy(pBody, pQueue->pBuffer + pQueue->head + 8 + headLen, remain - 8 - headLen); + memcpy(pBody + remain - 8 - headLen, pQueue->pBuffer, bodyLen - (remain - 8 - headLen)); + pQueue->head = bodyLen - (remain - 8 - headLen); + } else { + memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, headLen); + memcpy(pBody, pQueue->pBuffer + pQueue->head + headLen + 8, bodyLen); + pQueue->head = pQueue->head + headLen + bodyLen + 8; + } + } + + pQueue->avail = pQueue->avail + headLen + bodyLen + 8; + pQueue->items--; + taosThreadMutexUnlock(pQueue->mutex); + + *ppHead = pHead; + *ppBody = pBody; + *pHeadLen = headLen; + *pBodyLen = bodyLen; + + uTrace("proc:%s, get msg:%p:%d cont:%p:%d from queue:%p", pQueue->name, pHead, headLen, pBody, bodyLen, pQueue); + return 0; +} + +SProcObj *taosProcInit(const SProcCfg *pCfg) { + SProcObj *pProc = calloc(1, sizeof(SProcObj)); + if (pProc == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pProc->name = pCfg->name; + pProc->testFlag = pCfg->testFlag; + + pProc->pChildQueue = taosProcQueueInit(pCfg->childQueueSize); + pProc->pParentQueue = taosProcQueueInit(pCfg->parentQueueSize); + if (pProc->pChildQueue == NULL || pProc->pParentQueue == NULL) { + taosProcQueueCleanup(pProc->pChildQueue); + free(pProc); + return NULL; + } + + pProc->pChildQueue->name = pCfg->name; + pProc->pChildQueue->pParent = pCfg->pParent; + pProc->pChildQueue->mallocHeadFp = pCfg->childMallocHeadFp; + pProc->pChildQueue->freeHeadFp = pCfg->childFreeHeadFp; + pProc->pChildQueue->mallocBodyFp = pCfg->childMallocBodyFp; + pProc->pChildQueue->freeBodyFp = pCfg->childFreeBodyFp; + pProc->pChildQueue->consumeFp = pCfg->childConsumeFp; + pProc->pParentQueue->name = pCfg->name; + pProc->pParentQueue->pParent = pCfg->pParent; + pProc->pParentQueue->mallocHeadFp = pCfg->parentdMallocHeadFp; + pProc->pParentQueue->freeHeadFp = pCfg->parentFreeHeadFp; + pProc->pParentQueue->mallocBodyFp = pCfg->parentMallocBodyFp; + pProc->pParentQueue->freeBodyFp = pCfg->parentFreeBodyFp; + pProc->pParentQueue->consumeFp = pCfg->parentConsumeFp; + + uDebug("proc:%s, initialized, child queue:%p parent queue:%p", pProc->name, pProc->pChildQueue, pProc->pParentQueue); + + if (!pProc->testFlag) { + pProc->pid = fork(); + if (pProc->pid == 0) { + pProc->isChild = 1; + uInfo("this is child process, pid:%d", pProc->pid); + } else { + pProc->isChild = 0; + uInfo("this is parent process, pid:%d", pProc->pid); + } + } + + return pProc; +} + +static void taosProcThreadLoop(SProcQueue *pQueue) { + ProcConsumeFp consumeFp = pQueue->consumeFp; + void *pParent = pQueue->pParent; + void *pHead, *pBody; + int32_t headLen, bodyLen; + + uDebug("proc:%s, start to get message from queue:%p", pQueue->name, pQueue); + + while (1) { + int32_t code = taosProcQueuePop(pQueue, &pHead, &headLen, &pBody, &bodyLen); + if (code < 0) { + uDebug("proc:%s, get no message from queue:%p and exiting", pQueue->name, pQueue); + break; + } else if (code < 0) { + uTrace("proc:%s, get no message from queue:%p since %s", pQueue->name, pQueue, terrstr()); + taosMsleep(1); + continue; + } else { + (*consumeFp)(pParent, pHead, headLen, pBody, bodyLen); + } + } +} + +int32_t taosProcRun(SProcObj *pProc) { + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (pProc->isChild || pProc->testFlag) { + if (taosThreadCreate(&pProc->childThread, &thAttr, (ProcThreadFp)taosProcThreadLoop, pProc->pChildQueue) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to create thread since %s", terrstr()); + return -1; + } + uDebug("proc:%s, child start to consume queue:%p", pProc->name, pProc->pChildQueue); + } + + if (!pProc->isChild || pProc->testFlag) { + if (taosThreadCreate(&pProc->parentThread, &thAttr, (ProcThreadFp)taosProcThreadLoop, pProc->pParentQueue) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to create thread since %s", terrstr()); + return -1; + } + uDebug("proc:%s, parent start to consume queue:%p", pProc->name, pProc->pParentQueue); + } + + return 0; +} + +void taosProcStop(SProcObj *pProc) { + pProc->stopFlag = true; + // todo join +} + +bool taosProcIsChild(SProcObj *pProc) { return pProc->isChild; } + +void taosProcCleanup(SProcObj *pProc) { + if (pProc != NULL) { + uDebug("proc:%s, clean up", pProc->name); + taosProcStop(pProc); + taosProcQueueCleanup(pProc->pChildQueue); + taosProcQueueCleanup(pProc->pParentQueue); + free(pProc); + } +} + +int32_t taosProcPutToChildQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen) { + return taosProcQueuePush(pProc->pChildQueue, pHead, headLen, pBody, bodyLen); +} + +int32_t taosProcPutToParentQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen) { + return taosProcQueuePush(pProc->pParentQueue, pHead, headLen, pBody, bodyLen); +} diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 0003522033..99675630f4 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -37,13 +37,13 @@ typedef struct STaosQueue { void *ahandle; // for queue set FItem itemFp; FItems itemsFp; - pthread_mutex_t mutex; + TdThreadMutex mutex; } STaosQueue; typedef struct STaosQset { STaosQueue *head; STaosQueue *current; - pthread_mutex_t mutex; + TdThreadMutex mutex; int32_t numOfQueues; int32_t numOfItems; tsem_t sem; @@ -63,7 +63,7 @@ STaosQueue *taosOpenQueue() { return NULL; } - if (pthread_mutex_init(&queue->mutex, NULL) != 0) { + if (taosThreadMutexInit(&queue->mutex, NULL) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -84,11 +84,11 @@ void taosCloseQueue(STaosQueue *queue) { STaosQnode *pTemp; STaosQset *qset; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); STaosQnode *pNode = queue->head; queue->head = NULL; qset = queue->qset; - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); if (queue->qset) { taosRemoveFromQset(qset, queue); @@ -100,7 +100,7 @@ void taosCloseQueue(STaosQueue *queue) { free(pTemp); } - pthread_mutex_destroy(&queue->mutex); + taosThreadMutexDestroy(&queue->mutex); free(queue); uDebug("queue:%p is closed", queue); @@ -110,19 +110,19 @@ bool taosQueueEmpty(STaosQueue *queue) { if (queue == NULL) return true; bool empty = false; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); if (queue->head == NULL && queue->tail == NULL) { empty = true; } - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); return empty; } int32_t taosQueueSize(STaosQueue *queue) { - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); int32_t numOfItems = queue->numOfItems; - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); return numOfItems; } @@ -151,7 +151,7 @@ int32_t taosWriteQitem(STaosQueue *queue, void *pItem) { STaosQnode *pNode = (STaosQnode *)(((char *)pItem) - sizeof(STaosQnode)); pNode->next = NULL; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); if (queue->tail) { queue->tail->next = pNode; @@ -165,7 +165,7 @@ int32_t taosWriteQitem(STaosQueue *queue, void *pItem) { if (queue->qset) atomic_add_fetch_32(&queue->qset->numOfItems, 1); uTrace("item:%p is put into queue:%p, items:%d", pItem, queue, queue->numOfItems); - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); if (queue->qset) tsem_post(&queue->qset->sem); @@ -176,7 +176,7 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { STaosQnode *pNode = NULL; int32_t code = 0; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); if (queue->head) { pNode = queue->head; @@ -189,7 +189,7 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); } - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); return code; } @@ -202,7 +202,7 @@ int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall) { int32_t code = 0; bool empty; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); empty = queue->head == NULL; if (!empty) { @@ -219,7 +219,7 @@ int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall) { if (queue->qset) atomic_sub_fetch_32(&queue->qset->numOfItems, qall->numOfItems); } - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); // if source queue is empty, we set destination qall to empty too. if (empty) { @@ -255,7 +255,7 @@ STaosQset *taosOpenQset() { return NULL; } - pthread_mutex_init(&qset->mutex, NULL); + taosThreadMutexInit(&qset->mutex, NULL); tsem_init(&qset->sem, 0, 0); uDebug("qset:%p is opened", qset); @@ -266,7 +266,7 @@ void taosCloseQset(STaosQset *qset) { if (qset == NULL) return; // remove all the queues from qset - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); while (qset->head) { STaosQueue *queue = qset->head; qset->head = qset->head->next; @@ -274,9 +274,9 @@ void taosCloseQset(STaosQset *qset) { queue->qset = NULL; queue->next = NULL; } - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); - pthread_mutex_destroy(&qset->mutex); + taosThreadMutexDestroy(&qset->mutex); tsem_destroy(&qset->sem); free(qset); uDebug("qset:%p is closed", qset); @@ -293,19 +293,19 @@ void taosQsetThreadResume(STaosQset *qset) { int32_t taosAddIntoQset(STaosQset *qset, STaosQueue *queue, void *ahandle) { if (queue->qset) return -1; - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); queue->next = qset->head; queue->ahandle = ahandle; qset->head = queue; qset->numOfQueues++; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); atomic_add_fetch_32(&qset->numOfItems, queue->numOfItems); queue->qset = qset; - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); uTrace("queue:%p is added into qset:%p", queue, qset); return 0; @@ -314,7 +314,7 @@ int32_t taosAddIntoQset(STaosQset *qset, STaosQueue *queue, void *ahandle) { void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue) { STaosQueue *tqueue = NULL; - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); if (qset->head) { if (qset->head == queue) { @@ -339,15 +339,15 @@ void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue) { if (qset->current == queue) qset->current = tqueue->next; qset->numOfQueues--; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); atomic_sub_fetch_32(&qset->numOfItems, queue->numOfItems); queue->qset = NULL; queue->next = NULL; - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); } } - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); uDebug("queue:%p is removed from qset:%p", queue, qset); } @@ -360,7 +360,7 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FI tsem_wait(&qset->sem); - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); for (int32_t i = 0; i < qset->numOfQueues; ++i) { if (qset->current == NULL) qset->current = qset->head; @@ -369,7 +369,7 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FI if (queue == NULL) break; if (queue->head == NULL) continue; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); if (queue->head) { pNode = queue->head; @@ -385,11 +385,11 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FI uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); } - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); if (pNode) break; } - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); return code; } @@ -399,7 +399,7 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahand int32_t code = 0; tsem_wait(&qset->sem); - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); for (int32_t i = 0; i < qset->numOfQueues; ++i) { if (qset->current == NULL) qset->current = qset->head; @@ -408,7 +408,7 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahand if (queue == NULL) break; if (queue->head == NULL) continue; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); if (queue->head) { qall->current = queue->head; @@ -428,12 +428,12 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahand } } - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); if (code != 0) break; } - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); return code; } @@ -443,7 +443,7 @@ int32_t taosReadQitemFromQsetByThread(STaosQset *qset, void **ppItem, void **aha tsem_wait(&qset->sem); - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); for (int32_t i = 0; i < qset->numOfQueues; ++i) { if (qset->current == NULL) qset->current = qset->head; @@ -456,7 +456,7 @@ int32_t taosReadQitemFromQsetByThread(STaosQset *qset, void **ppItem, void **aha continue; } - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); if (queue->head) { pNode = queue->head; @@ -475,11 +475,11 @@ int32_t taosReadQitemFromQsetByThread(STaosQset *qset, void **ppItem, void **aha uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); } - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); if (pNode) break; } - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); return code; } @@ -488,21 +488,21 @@ void taosResetQsetThread(STaosQset *qset, void *pItem) { if (pItem == NULL) return; STaosQnode *pNode = (STaosQnode *)((char *)pItem - sizeof(STaosQnode)); - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); pNode->queue->threadId = -1; for (int32_t i = 0; i < pNode->queue->numOfItems; ++i) { tsem_post(&qset->sem); } - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); } int32_t taosGetQueueItemsNumber(STaosQueue *queue) { if (!queue) return 0; int32_t num; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); num = queue->numOfItems; - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); return num; } @@ -510,8 +510,8 @@ int32_t taosGetQsetItemsNumber(STaosQset *qset) { if (!qset) return 0; int32_t num = 0; - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); num = qset->numOfItems; - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); return num; } diff --git a/source/util/src/tref.c b/source/util/src/tref.c index a9f6c21bf8..86633fe07a 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -45,8 +45,8 @@ typedef struct { } SRefSet; static SRefSet tsRefSetList[TSDB_REF_OBJECTS]; -static pthread_once_t tsRefModuleInit = PTHREAD_ONCE_INIT; -static pthread_mutex_t tsRefMutex; +static TdThreadOnce tsRefModuleInit = PTHREAD_ONCE_INIT; +static TdThreadMutex tsRefMutex; static int32_t tsRefSetNum = 0; static int32_t tsNextId = 0; @@ -63,7 +63,7 @@ int32_t taosOpenRef(int32_t max, void (*fp)(void *)) { int64_t *lockedBy; int32_t i, rsetId; - pthread_once(&tsRefModuleInit, taosInitRefModule); + taosThreadOnce(&tsRefModuleInit, taosInitRefModule); nodeList = calloc(sizeof(SRefNode *), (size_t)max); if (nodeList == NULL) { @@ -78,7 +78,7 @@ int32_t taosOpenRef(int32_t max, void (*fp)(void *)) { return -1; } - pthread_mutex_lock(&tsRefMutex); + taosThreadMutexLock(&tsRefMutex); for (i = 0; i < TSDB_REF_OBJECTS; ++i) { tsNextId = (tsNextId + 1) % TSDB_REF_OBJECTS; @@ -107,7 +107,7 @@ int32_t taosOpenRef(int32_t max, void (*fp)(void *)) { uTrace("run out of Ref ID, maximum:%d refSetNum:%d", TSDB_REF_OBJECTS, tsRefSetNum); } - pthread_mutex_unlock(&tsRefMutex); + taosThreadMutexUnlock(&tsRefMutex); return rsetId; } @@ -124,7 +124,7 @@ int32_t taosCloseRef(int32_t rsetId) { pSet = tsRefSetList + rsetId; - pthread_mutex_lock(&tsRefMutex); + taosThreadMutexLock(&tsRefMutex); if (pSet->state == TSDB_REF_STATE_ACTIVE) { pSet->state = TSDB_REF_STATE_DELETED; @@ -134,7 +134,7 @@ int32_t taosCloseRef(int32_t rsetId) { uTrace("rsetId:%d is already closed, count:%d", rsetId, pSet->count); } - pthread_mutex_unlock(&tsRefMutex); + taosThreadMutexUnlock(&tsRefMutex); if (deleted) taosDecRsetCount(pSet); @@ -354,7 +354,7 @@ int32_t taosListRef() { SRefNode *pNode; int32_t num = 0; - pthread_mutex_lock(&tsRefMutex); + taosThreadMutexLock(&tsRefMutex); for (int32_t i = 0; i < TSDB_REF_OBJECTS; ++i) { pSet = tsRefSetList + i; @@ -374,7 +374,7 @@ int32_t taosListRef() { } } - pthread_mutex_unlock(&tsRefMutex); + taosThreadMutexUnlock(&tsRefMutex); return num; } @@ -470,7 +470,7 @@ static void taosUnlockList(int64_t *lockedBy) { } } -static void taosInitRefModule(void) { pthread_mutex_init(&tsRefMutex, NULL); } +static void taosInitRefModule(void) { taosThreadMutexInit(&tsRefMutex, NULL); } static void taosIncRsetCount(SRefSet *pSet) { atomic_add_fetch_32(&pSet->count, 1); @@ -483,7 +483,7 @@ static void taosDecRsetCount(SRefSet *pSet) { if (count > 0) return; - pthread_mutex_lock(&tsRefMutex); + taosThreadMutexLock(&tsRefMutex); if (pSet->state != TSDB_REF_STATE_EMPTY) { pSet->state = TSDB_REF_STATE_EMPTY; @@ -497,5 +497,5 @@ static void taosDecRsetCount(SRefSet *pSet) { uTrace("rsetId:%d is cleaned, refSetNum:%d count:%d", pSet->rsetId, tsRefSetNum, pSet->count); } - pthread_mutex_unlock(&tsRefMutex); + taosThreadMutexUnlock(&tsRefMutex); } diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 740e742bad..316c1f7ec1 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -26,12 +26,12 @@ typedef struct { char label[TSDB_LABEL_LEN]; tsem_t emptySem; tsem_t fullSem; - pthread_mutex_t queueMutex; + TdThreadMutex queueMutex; int32_t fullSlot; int32_t emptySlot; int32_t queueSize; int32_t numOfThreads; - pthread_t *qthread; + TdThread *qthread; SSchedMsg *queue; bool stop; void *pTmrCtrl; @@ -55,7 +55,7 @@ void *taosInitScheduler(int32_t queueSize, int32_t numOfThreads, const char *lab return NULL; } - pSched->qthread = calloc(sizeof(pthread_t), numOfThreads); + pSched->qthread = calloc(sizeof(TdThread), numOfThreads); if (pSched->qthread == NULL) { uError("%s: no enough memory for qthread", label); taosCleanUpScheduler(pSched); @@ -68,7 +68,7 @@ void *taosInitScheduler(int32_t queueSize, int32_t numOfThreads, const char *lab pSched->fullSlot = 0; pSched->emptySlot = 0; - if (pthread_mutex_init(&pSched->queueMutex, NULL) < 0) { + if (taosThreadMutexInit(&pSched->queueMutex, NULL) < 0) { uError("init %s:queueMutex failed(%s)", label, strerror(errno)); taosCleanUpScheduler(pSched); return NULL; @@ -88,11 +88,11 @@ void *taosInitScheduler(int32_t queueSize, int32_t numOfThreads, const char *lab pSched->stop = false; for (int32_t i = 0; i < numOfThreads; ++i) { - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - int32_t code = pthread_create(pSched->qthread + i, &attr, taosProcessSchedQueue, (void *)pSched); - pthread_attr_destroy(&attr); + TdThreadAttr attr; + taosThreadAttrInit(&attr); + taosThreadAttrSetDetachState(&attr, PTHREAD_CREATE_JOINABLE); + int32_t code = taosThreadCreate(pSched->qthread + i, &attr, taosProcessSchedQueue, (void *)pSched); + taosThreadAttrDestroy(&attr); if (code != 0) { uError("%s: failed to create rpc thread(%s)", label, strerror(errno)); taosCleanUpScheduler(pSched); @@ -135,7 +135,7 @@ void *taosProcessSchedQueue(void *scheduler) { break; } - if ((ret = pthread_mutex_lock(&pSched->queueMutex)) != 0) { + if ((ret = taosThreadMutexLock(&pSched->queueMutex)) != 0) { uFatal("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); exit(ret); } @@ -144,7 +144,7 @@ void *taosProcessSchedQueue(void *scheduler) { memset(pSched->queue + pSched->fullSlot, 0, sizeof(SSchedMsg)); pSched->fullSlot = (pSched->fullSlot + 1) % pSched->queueSize; - if ((ret = pthread_mutex_unlock(&pSched->queueMutex)) != 0) { + if ((ret = taosThreadMutexUnlock(&pSched->queueMutex)) != 0) { uFatal("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); exit(ret); } @@ -177,7 +177,7 @@ void taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg) { exit(ret); } - if ((ret = pthread_mutex_lock(&pSched->queueMutex)) != 0) { + if ((ret = taosThreadMutexLock(&pSched->queueMutex)) != 0) { uFatal("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); exit(ret); } @@ -185,7 +185,7 @@ void taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg) { pSched->queue[pSched->emptySlot] = *pMsg; pSched->emptySlot = (pSched->emptySlot + 1) % pSched->queueSize; - if ((ret = pthread_mutex_unlock(&pSched->queueMutex)) != 0) { + if ((ret = taosThreadMutexUnlock(&pSched->queueMutex)) != 0) { uFatal("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); exit(ret); } @@ -208,13 +208,13 @@ void taosCleanUpScheduler(void *param) { } for (int32_t i = 0; i < pSched->numOfThreads; ++i) { if (taosCheckPthreadValid(pSched->qthread[i])) { - pthread_join(pSched->qthread[i], NULL); + taosThreadJoin(pSched->qthread[i], NULL); } } tsem_destroy(&pSched->emptySem); tsem_destroy(&pSched->fullSem); - pthread_mutex_destroy(&pSched->queueMutex); + taosThreadMutexDestroy(&pSched->queueMutex); if (pSched->pTimer) { taosTmrStopA(&pSched->pTimer); diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index b5e54d12d3..f7e56b6f31 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -70,13 +70,13 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, _ } if (SL_IS_THREAD_SAFE(pSkipList)) { - pSkipList->lock = (pthread_rwlock_t *)calloc(1, sizeof(pthread_rwlock_t)); + pSkipList->lock = (TdThreadRwlock *)calloc(1, sizeof(TdThreadRwlock)); if (pSkipList->lock == NULL) { tSkipListDestroy(pSkipList); return NULL; } - if (pthread_rwlock_init(pSkipList->lock, NULL) != 0) { + if (taosThreadRwlockInit(pSkipList->lock, NULL) != 0) { tSkipListDestroy(pSkipList); return NULL; } @@ -109,7 +109,7 @@ void tSkipListDestroy(SSkipList *pSkipList) { tSkipListUnlock(pSkipList); if (pSkipList->lock != NULL) { - pthread_rwlock_destroy(pSkipList->lock); + taosThreadRwlockDestroy(pSkipList->lock); tfree(pSkipList->lock); } @@ -435,21 +435,21 @@ static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t static FORCE_INLINE int32_t tSkipListWLock(SSkipList *pSkipList) { if (pSkipList->lock) { - return pthread_rwlock_wrlock(pSkipList->lock); + return taosThreadRwlockWrlock(pSkipList->lock); } return 0; } static FORCE_INLINE int32_t tSkipListRLock(SSkipList *pSkipList) { if (pSkipList->lock) { - return pthread_rwlock_rdlock(pSkipList->lock); + return taosThreadRwlockRdlock(pSkipList->lock); } return 0; } static FORCE_INLINE int32_t tSkipListUnlock(SSkipList *pSkipList) { if (pSkipList->lock) { - return pthread_rwlock_unlock(pSkipList->lock); + return taosThreadRwlockUnlock(pSkipList->lock); } return 0; } diff --git a/source/util/src/tthread.c b/source/util/src/tthread.c index f9e28d7b62..da76135b68 100644 --- a/source/util/src/tthread.c +++ b/source/util/src/tthread.c @@ -16,13 +16,13 @@ #define _DEFAULT_SOURCE #include "tthread.h" -pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param) { - pthread_t* pthread = (pthread_t*)malloc(sizeof(pthread_t)); - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - int32_t ret = pthread_create(pthread, &thattr, __start_routine, param); - pthread_attr_destroy(&thattr); +TdThread* taosCreateThread(void* (*__start_routine)(void*), void* param) { + TdThread* pthread = (TdThread*)malloc(sizeof(TdThread)); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + int32_t ret = taosThreadCreate(pthread, &thattr, __start_routine, param); + taosThreadAttrDestroy(&thattr); if (ret != 0) { free(pthread); @@ -31,20 +31,20 @@ pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param) { return pthread; } -bool taosDestoryThread(pthread_t* pthread) { +bool taosDestoryThread(TdThread* pthread) { if (pthread == NULL) return false; if (taosThreadRunning(pthread)) { - pthread_cancel(*pthread); - pthread_join(*pthread, NULL); + taosThreadCancel(*pthread); + taosThreadJoin(*pthread, NULL); } free(pthread); return true; } -bool taosThreadRunning(pthread_t* pthread) { +bool taosThreadRunning(TdThread* pthread) { if (pthread == NULL) return false; - int32_t ret = pthread_kill(*pthread, 0); + int32_t ret = taosThreadKill(*pthread, 0); if (ret == ESRCH) return false; if (ret == EINVAL) return false; // alive diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 46bca6e6cb..cc31d9c94a 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -102,7 +102,7 @@ typedef struct timer_map_t { } timer_map_t; typedef struct time_wheel_t { - pthread_mutex_t mutex; + TdThreadMutex mutex; int64_t nextScanAt; uint32_t resolution; uint16_t size; @@ -112,8 +112,8 @@ typedef struct time_wheel_t { int32_t tsMaxTmrCtrl = 512; -static pthread_once_t tmrModuleInit = PTHREAD_ONCE_INIT; -static pthread_mutex_t tmrCtrlMutex; +static TdThreadOnce tmrModuleInit = PTHREAD_ONCE_INIT; +static TdThreadMutex tmrCtrlMutex; static tmr_ctrl_t* tmrCtrls; static tmr_ctrl_t* unusedTmrCtrl = NULL; static void* tmrQhandle; @@ -230,7 +230,7 @@ static void addToWheel(tmr_obj_t* timer, uint32_t delay) { timer->prev = NULL; timer->expireAt = taosGetMonotonicMs() + delay; - pthread_mutex_lock(&wheel->mutex); + taosThreadMutexLock(&wheel->mutex); uint32_t idx = 0; if (timer->expireAt > wheel->nextScanAt) { @@ -248,7 +248,7 @@ static void addToWheel(tmr_obj_t* timer, uint32_t delay) { p->prev = timer; } - pthread_mutex_unlock(&wheel->mutex); + taosThreadMutexUnlock(&wheel->mutex); } static bool removeFromWheel(tmr_obj_t* timer) { @@ -259,7 +259,7 @@ static bool removeFromWheel(tmr_obj_t* timer) { time_wheel_t* wheel = wheels + wheelIdx; bool removed = false; - pthread_mutex_lock(&wheel->mutex); + taosThreadMutexLock(&wheel->mutex); // other thread may modify timer->wheel, check again. if (timer->wheel < tListLen(wheels)) { if (timer->prev != NULL) { @@ -277,7 +277,7 @@ static bool removeFromWheel(tmr_obj_t* timer) { timerDecRef(timer); removed = true; } - pthread_mutex_unlock(&wheel->mutex); + taosThreadMutexUnlock(&wheel->mutex); return removed; } @@ -372,7 +372,7 @@ static void taosTimerLoopFunc(int32_t signo) { time_wheel_t* wheel = wheels + i; while (now >= wheel->nextScanAt) { - pthread_mutex_lock(&wheel->mutex); + taosThreadMutexLock(&wheel->mutex); wheel->index = (wheel->index + 1) % wheel->size; tmr_obj_t* timer = wheel->slots[wheel->index]; while (timer != NULL) { @@ -407,7 +407,7 @@ static void taosTimerLoopFunc(int32_t signo) { timer = next; } wheel->nextScanAt += wheel->resolution; - pthread_mutex_unlock(&wheel->mutex); + taosThreadMutexUnlock(&wheel->mutex); } addToExpired(expired); @@ -528,12 +528,12 @@ static void taosTmrModuleInit(void) { (tmrCtrls + tsMaxTmrCtrl - 1)->next = NULL; unusedTmrCtrl = tmrCtrls; - pthread_mutex_init(&tmrCtrlMutex, NULL); + taosThreadMutexInit(&tmrCtrlMutex, NULL); int64_t now = taosGetMonotonicMs(); for (int32_t i = 0; i < tListLen(wheels); i++) { time_wheel_t* wheel = wheels + i; - if (pthread_mutex_init(&wheel->mutex, NULL) != 0) { + if (taosThreadMutexInit(&wheel->mutex, NULL) != 0) { tmrError("failed to create the mutex for wheel, reason:%s", strerror(errno)); return; } @@ -564,15 +564,15 @@ void* taosTmrInit(int32_t maxNumOfTmrs, int32_t resolution, int32_t longest, con const char* ret = taosMonotonicInit(); tmrDebug("ttimer monotonic clock source:%s", ret); - pthread_once(&tmrModuleInit, taosTmrModuleInit); + taosThreadOnce(&tmrModuleInit, taosTmrModuleInit); - pthread_mutex_lock(&tmrCtrlMutex); + taosThreadMutexLock(&tmrCtrlMutex); tmr_ctrl_t* ctrl = unusedTmrCtrl; if (ctrl != NULL) { unusedTmrCtrl = ctrl->next; numOfTmrCtrl++; } - pthread_mutex_unlock(&tmrCtrlMutex); + taosThreadMutexUnlock(&tmrCtrlMutex); if (ctrl == NULL) { tmrError("%s too many timer controllers, failed to create timer controller.", label); @@ -594,11 +594,11 @@ void taosTmrCleanUp(void* handle) { tmrDebug("%s timer controller is cleaned up.", ctrl->label); ctrl->label[0] = 0; - pthread_mutex_lock(&tmrCtrlMutex); + taosThreadMutexLock(&tmrCtrlMutex); ctrl->next = unusedTmrCtrl; numOfTmrCtrl--; unusedTmrCtrl = ctrl; - pthread_mutex_unlock(&tmrCtrlMutex); + taosThreadMutexUnlock(&tmrCtrlMutex); tmrDebug("time controller's tmr ctrl size: %d", numOfTmrCtrl); if (numOfTmrCtrl <= 0) { @@ -608,11 +608,11 @@ void taosTmrCleanUp(void* handle) { for (int32_t i = 0; i < tListLen(wheels); i++) { time_wheel_t* wheel = wheels + i; - pthread_mutex_destroy(&wheel->mutex); + taosThreadMutexDestroy(&wheel->mutex); free(wheel->slots); } - pthread_mutex_destroy(&tmrCtrlMutex); + taosThreadMutexDestroy(&tmrCtrlMutex); for (size_t i = 0; i < timerMap.size; i++) { timer_list_t* list = timerMap.slots + i; diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 1fa70da870..78098af3bd 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -28,7 +28,7 @@ int32_t tQWorkerInit(SQWorkerPool *pool) { return -1; } - if (pthread_mutex_init(&pool->mutex, NULL)) { + if (taosThreadMutexInit(&pool->mutex, NULL)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -56,13 +56,13 @@ void tQWorkerCleanup(SQWorkerPool *pool) { SQWorker *worker = pool->workers + i; if (worker == NULL) continue; if (taosCheckPthreadValid(worker->thread)) { - pthread_join(worker->thread, NULL); + taosThreadJoin(worker->thread, NULL); } } tfree(pool->workers); taosCloseQset(pool->qset); - pthread_mutex_destroy(&pool->mutex); + taosThreadMutexDestroy(&pool->mutex); uDebug("worker:%s is closed", pool->name); } @@ -94,10 +94,10 @@ static void *tQWorkerThreadFp(SQWorker *worker) { } STaosQueue *tWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp, ThreadFp threadFp) { - pthread_mutex_lock(&pool->mutex); + taosThreadMutexLock(&pool->mutex); STaosQueue *queue = taosOpenQueue(); if (queue == NULL) { - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -110,11 +110,11 @@ STaosQueue *tWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp, Threa do { SQWorker *worker = pool->workers + pool->num; - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&worker->thread, &thAttr, threadFp, worker) != 0) { + if (taosThreadCreate(&worker->thread, &thAttr, threadFp, worker) != 0) { uError("worker:%s:%d failed to create thread to process since %s", pool->name, worker->id, strerror(errno)); taosCloseQueue(queue); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -122,13 +122,13 @@ STaosQueue *tWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp, Threa break; } - pthread_attr_destroy(&thAttr); + taosThreadAttrDestroy(&thAttr); pool->num++; uDebug("worker:%s:%d is launched, total:%d", pool->name, worker->id, pool->num); } while (pool->num < pool->min); } - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); uDebug("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle); return queue; @@ -194,7 +194,7 @@ int32_t tWWorkerInit(SWWorkerPool *pool) { return -1; } - if (pthread_mutex_init(&pool->mutex, NULL) != 0) { + if (taosThreadMutexInit(&pool->mutex, NULL) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -224,14 +224,14 @@ void tWWorkerCleanup(SWWorkerPool *pool) { for (int32_t i = 0; i < pool->max; ++i) { SWWorker *worker = pool->workers + i; if (taosCheckPthreadValid(worker->thread)) { - pthread_join(worker->thread, NULL); + taosThreadJoin(worker->thread, NULL); taosFreeQall(worker->qall); taosCloseQset(worker->qset); } } tfree(pool->workers); - pthread_mutex_destroy(&pool->mutex); + taosThreadMutexDestroy(&pool->mutex); uInfo("worker:%s is closed", pool->name); } @@ -265,12 +265,12 @@ static void *tWWorkerThreadFp(SWWorker *worker) { } STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { - pthread_mutex_lock(&pool->mutex); + taosThreadMutexLock(&pool->mutex); SWWorker *worker = pool->workers + pool->nextId; STaosQueue *queue = taosOpenQueue(); if (queue == NULL) { - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -281,7 +281,7 @@ STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { worker->qset = taosOpenQset(); if (worker->qset == NULL) { taosCloseQueue(queue); - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); return NULL; } @@ -290,15 +290,15 @@ STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { if (worker->qall == NULL) { taosCloseQset(worker->qset); taosCloseQueue(queue); - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&worker->thread, &thAttr, (ThreadFp)tWWorkerThreadFp, worker) != 0) { + if (taosThreadCreate(&worker->thread, &thAttr, (ThreadFp)tWWorkerThreadFp, worker) != 0) { uError("worker:%s:%d failed to create thread to process since %s", pool->name, worker->id, strerror(errno)); taosFreeQall(worker->qall); taosCloseQset(worker->qset); @@ -310,13 +310,13 @@ STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { pool->nextId = (pool->nextId + 1) % pool->max; } - pthread_attr_destroy(&thAttr); + taosThreadAttrDestroy(&thAttr); } else { taosAddIntoQset(worker->qset, queue, ahandle); pool->nextId = (pool->nextId + 1) % pool->max; } - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); uDebug("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle); return queue; diff --git a/source/util/test/queueTest.cpp b/source/util/test/queueTest.cpp index 310ae4350e..09c544df9f 100644 --- a/source/util/test/queueTest.cpp +++ b/source/util/test/queueTest.cpp @@ -14,6 +14,9 @@ #include "os.h" #include "tqueue.h" +#include +#include + class UtilTestQueue : public ::testing::Test { public: void SetUp() override {} @@ -24,6 +27,107 @@ class UtilTestQueue : public ::testing::Test { static void TearDownTestSuite() {} }; -TEST_F(UtilTestQueue, 01_ReadQitemFromQsetByThread) { - EXPECT_EQ(0, 0); -} \ No newline at end of file +#if 0 +TEST_F(UtilTestQueue, 01_fork) { + pid_t pid; + int shmid; + int* shmptr; + int* tmp; + + int err; + pthread_mutexattr_t mattr; + if ((err = pthread_mutexattr_init(&mattr)) < 0) { + printf("mutex addr init error:%s\n", strerror(err)); + exit(1); + } + + if ((err = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED)) < 0) { + printf("mutex addr get shared error:%s\n", strerror(err)); + exit(1); + } + + pthread_mutex_t* m; + int mid = shmget(IPC_PRIVATE, sizeof(pthread_mutex_t), 0600); + m = (pthread_mutex_t*)shmat(mid, NULL, 0); + + if ((err = pthread_mutex_init(m, &mattr)) < 0) { + printf("mutex mutex init error:%s\n", strerror(err)); + exit(1); + } + + if ((shmid = shmget(IPC_PRIVATE, 1000, IPC_CREAT | 0600)) < 0) { + perror("shmget error"); + exit(1); + } + + if ((shmptr = (int*)shmat(shmid, 0, 0)) == (void*)-1) { + perror("shmat error"); + exit(1); + } + + tmp = shmptr; + + int shmid2; + int** shmptr2; + if ((shmid2 = shmget(IPC_PRIVATE, 20, IPC_CREAT | 0600)) < 0) { + perror("shmget2 error"); + exit(1); + } + + if ((shmptr2 = (int**)shmat(shmid2, 0, 0)) == (void*)-1) { + perror("shmat2 error"); + exit(1); + } + + *shmptr2 = shmptr; + + if ((pid = fork()) < 0) { + perror("fork error"); + exit(1); + } else if (pid == 0) { + if ((err = taosThreadMutexLock(m)) < 0) { + printf("lock error:%s\n", strerror(err)); + exit(1); + } + for (int i = 0; i < 30; ++i) { + **shmptr2 = i; + (*shmptr2)++; + } + + if ((err = taosThreadMutexUnlock(m)) < 0) { + printf("unlock error:%s\n", strerror(err)); + exit(1); + } + exit(0); + + } else { + if ((err = taosThreadMutexLock(m)) < 0) { + printf("lock error:%s\n", strerror(err)); + exit(1); + } + for (int i = 10; i < 42; ++i) { + **shmptr2 = i; + (*shmptr2)++; + } + if ((err = taosThreadMutexUnlock(m)) < 0) { + printf("unlock error:%s\n", strerror(err)); + exit(1); + } + } + + wait(NULL); + + for (int i = 0; i < 70; ++i) { + printf("%d ", tmp[i]); + } + + printf("\n"); + + taosThreadAttrDestroy(&mattr); + //销毁mutex + pthread_mutex_destroy(m); + + exit(0); +} + +#endif \ No newline at end of file diff --git a/source/util/test/trefTest.c b/source/util/test/trefTest.c index 58d9d2202e..a439a84562 100644 --- a/source/util/test/trefTest.c +++ b/source/util/test/trefTest.c @@ -1,6 +1,5 @@ #include #include -#include #include #include #include "os.h" @@ -102,18 +101,18 @@ void *openRefSpace(void *param) { pSpace->p = (void **) calloc(sizeof(void *), pSpace->refNum); pSpace->rid = calloc(pSpace->refNum, sizeof(int64_t)); - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - pthread_t thread1, thread2, thread3; - pthread_create(&(thread1), &thattr, addRef, (void *)(pSpace)); - pthread_create(&(thread2), &thattr, removeRef, (void *)(pSpace)); - pthread_create(&(thread3), &thattr, acquireRelease, (void *)(pSpace)); + TdThread thread1, thread2, thread3; + taosThreadCreate(&(thread1), &thattr, addRef, (void *)(pSpace)); + taosThreadCreate(&(thread2), &thattr, removeRef, (void *)(pSpace)); + taosThreadCreate(&(thread3), &thattr, acquireRelease, (void *)(pSpace)); - pthread_join(thread1, NULL); - pthread_join(thread2, NULL); - pthread_join(thread3, NULL); + taosThreadJoin(thread1, NULL); + taosThreadJoin(thread2, NULL); + taosThreadJoin(thread3, NULL); for (int i=0; irefNum; ++i) { taosRemoveRef(pSpace->rsetId, pSpace->rid[i]); @@ -161,22 +160,22 @@ int main(int argc, char *argv[]) { taosInitLog("tref.log", 10); SRefSpace *pSpaceList = (SRefSpace *) calloc(sizeof(SRefSpace), threads); - pthread_t *pThreadList = (pthread_t *) calloc(sizeof(pthread_t), threads); + TdThread *pThreadList = (TdThread *) calloc(sizeof(TdThread), threads); - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); for (int i=0; i #include #include #include @@ -5067,11 +5066,11 @@ int main(int argc, char *argv[]) exit(1); } - pthread_t *pThreadList = (pthread_t *) calloc(sizeof(pthread_t), 4); + TdThread *pThreadList = (TdThread *) calloc(sizeof(TdThread), 4); - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); T_par par[4]; par[0].taos = taos[0]; @@ -5083,10 +5082,10 @@ int main(int argc, char *argv[]) par[3].taos = taos[3]; par[3].idx = 3; - pthread_create(&(pThreadList[0]), &thattr, runcase, (void *)&par[0]); - //pthread_create(&(pThreadList[1]), &thattr, runcase, (void *)&par[1]); - //pthread_create(&(pThreadList[2]), &thattr, runcase, (void *)&par[2]); - //pthread_create(&(pThreadList[3]), &thattr, runcase, (void *)&par[3]); + taosThreadCreate(&(pThreadList[0]), &thattr, runcase, (void *)&par[0]); + //taosThreadCreate(&(pThreadList[1]), &thattr, runcase, (void *)&par[1]); + //taosThreadCreate(&(pThreadList[2]), &thattr, runcase, (void *)&par[2]); + //taosThreadCreate(&(pThreadList[3]), &thattr, runcase, (void *)&par[3]); while(1) { taosSsleep(1); diff --git a/tests/script/api/stmtBatchTest.c b/tests/script/api/stmtBatchTest.c index 57cbdf1090..a6c20d282e 100644 --- a/tests/script/api/stmtBatchTest.c +++ b/tests/script/api/stmtBatchTest.c @@ -1,7 +1,6 @@ // TAOS standard API example. The same syntax as MySQL, but only a subet // to compile: gcc -o prepare prepare.c -ltaos -#include #include #include #include @@ -5080,7 +5079,7 @@ int main(int argc, char *argv[]) #if 0 printf("server:%s, threadNum:%d, rows:%d\n\n", serverIp, threadNum, g_rows); - pthread_t *pThreadList = (pthread_t *) calloc(sizeof(pthread_t), (size_t)threadNum); + TdThread *pThreadList = (TdThread *) calloc(sizeof(TdThread), (size_t)threadNum); ThreadInfo* threadInfo = (ThreadInfo *) calloc(sizeof(ThreadInfo), (size_t)threadNum); ThreadInfo* tInfo = threadInfo; @@ -5094,16 +5093,16 @@ int main(int argc, char *argv[]) tInfo->taos = taos; tInfo->idx = i; if (0 == i) { - //pthread_create(&(pThreadList[0]), NULL, runCase, (void *)tInfo); - pthread_create(&(pThreadList[0]), NULL, SpecifyColumnBatchCase, (void *)tInfo); + //taosThreadCreate(&(pThreadList[0]), NULL, runCase, (void *)tInfo); + taosThreadCreate(&(pThreadList[0]), NULL, SpecifyColumnBatchCase, (void *)tInfo); } else if (1 == i){ - pthread_create(&(pThreadList[0]), NULL, runCase_long, (void *)tInfo); + taosThreadCreate(&(pThreadList[0]), NULL, runCase_long, (void *)tInfo); } tInfo++; } for (int i = 0; i < threadNum; i++) { - pthread_join(pThreadList[i], NULL); + taosThreadJoin(pThreadList[i], NULL); } free(pThreadList); diff --git a/tests/script/api/stmtTest.c b/tests/script/api/stmtTest.c index c69de46f84..46f1c7c8f8 100644 --- a/tests/script/api/stmtTest.c +++ b/tests/script/api/stmtTest.c @@ -1,4 +1,3 @@ -#include #include #include #include diff --git a/tests/script/api/stmt_function.c b/tests/script/api/stmt_function.c index 4521ad4fa3..a5427844d7 100644 --- a/tests/script/api/stmt_function.c +++ b/tests/script/api/stmt_function.c @@ -1,5 +1,4 @@ #include -#include #include #include #include diff --git a/tests/script/http/httpTest.c b/tests/script/http/httpTest.c index 36ce6b95ba..0fe4265a9d 100644 --- a/tests/script/http/httpTest.c +++ b/tests/script/http/httpTest.c @@ -5,12 +5,11 @@ #include #include #include -#include #define MAXLINE 1024 typedef struct { - pthread_t pid; + TdThread pid; int threadId; int rows; int tables; @@ -107,16 +106,16 @@ void multiThread() { ThreadObj *threads = calloc((size_t)numOfThreads, sizeof(ThreadObj)); for (int i = 0; i < numOfThreads; i++) { ThreadObj *pthread = threads + i; - pthread_attr_t thattr; + TdThreadAttr thattr; pthread->threadId = i + 1; pthread->rows = numOfRows; pthread->tables = numOfTables; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - pthread_create(&pthread->pid, &thattr, (void *(*)(void *))execute, pthread); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadCreate(&pthread->pid, &thattr, (void *(*)(void *))execute, pthread); } for (int i = 0; i < numOfThreads; i++) { - pthread_join(threads[i].pid, NULL); + taosThreadJoin(threads[i].pid, NULL); } free(threads); } diff --git a/tests/script/http/httpTestSqlUtc.c b/tests/script/http/httpTestSqlUtc.c index 643c884a1a..c6c5829b95 100644 --- a/tests/script/http/httpTestSqlUtc.c +++ b/tests/script/http/httpTestSqlUtc.c @@ -5,12 +5,11 @@ #include #include #include -#include #define MAXLINE 1024 typedef struct { - pthread_t pid; + TdThread pid; int threadId; int rows; int tables; @@ -107,16 +106,16 @@ void multiThread() { ThreadObj *threads = calloc((size_t)numOfThreads, sizeof(ThreadObj)); for (int i = 0; i < numOfThreads; i++) { ThreadObj *pthread = threads + i; - pthread_attr_t thattr; + TdThreadAttr thattr; pthread->threadId = i + 1; pthread->rows = numOfRows; pthread->tables = numOfTables; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - pthread_create(&pthread->pid, &thattr, (void *(*)(void *))execute, pthread); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadCreate(&pthread->pid, &thattr, (void *(*)(void *))execute, pthread); } for (int i = 0; i < numOfThreads; i++) { - pthread_join(threads[i].pid, NULL); + taosThreadJoin(threads[i].pid, NULL); } free(threads); } diff --git a/tests/script/http/httpTestSqlt.c b/tests/script/http/httpTestSqlt.c index 2eaaee0f99..400428f471 100644 --- a/tests/script/http/httpTestSqlt.c +++ b/tests/script/http/httpTestSqlt.c @@ -5,12 +5,11 @@ #include #include #include -#include #define MAXLINE 1024 typedef struct { - pthread_t pid; + TdThread pid; int threadId; int rows; int tables; @@ -107,16 +106,16 @@ void multiThread() { ThreadObj *threads = calloc((size_t)numOfThreads, sizeof(ThreadObj)); for (int i = 0; i < numOfThreads; i++) { ThreadObj *pthread = threads + i; - pthread_attr_t thattr; + TdThreadAttr thattr; pthread->threadId = i + 1; pthread->rows = numOfRows; pthread->tables = numOfTables; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - pthread_create(&pthread->pid, &thattr, (void *(*)(void *))execute, pthread); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadCreate(&pthread->pid, &thattr, (void *(*)(void *))execute, pthread); } for (int i = 0; i < numOfThreads; i++) { - pthread_join(threads[i].pid, NULL); + taosThreadJoin(threads[i].pid, NULL); } free(threads); } diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index 05f756ebb6..8d7dba2de2 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -74,7 +74,7 @@ BUILD_DIR=$TAOS_DIR/$BIN_DIR SIM_DIR=$TAOS_DIR/sim NODE_DIR=$SIM_DIR/$NODE_NAME -EXE_DIR=$BUILD_DIR/source/dnode/mgmt/daemon +EXE_DIR=$BUILD_DIR/source/dnode/mgmt/main CFG_DIR=$NODE_DIR/cfg LOG_DIR=$NODE_DIR/log DATA_DIR=$NODE_DIR/data diff --git a/tests/script/sh/massiveTable/compileVersion.sh b/tests/script/sh/massiveTable/compileVersion.sh index c6c92bf724..dd6382992a 100755 --- a/tests/script/sh/massiveTable/compileVersion.sh +++ b/tests/script/sh/massiveTable/compileVersion.sh @@ -68,7 +68,7 @@ gitPullBranchInfo $TDengineBrVer compileTDengineVersion taos_dir=${projectDir}/debug/tools/shell -taosd_dir=${projectDir}/debug/source/dnode/mgmt/daemon +taosd_dir=${projectDir}/debug/source/dnode/mgmt/main exec_process_dir=${projectDir}/debug/tests/test/c rm -f /usr/bin/taos diff --git a/tests/script/tsim/insert/basic0.sim b/tests/script/tsim/insert/basic0.sim index eb4780caac..6c24662be7 100644 --- a/tests/script/tsim/insert/basic0.sim +++ b/tests/script/tsim/insert/basic0.sim @@ -26,9 +26,10 @@ endi print =============== create child table sql create table ct1 using stb tags(1000) sql create table ct2 using stb tags(2000) +sql create table ct3 using stb tags(3000) sql show tables -if $rows != 2 then +if $rows != 3 then return -1 endi @@ -45,6 +46,8 @@ sql insert into ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 1 sql_error insert into ct1 values(now+4s, -14, -2.4, -3.4) ct2 values(now+4s, -14, -2.4, -3.4) sql_error insert into ct1 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) ct2 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) +sql insert into ct3 values('2021-01-01 00:00:00.000', 10, 2.0, 3.0) + #=================================================================== #=================================================================== print =============== query data from child table diff --git a/tests/script/tsim/insert/null.sim b/tests/script/tsim/insert/null.sim new file mode 100644 index 0000000000..9dcc435486 --- /dev/null +++ b/tests/script/tsim/insert/null.sim @@ -0,0 +1,358 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print =============== create database +sql create database d0 +sql show databases +if $rows != 1 then + return -1 +endi + +print $data00 $data01 $data02 + +sql use d0 + +print =============== create super table, include column type for count/sum/min/max/first +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double, c4 bigint) tags (t1 int unsigned) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +sql create table ct1 using stb tags(1000) +sql create table ct2 using stb tags(2000) +sql create table ct3 using stb tags(3000) + +sql show tables +if $rows != 3 then + return -1 +endi + +print =============== insert data, include NULL +sql insert into ct1 values (now+0s, 10, 2.0, 3.0, 90)(now+1s, NULL, NULL, NULL, NULL)(now+2s, NULL, 2.1, 3.1, 91)(now+3s, 11, NULL, 3.2, 92)(now+4s, 12, 2.2, NULL, 93)(now+5s, 13, 2.3, 3.3, NULL) +sql insert into ct1 values (now+6s, NULL, 2.4, 3.4, 94) +sql insert into ct1 values (now+7s, 14, NULL, 3.5, 95) +sql insert into ct1 values (now+8s, 15, 2.5, NULL, 96) +sql insert into ct1 values (now+9s, 16, 2.6, 3.6, NULL) +sql insert into ct1 values (now+10s, NULL, NULL, NULL, NULL) +sql insert into ct1 values (now+11s, -2147483648, 2.7, 3.7, 97) + +#=================================================================== +#=================================================================== +print =============== query data from child table +sql select * from ct1 +print ===> select * from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +if $rows != 12 then + return -1 +endi +if $data01 != 10 then + return -1 +endi +if $data02 != 2.00000 then + return -1 +endi +if $data03 != 3.000000000 then + return -1 +endi +#if $data41 != -14 then +# return -1 +#endi +#if $data42 != -2.40000 then +# return -1 +#endi +#if $data43 != -3.400000000 then +# return -1 +#endi + + +print =============== select count(*) from child table +sql select count(*) from ct1 +print ===> select count(*) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +if $rows != 1 then + return -1 +endi + +print $data00 $data01 $data02 +if $data00 != 4 then + return -1 +endi + +print =============== select count(column) from child table +sql select count(ts), count(c1), count(c2), count(c3) from ct1 +print ===> select count(ts), count(c1), count(c2), count(c3) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 + +if $data00 != 4 then + return -1 +endi +if $data01 != 4 then + return -1 +endi +if $data02 != 4 then + return -1 +endi +if $data03 != 4 then + return -1 +endi + +#print =============== select first(*)/first(column) from child table +#sql select first(*) from ct1 +#sql select first(ts), first(c1), first(c2), first(c3) from ct1 + +print =============== select min(column) from child table +sql select min(c1), min(c2), min(c3) from ct1 +print ===> select min(c1), min(c2), min(c3) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +if $rows != 1 then + return -1 +endi +if $data00 != 10 then + return -1 +endi +if $data01 != 2.00000 then + return -1 +endi +if $data02 != 3.000000000 then + return -1 +endi + +print =============== select max(column) from child table +sql select max(c1), max(c2), max(c3) from ct1 +print ===> select max(c1), max(c2), max(c3) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +if $rows != 1 then + return -1 +endi +if $data00 != 13 then + return -1 +endi +if $data01 != 2.30000 then + return -1 +endi +if $data02 != 3.300000000 then + return -1 +endi + +print =============== select sum(column) from child table +sql select sum(c1), sum(c2), sum(c3) from ct1 +print ===> select sum(c1), sum(c2), sum(c3) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +if $rows != 1 then + return -1 +endi +if $data00 != 46 then + return -1 +endi +if $data01 != 8.599999905 then + return -1 +endi +if $data02 != 12.600000000 then + return -1 +endi + +print =============== select column, from child table +sql select c1, c2, c3 from ct1 +print ===> select c1, c2, c3 from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +#if $rows != 4 then +# return -1 +#endi +#if $data00 != 10 then +# return -1 +#endi +#if $data01 != 2.00000 then +# return -1 +#endi +#if $data02 != 3.000000000 then +# return -1 +#endi +#if $data10 != 11 then +# return -1 +#endi +#if $data11 != 2.10000 then +# return -1 +#endi +#if $data12 != 3.100000000 then +# return -1 +#endi +#if $data30 != 13 then +# return -1 +#endi +#if $data31 != 2.30000 then +# return -1 +#endi +#if $data32 != 3.300000000 then +# return -1 +#endi +#=================================================================== +#=================================================================== + + +return + +#print =============== query data from stb +#sql select * from stb +#print ===> +#print ===> rows: $rows +#print ===> rows0: $data00 $data01 $data02 $data03 $data04 +#if $rows != 4 then +# return -1 +#endi +#print =============== select count(*) from supter table +#sql select count(*) from stb +#if $rows != 1 then +# return -1 +#endi +# +#print $data00 $data01 $data02 +#if $data00 != 8 then +# return -1 +#endi +# +#print =============== select count(column) from supter table +#sql select count(ts), count(c1), count(c2), count(c3) from stb +#print $data00 $data01 $data02 $data03 +#if $data00 != 8 then +# return -1 +#endi +#if $data01 != 8 then +# return -1 +#endi +#if $data02 != 8 then +# return -1 +#endi +#if $data03 != 8 then +# return -1 +#endi + + +#=================================================================== +#=================================================================== + +print =============== stop and restart taosd, then again do query above +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + +sleep 2000 +sql select * from ct1 +if $rows != 4 then # after fix bug, modify 4 to 7 + return -1 +endi +if $data01 != 10 then + return -1 +endi +if $data02 != 2.00000 then + return -1 +endi +if $data03 != 3.000000000 then + return -1 +endi +#if $data41 != -14 then +# return -1 +#endi +#if $data42 != -2.40000 then +# return -1 +#endi +#if $data43 != -3.400000000 then +# return -1 +#endi + + +print =============== select count(*) from child table +sql select count(*) from ct1 +if $rows != 1 then + return -1 +endi + +print $data00 $data01 $data02 +if $data00 != 4 then + return -1 +endi + +print =============== select count(column) from child table +sql select count(ts), count(c1), count(c2), count(c3) from ct1 +print $data00 $data01 $data02 $data03 +if $data00 != 4 then + return -1 +endi +if $data01 != 4 then + return -1 +endi +if $data02 != 4 then + return -1 +endi +if $data03 != 4 then + return -1 +endi + +#print =============== select first(*)/first(column) from child table +#sql select first(*) from ct1 +#sql select first(ts), first(c1), first(c2), first(c3) from ct1 + +print =============== select min(column) from child table +sql select min(c1), min(c2), min(c3) from ct1 +print $data00 $data01 $data02 $data03 +if $rows != 1 then + return -1 +endi +if $data00 != 10 then + return -1 +endi +if $data01 != 2.00000 then + return -1 +endi +if $data02 != 3.000000000 then + return -1 +endi + +print =============== select max(column) from child table +sql select max(c1), max(c2), max(c3) from ct1 +print $data00 $data01 $data02 $data03 +if $rows != 1 then + return -1 +endi +if $data00 != 13 then + return -1 +endi +if $data01 != 2.30000 then + return -1 +endi +if $data02 != 3.300000000 then + return -1 +endi + +print =============== select sum(column) from child table +sql select sum(c1), sum(c2), sum(c3) from ct1 +print $data00 $data01 $data02 $data03 +if $rows != 1 then + return -1 +endi +if $data00 != 46 then + return -1 +endi +if $data01 != 8.599999905 then + return -1 +endi +if $data02 != 12.600000000 then + return -1 +endi + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/interval-offset.sim b/tests/script/tsim/query/interval-offset.sim new file mode 100644 index 0000000000..a463d69fe5 --- /dev/null +++ b/tests/script/tsim/query/interval-offset.sim @@ -0,0 +1,365 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 500 +sql connect + +print =============== create database +sql drop database d0 -x step1 +step1: +sql create database d0 +sql show databases +if $rows != 1 then + return -1 +endi + +sql use d0 + +print =============== create super table and child table +sql create table stb (ts timestamp, tbcol int) tags (t1 int) +sql show stables +print $rows $data00 $data01 $data02 +if $rows != 1 then + return -1 +endi + +sql create table ct1 using stb tags ( 1 ) +sql create table ct2 using stb tags ( 2 ) +sql show tables +print $rows $data00 $data10 $data20 +if $rows != 2 then + return -1 +endi + +print =============== insert data into child table ct1 (s) +sql insert into ct1 values ( '2022-01-01 01:01:01.000', 1 ) +sql insert into ct1 values ( '2022-01-01 01:01:06.000', 2 ) +sql insert into ct1 values ( '2022-01-01 01:01:10.000', 3 ) +sql insert into ct1 values ( '2022-01-01 01:01:16.000', 4 ) +sql insert into ct1 values ( '2022-01-01 01:01:20.000', 5 ) +sql insert into ct1 values ( '2022-01-01 01:01:26.000', 6 ) +sql insert into ct1 values ( '2022-01-01 01:01:30.000', 7 ) +sql insert into ct1 values ( '2022-01-01 01:01:36.000', 8 ) + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +if $rows != 5 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data40 != 1 then + return -1 +endi + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) sliding(10s) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) sliding(10s) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +if $rows != 5 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data40 != 1 then + return -1 +endi + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) sliding(5s) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) sliding(5s) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +print ===> rows5: $data50 $data51 $data52 $data53 $data54 +print ===> rows6: $data60 $data61 $data62 $data63 $data64 +print ===> rows7: $data70 $data71 $data72 $data73 $data74 +if $rows != 8 then + return -1 +endi +if $data00 != 2 then + return -1 +endi +if $data70 != 1 then + return -1 +endi + +print =============== insert data into child table ct2 (d) +sql insert into ct2 values ( '2022-01-01 01:00:01.000', 1 ) +sql insert into ct2 values ( '2022-01-01 10:00:01.000', 2 ) +sql insert into ct2 values ( '2022-01-01 20:00:01.000', 3 ) +sql insert into ct2 values ( '2022-01-02 10:00:01.000', 4 ) +sql insert into ct2 values ( '2022-01-02 20:00:01.000', 5 ) +sql insert into ct2 values ( '2022-01-03 10:00:01.000', 6 ) +sql insert into ct2 values ( '2022-01-03 20:00:01.000', 7 ) + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 2 then + return -1 +endi + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) sliding(12h) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) sliding(12h) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +print ===> rows5: $data50 $data51 $data52 $data53 $data54 +print ===> rows6: $data60 $data61 $data62 $data63 $data64 +print ===> rows7: $data70 $data71 $data72 $data73 $data74 +if $rows != 7 then + return -1 +endi +if $data00 != 2 then + return -1 +endi +if $data60 != 1 then + return -1 +endi + +return + + + + + + + + + +sql select count(*) from car interval(1n, 10d) order by ts desc +# tdSql.checkData(0, 1, 1) +# tdSql.checkData(1, 1, 2) +# tdSql.checkData(2, 1, 3) +# tdSql.checkData(3, 1, 3) +# tdSql.checkData(4, 1, 6) +# tdSql.checkData(5, 1, 1) +# tdSql.checkData(6, 1, 1) +# +sql select count(*) from car interval(2n, 5d) +# tdSql.checkData(0, 1, 1) +# tdSql.checkData(1, 1, 1) +# tdSql.checkData(2, 1, 6) +# tdSql.checkData(3, 1, 6) +# tdSql.checkData(4, 1, 3) + +sql select count(*) from car interval(2n) order by ts desc +# tdSql.checkData(0, 1, 3) +# tdSql.checkData(1, 1, 6) +# tdSql.checkData(2, 1, 6) +# tdSql.checkData(3, 1, 1) +# tdSql.checkData(4, 1, 1) +# +sql select count(*) from car interval(1y, 1n) +# tdSql.checkData(0, 1, 1) +# tdSql.checkData(1, 1, 8) +# tdSql.checkData(2, 1, 8) +# +sql select count(*) from car interval(1y, 2n) +# tdSql.checkData(0, 1, 1) +# tdSql.checkData(1, 1, 11) +# tdSql.checkData(2, 1, 5) + +sql select count(*) from car where ts > '2019-05-14 00:00:00' interval(1y, 5d) +# tdSql.checkData(0, 1, 6) +# tdSql.checkData(1, 1, 9) + + + + + + + + + + + + +sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol int) + +print ====== start create child tables and insert data +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $mt tags( $i ) + + $x = 0 + while $x < $rowNum + $cc = $x * 60000 + $ms = 1601481600000 + $cc + + sql insert into $tb values ($ms , $x ) + $x = $x + 1 + endw + + $i = $i + 1 +endw + +print =============== step2 +$i = 1 +$tb = $tbPrefix . $i + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb interval(1m) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb interval(1m) +print ===> $rows $data01 $data05 +if $rows != $rowNum then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data04 != 1 then + return -1 +endi + +#print =============== step3 +#$cc = 4 * 60000 +#$ms = 1601481600000 + $cc +#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms interval(1m) +#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms interval(1m) +#print ===> $rows $data01 $data05 +#if $rows != 5 then +# return -1 +#endi +#if $data00 != 1 then +# return -1 +#endi +#if $data04 != 1 then +# return -1 +#endi + +#print =============== step4 +#$cc = 40 * 60000 +#$ms = 1601481600000 + $cc + +#$cc = 1 * 60000 +#$ms2 = 1601481600000 - $cc + +#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) +#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) +#print ===> $rows $data01 $data05 +#if $rows != 20 then +# return -1 +#endi +#if $data00 != 1 then +# return -1 +#endi +#if $data04 != 1 then +# return -1 +#endi + +#print =============== step5 +#$cc = 40 * 60000 +#$ms = 1601481600000 + $cc + +#$cc = 1 * 60000 +#$ms2 = 1601481600000 - $cc + +#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) fill(value,0) +#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) fill(value,0) +#print ===> $rows $data21 $data25 +#if $rows != 42 then +# return -1 +#endi +#if $data20 != 1 then +# return -1 +#endi +#if $data24 != 1 then +# return -1 +#endi + +#print =============== step6 +#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt interval(1m) +#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt interval(1m) +#print ===> $rows $data11 +#if $rows != 20 then +# return -1 +#endi +#if $data11 != 10 then +# return -1 +#endi + +#print =============== step7 +#$cc = 4 * 60000 +#$ms = 1601481600000 + $cc +#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms interval(1m) +#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms interval(1m) +#print ===> $rows $data11 +#if $rows != 5 then +# return -1 +#endi +#if $data11 != 10 then +# return -1 +#endi + +#print =============== step8 +#$cc = 40 * 60000 +#$ms1 = 1601481600000 + $cc +# +#$cc = 1 * 60000 +#$ms2 = 1601481600000 - $cc +# +#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) +#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) +#print ===> $rows $data11 +#if $rows != 20 then +# return -1 +#endi +#if $data11 != 10 then +# return -1 +#endi +# +#print =============== step9 +#$cc = 40 * 60000 +#$ms1 = 1601481600000 + $cc +# +#$cc = 1 * 60000 +#$ms2 = 1601481600000 - $cc +# +#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) fill(value, 0) +#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) fill(value, 0) +#print ===> $rows $data11 +#if $rows != 42 then +# return -1 +#endi +#if $data11 != 10 then +# return -1 +#endi + +print =============== clear +#sql drop database $db +#sql show databases +#if $rows != 0 then +# return -1 +#endi + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/query/interval.sim b/tests/script/tsim/query/interval.sim index 47be71bdc6..6dd0a9537e 100644 --- a/tests/script/tsim/query/interval.sim +++ b/tests/script/tsim/query/interval.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wal -v 1 system sh/exec.sh -n dnode1 -s start -sleep 2000 +sleep 500 sql connect $dbPrefix = m_in_db diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index 57fbd4c9c1..5e539780f4 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -49,7 +49,7 @@ typedef struct { int64_t startMs; int64_t maxDelay; int64_t minDelay; - pthread_t thread; + TdThread thread; } SThreadInfo; // void parseArgument(int32_t argc, char *argv[]); @@ -400,9 +400,9 @@ int32_t main(int32_t argc, char *argv[]) { pPrint("%d threads are spawned to create %" PRId64 " tables, offset is %" PRId64 " ", numOfThreads, numOfTables, startOffset); - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo)); // int64_t numOfTablesPerThread = numOfTables / numOfThreads; @@ -430,12 +430,12 @@ int32_t main(int32_t argc, char *argv[]) { pInfo[i].minDelay = INT64_MAX; strcpy(pInfo[i].dbName, dbName); strcpy(pInfo[i].stbName, stbName); - pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); + taosThreadCreate(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); } taosMsleep(300); for (int32_t i = 0; i < numOfThreads; i++) { - pthread_join(pInfo[i].thread, NULL); + taosThreadJoin(pInfo[i].thread, NULL); } int64_t maxDelay = 0; @@ -465,6 +465,6 @@ int32_t main(int32_t argc, char *argv[]) { numOfThreads, NC); } - pthread_attr_destroy(&thattr); + taosThreadAttrDestroy(&thattr); free(pInfo); } diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 609f8d6b69..08e49a7efe 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -306,8 +306,9 @@ int32_t init_env() { } //const char* sql = "select * from tu1"; - sprintf(sqlStr, "select * from %s%d", g_stConfInfo.stbName, 0); - pRes = tmq_create_topic(pConn, "test_stb_topic_1", sqlStr, strlen(sqlStr)); + sprintf(sqlStr, "create topic test_stb_topic_1 as select * from %s%d", g_stConfInfo.stbName, 0); + /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sqlStr, strlen(sqlStr));*/ + pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/tests/tsim/inc/simInt.h b/tests/tsim/inc/simInt.h index 24619a4ad5..1e2190e308 100644 --- a/tests/tsim/inc/simInt.h +++ b/tests/tsim/inc/simInt.h @@ -143,7 +143,7 @@ typedef struct _script_t { char *optionBuffer; SCmdLine *lines; // command list SVariable variables[MAX_VAR_LEN]; - pthread_t bgPid; + TdThread bgPid; char auth[128]; struct _script_t *bgScripts[MAX_BACKGROUND_SCRIPT_NUM]; } SScript; diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 1857e21753..855705e904 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -293,7 +293,7 @@ bool simExecuteRunBackCmd(SScript *script, char *option) { script->bgScripts[script->bgScriptLen++] = newScript; simInfo("script:%s, start to execute in background,", newScript->fileName); - if (pthread_create(&newScript->bgPid, NULL, simExecuteScript, (void *)newScript) != 0) { + if (taosThreadCreate(&newScript->bgPid, NULL, simExecuteScript, (void *)newScript) != 0) { sprintf(script->error, "lineNum:%d. create background thread failed", script->lines[script->linePos].lineNum); return false; } else { diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index bfbfe2b0df..eddcd3afc2 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -55,7 +55,7 @@ void simFreeScript(SScript *script) { simDebug("script:%s, is background script, set stop flag", bgScript->fileName); bgScript->killed = true; if (taosCheckPthreadValid(bgScript->bgPid)) { - pthread_join(bgScript->bgPid, NULL); + taosThreadJoin(bgScript->bgPid, NULL); } simDebug("script:%s, background thread joined", bgScript->fileName); diff --git a/tools/shell/src/backup/shellCheck.c b/tools/shell/src/backup/shellCheck.c index 33d25b6746..919cb0d846 100644 --- a/tools/shell/src/backup/shellCheck.c +++ b/tools/shell/src/backup/shellCheck.c @@ -31,7 +31,7 @@ static int32_t checkedNum = 0; static int32_t errorNum = 0; typedef struct { - pthread_t threadID; + TdThread threadID; int threadIndex; int totalThreads; void * taos; @@ -152,7 +152,7 @@ static void *shellCheckThreadFp(void *arg) { } static void shellRunCheckThreads(TAOS *con, SShellArguments *_args) { - pthread_attr_t thattr; + TdThreadAttr thattr; ShellThreadObj *threadObj = (ShellThreadObj *)calloc(_args->threadNum, sizeof(ShellThreadObj)); for (int t = 0; t < _args->threadNum; ++t) { ShellThreadObj *pThread = threadObj + t; @@ -161,17 +161,17 @@ static void shellRunCheckThreads(TAOS *con, SShellArguments *_args) { pThread->taos = con; pThread->db = _args->database; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&(pThread->threadID), &thattr, shellCheckThreadFp, (void *)pThread) != 0) { + if (taosThreadCreate(&(pThread->threadID), &thattr, shellCheckThreadFp, (void *)pThread) != 0) { fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex); exit(0); } } for (int t = 0; t < _args->threadNum; ++t) { - pthread_join(threadObj[t].threadID, NULL); + taosThreadJoin(threadObj[t].threadID, NULL); } for (int t = 0; t < _args->threadNum; ++t) { diff --git a/tools/shell/src/backup/shellDarwin.c b/tools/shell/src/backup/shellDarwin.c index d7a976d52c..69c7a7bc4e 100644 --- a/tools/shell/src/backup/shellDarwin.c +++ b/tools/shell/src/backup/shellDarwin.c @@ -365,7 +365,7 @@ void *shellLoopQuery(void *arg) { setThreadName("shellLoopQuery"); - pthread_cleanup_push(cleanup_handler, NULL); + taosThreadCleanupPush(cleanup_handler, NULL); char *command = malloc(MAX_COMMAND_SIZE); if (command == NULL){ @@ -389,7 +389,7 @@ void *shellLoopQuery(void *arg) { tfree(command); exitShell(); - pthread_cleanup_pop(1); + taosThreadCleanupPop(1); return NULL; } diff --git a/tools/shell/src/backup/shellImport.c b/tools/shell/src/backup/shellImport.c index b42f77e87e..398bcf2280 100644 --- a/tools/shell/src/backup/shellImport.c +++ b/tools/shell/src/backup/shellImport.c @@ -28,7 +28,7 @@ static int32_t shellSQLFileNum = 0; static char shellTablesSQLFile[TSDB_FILENAME_LEN] = {0}; typedef struct { - pthread_t threadID; + TdThread threadID; int threadIndex; int totalThreads; void *taos; @@ -232,7 +232,7 @@ void* shellImportThreadFp(void *arg) static void shellRunImportThreads(SShellArguments* _args) { - pthread_attr_t thattr; + TdThreadAttr thattr; ShellThreadObj *threadObj = (ShellThreadObj *)calloc(_args->threadNum, sizeof(ShellThreadObj)); for (int t = 0; t < _args->threadNum; ++t) { ShellThreadObj *pThread = threadObj + t; @@ -244,17 +244,17 @@ static void shellRunImportThreads(SShellArguments* _args) exit(0); } - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&(pThread->threadID), &thattr, shellImportThreadFp, (void*)pThread) != 0) { + if (taosThreadCreate(&(pThread->threadID), &thattr, shellImportThreadFp, (void*)pThread) != 0) { fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex); exit(0); } } for (int t = 0; t < _args->threadNum; ++t) { - pthread_join(threadObj[t].threadID, NULL); + taosThreadJoin(threadObj[t].threadID, NULL); } for (int t = 0; t < _args->threadNum; ++t) { diff --git a/tools/shell/src/backup/tnettest.c b/tools/shell/src/backup/tnettest.c index 772d92d8c6..ee32bfb6be 100644 --- a/tools/shell/src/backup/tnettest.c +++ b/tools/shell/src/backup/tnettest.c @@ -519,7 +519,7 @@ static void taosNetTestServer(char *host, int32_t startPort, int32_t pkgLen) { int32_t num = endPort - startPort + 1; if (num < 0) num = 1; - pthread_t *pids = malloc(2 * num * sizeof(pthread_t)); + TdThread *pids = malloc(2 * num * sizeof(TdThread)); STestInfo *tinfos = malloc(num * sizeof(STestInfo)); STestInfo *uinfos = malloc(num * sizeof(STestInfo)); @@ -528,7 +528,7 @@ static void taosNetTestServer(char *host, int32_t startPort, int32_t pkgLen) { tcpInfo->port = port + i; tcpInfo->pktLen = pkgLen; - if (pthread_create(pids + i, NULL, taosNetBindTcpPort, tcpInfo) != 0) { + if (taosThreadCreate(pids + i, NULL, taosNetBindTcpPort, tcpInfo) != 0) { uInfo("failed to create TCP test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port); exit(-1); } @@ -536,15 +536,15 @@ static void taosNetTestServer(char *host, int32_t startPort, int32_t pkgLen) { STestInfo *udpInfo = uinfos + i; udpInfo->port = port + i; tcpInfo->pktLen = pkgLen; - if (pthread_create(pids + num + i, NULL, taosNetBindUdpPort, udpInfo) != 0) { + if (taosThreadCreate(pids + num + i, NULL, taosNetBindUdpPort, udpInfo) != 0) { uInfo("failed to create UDP test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port); exit(-1); } } for (int32_t i = 0; i < num; i++) { - pthread_join(pids[i], NULL); - pthread_join(pids[(num + i)], NULL); + taosThreadJoin(pids[i], NULL); + taosThreadJoin(pids[(num + i)], NULL); } } diff --git a/tools/shell/src/shellLinux.c b/tools/shell/src/shellLinux.c index 6da05f28df..de5db0b288 100644 --- a/tools/shell/src/shellLinux.c +++ b/tools/shell/src/shellLinux.c @@ -395,7 +395,7 @@ void *shellLoopQuery(void *arg) { setThreadName("shellLoopQuery"); - pthread_cleanup_push(cleanup_handler, NULL); + taosThreadCleanupPush(cleanup_handler, NULL); char *command = malloc(MAX_COMMAND_SIZE); if (command == NULL){ @@ -419,7 +419,7 @@ void *shellLoopQuery(void *arg) { tfree(command); exitShell(); - pthread_cleanup_pop(1); + taosThreadCleanupPop(1); return NULL; } diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 2832855517..574e3fa8b8 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -17,7 +17,7 @@ #include "shell.h" #include "tglobal.h" -pthread_t pid; +TdThread pid; static tsem_t cancelSem; void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) { @@ -140,8 +140,8 @@ int main(int argc, char *argv[]) { exit(EXIT_FAILURE); } - pthread_t spid; - pthread_create(&spid, NULL, cancelHandler, NULL); + TdThread spid; + taosThreadCreate(&spid, NULL, cancelHandler, NULL); /* Interrupt handler. */ taosSetSignal(SIGTERM, shellQueryInterruptHandler); @@ -154,7 +154,7 @@ int main(int argc, char *argv[]) { /* Loop to query the input. */ while (1) { - pthread_create(&pid, NULL, shellLoopQuery, con); - pthread_join(pid, NULL); + taosThreadCreate(&pid, NULL, shellLoopQuery, con); + taosThreadJoin(pid, NULL); } }